You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2014/11/27 12:25:18 UTC
svn commit: r1642092 [1/2] - in /jackrabbit/oak/branches/1.0: ./
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/
oak-lucene/src/test/java/org/apache/...
Author: chetanm
Date: Thu Nov 27 11:25:17 2014
New Revision: 1642092
URL: http://svn.apache.org/r1642092
Log:
OAK-2261 - Enable support for NodeType based indexing rules
Merging revisions 1638779,1638780,1638781,1638782,1638783,1639260,1639577,1639963,1639973,1640134,1640143,1640555,1640556,1640694
Added:
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java
- copied, changed from r1638779, jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java
Modified:
jackrabbit/oak/branches/1.0/ (props changed)
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/RelativeProperty.java
jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java
jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java
Propchange: jackrabbit/oak/branches/1.0/
------------------------------------------------------------------------------
Merged /jackrabbit/oak/trunk:r1638779-1638783,1639260,1639577,1639963,1639973,1640134,1640143,1640555-1640556,1640694
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1642092&r1=1642091&r2=1642092&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java Thu Nov 27 11:25:17 2014
@@ -19,38 +19,66 @@
package org.apache.jackrabbit.oak.plugins.index.lucene;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Pattern;
import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import javax.jcr.PropertyType;
+import javax.jcr.RepositoryException;
+import javax.jcr.nodetype.NodeTypeIterator;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
-import com.google.common.primitives.Ints;
+import com.google.common.collect.Sets;
+import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.api.Type;
-import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper;
+import org.apache.jackrabbit.oak.namepath.NamePathMapper;
+import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
+import org.apache.jackrabbit.oak.plugins.nodetype.ReadOnlyNodeTypeManager;
+import org.apache.jackrabbit.oak.plugins.tree.ImmutableTree;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.util.TreeUtil;
import org.apache.lucene.codecs.Codec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Lists.newArrayListWithCapacity;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
+import static org.apache.jackrabbit.JcrConstants.NT_BASE;
+import static org.apache.jackrabbit.oak.api.Type.NAMES;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.DECLARING_NODE_TYPES;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.ENTRY_COUNT_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_COUNT;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.BLOB_SIZE;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXCLUDE_PROPERTY_NAMES;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXPERIMENTAL_STORAGE;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.FIELD_BOOST;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.FULL_TEXT_ENABLED;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_NAMES;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ORDERED_PROP_NAMES;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_NODE;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.PropertyDefinition.DEFAULT_BOOST;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.util.ConfigUtil.getOptionalValue;
+import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.apache.jackrabbit.oak.plugins.nodetype.NodeTypeConstants.NODE_TYPES_PATH;
+import static org.apache.jackrabbit.oak.plugins.tree.TreeConstants.OAK_CHILD_ORDER;
class IndexDefinition {
private static final Logger log = LoggerFactory.getLogger(IndexDefinition.class);
@@ -66,36 +94,30 @@ class IndexDefinition {
*/
static final long DEFAULT_ENTRY_COUNT = 1000;
- private final int propertyTypes;
+ private static String TYPES_ALLOW_ALL_NAME = "all";
- private final Set<String> excludes;
+ private static final int TYPES_ALLOW_NONE = PropertyType.UNDEFINED;
- private final Set<String> includes;
-
- private final Set<String> orderedProps;
-
- private final Set<String> declaringNodeTypes;
+ private static final int TYPES_ALLOW_ALL = -1;
private final boolean fullTextEnabled;
- private final boolean storageEnabled;
+ private final boolean propertyIndexEnabled;
private final NodeState definition;
private final NodeState root;
- private final Map<String, PropertyDefinition> propDefns;
-
private final String funcName;
private final int blobSize;
private final Codec codec;
- private final Map<String,RelativeProperty> relativeProps;
-
private final Set<String> relativePropNames;
+ private final List<RelativeProperty> relativeProperties;
+
private final int relativePropsMaxLevels;
/**
@@ -104,39 +126,40 @@ class IndexDefinition {
*/
private final long entryCount;
+ /**
+ * The {@link IndexingRule}s inside this configuration. Keys being the NodeType names
+ */
+ private final Map<String, List<IndexingRule>> indexRules;
+
+ private final List<IndexingRule> definedRules;
+
+ private final String indexName;
+
public IndexDefinition(NodeState root, NodeState defn) {
+ this(root, defn, null);
+ }
+
+ public IndexDefinition(NodeState root, NodeState defn, @Nullable String indexPath) {
this.root = root;
this.definition = defn;
- PropertyState pst = defn.getProperty(INCLUDE_PROPERTY_TYPES);
- if (pst != null) {
- int types = 0;
- for (String inc : pst.getValue(Type.STRINGS)) {
- try {
- types |= 1 << PropertyType.valueFromName(inc);
- } catch (IllegalArgumentException e) {
- log.warn("Unknown property type: " + inc);
- }
- }
- this.propertyTypes = types;
- } else {
- this.propertyTypes = -1;
+ this.indexName = determineIndexName(defn, indexPath);
+ this.blobSize = getOptionalValue(defn, BLOB_SIZE, DEFAULT_BLOB_SIZE);
+
+ NodeState rulesState = defn.getChildNode(LuceneIndexConstants.INDEX_RULES);
+ if (!rulesState.exists()){
+ rulesState = createIndexRules(defn).getNodeState();
}
- this.excludes = toLowerCase(getMultiProperty(defn, EXCLUDE_PROPERTY_NAMES));
- this.includes = getMultiProperty(defn, INCLUDE_PROPERTY_NAMES);
- this.orderedProps = getMultiProperty(defn, ORDERED_PROP_NAMES);
- this.declaringNodeTypes = getMultiProperty(defn, DECLARING_NODE_TYPES);
+ List<IndexingRule> definedIndexRules = newArrayList();
+ this.indexRules = collectIndexRules(rulesState, definedIndexRules);
+ this.definedRules = ImmutableList.copyOf(definedIndexRules);
- this.blobSize = getOptionalValue(defn, BLOB_SIZE, DEFAULT_BLOB_SIZE);
+ this.relativeProperties = collectRelativeProperties(definedIndexRules);
+ this.fullTextEnabled = hasFulltextEnabledIndexRule(definedIndexRules);
+ this.propertyIndexEnabled = hasPropertyIndexEnabledIndexRule(definedIndexRules);
- this.fullTextEnabled = getOptionalValue(defn, FULL_TEXT_ENABLED, true);
- //Storage is disabled for non full text indexes
- this.storageEnabled = this.fullTextEnabled && getOptionalValue(defn, EXPERIMENTAL_STORAGE, true);
- //TODO Flag out invalid propertyNames like one which are absolute
- this.relativeProps = collectRelativeProps(Iterables.concat(includes, orderedProps));
- this.propDefns = collectPropertyDefns(defn);
- this.relativePropNames = collectRelPropertyNames(this.relativeProps.values());
- this.relativePropsMaxLevels = getRelPropertyMaxLevels(this.relativeProps.values());
+ this.relativePropNames = collectRelPropertyNames(relativeProperties);
+ this.relativePropsMaxLevels = getRelPropertyMaxLevels(relativeProperties);
String functionName = getOptionalValue(defn, LuceneIndexConstants.FUNC_NAME, null);
this.funcName = functionName != null ? "native*" + functionName : null;
@@ -150,66 +173,12 @@ class IndexDefinition {
}
}
- boolean includeProperty(String name) {
- if(!includes.isEmpty()){
- return includes.contains(name);
- }
- return !excludes.contains(name.toLowerCase());
- }
-
- boolean includePropertyType(int type){
- if(propertyTypes < 0){
- return false;
- }
- return (propertyTypes & (1 << type)) != 0;
- }
-
- boolean isOrdered(String name) {
- return orderedProps.contains(name);
- }
-
- public NodeState getDefinition() {
- return definition;
- }
-
public boolean isFullTextEnabled() {
return fullTextEnabled;
}
- public int getPropertyTypes() {
- return propertyTypes;
- }
-
- public Set<String> getDeclaringNodeTypes() {
- return declaringNodeTypes;
- }
-
- public boolean hasDeclaredNodeTypes(){
- return !declaringNodeTypes.isEmpty();
- }
- /**
- * Checks if a given property should be stored in the lucene index or not
- */
- public boolean isStored(String name) {
- return storageEnabled;
- }
-
- public boolean skipTokenization(String propertyName) {
- //If fulltext is not enabled then we never tokenize
- //irrespective of property name
- if (!isFullTextEnabled()) {
- return true;
- }
- return LuceneIndexHelper.skipTokenization(propertyName);
- }
-
- @CheckForNull
- public PropertyDefinition getPropDefn(String propName){
- return propDefns.get(propName);
- }
-
- public boolean hasPropertyDefinition(String propName){
- return propDefns.containsKey(propName);
+ public boolean isPropertyIndexEnabled() {
+ return propertyIndexEnabled;
}
public String getFunctionName(){
@@ -244,7 +213,7 @@ class IndexDefinition {
}
public Collection<RelativeProperty> getRelativeProps() {
- return relativeProps.values();
+ return relativeProperties;
}
/**
@@ -257,7 +226,7 @@ class IndexDefinition {
*/
public void collectRelPropsForName(String name, Collection<RelativeProperty> relProps){
if(hasRelativeProperty(name)){
- for(RelativeProperty rp : relativeProps.values()){
+ for(RelativeProperty rp : relativeProperties){
if(rp.name.equals(name)){
relProps.add(rp);
}
@@ -266,51 +235,19 @@ class IndexDefinition {
}
boolean hasRelativeProperties(){
- return !relativePropNames.isEmpty();
+ return !relativeProperties.isEmpty();
}
boolean hasRelativeProperty(String name) {
return relativePropNames.contains(name);
}
- //~------------------------------------------< Internal >
-
- private Map<String, RelativeProperty> collectRelativeProps(Iterable<String> propNames) {
- Map<String, RelativeProperty> relProps = newHashMap();
- for (String propName : propNames) {
- if (RelativeProperty.isRelativeProperty(propName)) {
- relProps.put(propName, new RelativeProperty(propName));
- }
- }
- return ImmutableMap.copyOf(relProps);
- }
-
- private Map<String, PropertyDefinition> collectPropertyDefns(NodeState defn) {
- Map<String, PropertyDefinition> propDefns = newHashMap();
- NodeState propNode = defn.getChildNode(LuceneIndexConstants.PROP_NODE);
- //Include all immediate child nodes to 'properties' node by default
- for (String propName : Iterables.concat(includes, orderedProps, propNode.getChildNodeNames())) {
- NodeState propDefnNode;
- if (relativeProps.containsKey(propName)) {
- propDefnNode = relativeProps.get(propName).getPropDefnNode(propNode);
- } else {
- propDefnNode = propNode.getChildNode(propName);
- }
-
- if (propDefnNode.exists() && !propDefns.containsKey(propName)) {
- propDefns.put(propName, new PropertyDefinition(this, propName, propDefnNode));
- }
- }
- return ImmutableMap.copyOf(propDefns);
+ @Override
+ public String toString() {
+ return "IndexDefinition : " + indexName;
}
- private Set<String> collectRelPropertyNames(Collection<RelativeProperty> props) {
- Set<String> propNames = newHashSet();
- for (RelativeProperty prop : props) {
- propNames.add(prop.name);
- }
- return ImmutableSet.copyOf(propNames);
- }
+ //~------------------------------------------< Internal >
private int getRelPropertyMaxLevels(Collection<RelativeProperty> props) {
int max = -1;
@@ -335,19 +272,553 @@ class IndexDefinition {
return codec;
}
- private static boolean getOptionalValue(NodeState definition, String propName, boolean defaultVal){
- PropertyState ps = definition.getProperty(propName);
- return ps == null ? defaultVal : ps.getValue(Type.BOOLEAN);
+ //~---------------------------------------------------< IndexRule >
+
+
+ public List<IndexingRule> getDefinedRules() {
+ return definedRules;
+ }
+
+ @CheckForNull
+ public IndexingRule getApplicableIndexingRule(String primaryNodeType) {
+ //This method would be invoked for every node. So be as
+ //conservative as possible in object creation
+ List<IndexingRule> rules = null;
+ List<IndexingRule> r = indexRules.get(primaryNodeType);
+ if (r != null) {
+ rules = new ArrayList<IndexingRule>();
+ rules.addAll(r);
+ }
+
+ if (rules != null) {
+ for (IndexingRule rule : rules) {
+ if (rule.appliesTo(primaryNodeType)) {
+ return rule;
+ }
+ }
+ }
+
+ // no applicable rule
+ return null;
}
- private static int getOptionalValue(NodeState definition, String propName, int defaultVal){
- PropertyState ps = definition.getProperty(propName);
- return ps == null ? defaultVal : Ints.checkedCast(ps.getValue(Type.LONG));
+ /**
+ * Returns the first indexing rule that applies to the given node
+ * <code>state</code>.
+ *
+ * @param state a node state.
+ * @return the indexing rule or <code>null</code> if none applies.
+ */
+ @CheckForNull
+ public IndexingRule getApplicableIndexingRule(Tree state) {
+ //This method would be invoked for every node. So be as
+ //conservative as possible in object creation
+ List<IndexingRule> rules = null;
+ List<IndexingRule> r = indexRules.get(getPrimaryTypeName(state));
+ if (r != null) {
+ rules = new ArrayList<IndexingRule>();
+ rules.addAll(r);
+ }
+
+ for (String name : getMixinTypeNames(state)) {
+ r = indexRules.get(name);
+ if (r != null) {
+ if (rules == null) {
+ rules = new ArrayList<IndexingRule>();
+ }
+ rules.addAll(r);
+ }
+ }
+
+ if (rules != null) {
+ for (IndexingRule rule : rules) {
+ if (rule.appliesTo(state)) {
+ return rule;
+ }
+ }
+ }
+
+ // no applicable rule
+ return null;
}
- private static String getOptionalValue(NodeState definition, String propName, String defaultVal){
- PropertyState ps = definition.getProperty(propName);
- return ps == null ? defaultVal : ps.getValue(Type.STRING);
+ private Map<String, List<IndexingRule>> collectIndexRules(NodeState indexRules,
+ List<IndexingRule> definedIndexRules){
+ //TODO if a rule is defined for nt:base then this map would have entry for each
+ //registered nodeType in the system
+
+ if (!indexRules.exists()) {
+ return Collections.emptyMap();
+ }
+
+ if (!hasOrderableChildren(indexRules)){
+ log.warn("IndexRule node does not have orderable children in [{}]", IndexDefinition.this);
+ }
+
+ Map<String, List<IndexingRule>> nt2rules = newHashMap();
+ ReadOnlyNodeTypeManager ntReg = createNodeTypeManager(new ImmutableTree(root));
+
+ //Use Tree API to read ordered child nodes
+ ImmutableTree ruleTree = new ImmutableTree(indexRules);
+ final List<String> allNames = getAllNodeTypes(ntReg);
+ for (Tree ruleEntry : ruleTree.getChildren()) {
+ IndexingRule rule = new IndexingRule(ruleEntry.getName(), indexRules.getChildNode(ruleEntry.getName()));
+ definedIndexRules.add(rule);
+
+ // register under node type and all its sub types
+ log.debug("Found rule '{}' for NodeType '{}'", rule, rule.getNodeTypeName());
+
+ List<String> ntNames = allNames;
+ if (!rule.inherited){
+ //Trim the list to rule's nodeType so that inheritance check
+ //is not performed for other nodeTypes
+ ntNames = Collections.singletonList(rule.getNodeTypeName());
+ }
+
+ for (String ntName : ntNames) {
+ if (ntReg.isNodeType(ntName, rule.getNodeTypeName())) {
+ List<IndexingRule> perNtConfig = nt2rules.get(ntName);
+ if (perNtConfig == null) {
+ perNtConfig = new ArrayList<IndexingRule>();
+ nt2rules.put(ntName, perNtConfig);
+ }
+ log.debug("Registering it for name '{}'", ntName);
+ perNtConfig.add(new IndexingRule(rule, ntName));
+ }
+ }
+ }
+
+ for (Map.Entry<String, List<IndexingRule>> e : nt2rules.entrySet()){
+ e.setValue(ImmutableList.copyOf(e.getValue()));
+ }
+
+ return ImmutableMap.copyOf(nt2rules);
+ }
+
+ public class IndexingRule {
+ private final String baseNodeType;
+ private final String nodeTypeName;
+ private final Map<String, PropertyDefinition> propConfigs;
+ private final List<NamePattern> namePatterns;
+ final float boost;
+ final boolean inherited;
+ final boolean defaultFulltextEnabled;
+ final boolean defaultStorageEnabled;
+ final int propertyTypes;
+ final boolean fulltextEnabled;
+ final boolean propertyIndexEnabled;
+
+ //TODO To be relooked with support for aggregation
+ final Map<String,RelativeProperty> relativeProps;
+ final Set<String> relativePropNames;
+
+
+ IndexingRule(String nodeTypeName, NodeState config) {
+ this.nodeTypeName = nodeTypeName;
+ this.baseNodeType = nodeTypeName;
+ this.boost = getOptionalValue(config, FIELD_BOOST, DEFAULT_BOOST);
+ this.inherited = getOptionalValue(config, LuceneIndexConstants.RULE_INHERITED, true);
+ this.defaultFulltextEnabled = getOptionalValue(config, LuceneIndexConstants.FULL_TEXT_ENABLED, false);
+ //TODO Provide a new proper propertyName for enabling storage
+ this.defaultStorageEnabled = getOptionalValue(config, LuceneIndexConstants.EXPERIMENTAL_STORAGE, false);
+ this.propertyTypes = getSupportedTypes(config, TYPES_ALLOW_ALL);
+
+ List<NamePattern> namePatterns = newArrayList();
+ Map<String,RelativeProperty> relativeProps = newHashMap();
+ this.propConfigs = collectPropConfigs(config, namePatterns, relativeProps);
+
+ this.namePatterns = ImmutableList.copyOf(namePatterns);
+ this.fulltextEnabled = hasAnyFullTextEnabledProperty();
+ this.propertyIndexEnabled = hasAnyPropertyIndexConfigured();
+ this.relativeProps = ImmutableMap.copyOf(relativeProps);
+ this.relativePropNames = collectRelPropertyNames(this.relativeProps.values());
+ }
+
+ /**
+ * Creates a new indexing rule base on an existing one, but for a
+ * different node type name.
+ *
+ * @param original the existing rule.
+ * @param nodeTypeName the node type name for the rule.
+ */
+ IndexingRule(IndexingRule original, String nodeTypeName) {
+ this.nodeTypeName = nodeTypeName;
+ this.baseNodeType = original.getNodeTypeName();
+ this.propConfigs = original.propConfigs;
+ this.namePatterns = original.namePatterns;
+ this.boost = original.boost;
+ this.defaultFulltextEnabled = original.defaultFulltextEnabled;
+ this.defaultStorageEnabled = original.defaultStorageEnabled;
+ this.inherited = original.inherited;
+ this.propertyTypes = original.propertyTypes;
+ this.fulltextEnabled = original.fulltextEnabled;
+ this.relativeProps = original.relativeProps;
+ this.relativePropNames = original.relativePropNames;
+ this.propertyIndexEnabled = original.propertyIndexEnabled;
+ }
+
+ /**
+ * Returns <code>true</code> if the property with the given name is
+ * indexed according to this rule.
+ *
+ * @param propertyName the name of a property.
+ * @return <code>true</code> if the property is indexed;
+ * <code>false</code> otherwise.
+ */
+ public boolean isIndexed(String propertyName) {
+ return getConfig(propertyName) != null;
+ }
+
+
+ /**
+ * Returns the name of the node type where this rule applies to.
+ *
+ * @return name of the node type.
+ */
+ public String getNodeTypeName() {
+ return nodeTypeName;
+ }
+
+ @Override
+ public String toString() {
+ String str = "IndexRule: "+ nodeTypeName;
+ if (!baseNodeType.equals(nodeTypeName)){
+ str += "(" + baseNodeType + ")";
+ }
+ return str;
+ }
+
+ /**
+ * Returns <code>true</code> if this rule applies to the given node
+ * <code>state</code>.
+ *
+ * @param state the state to check.
+ * @return <code>true</code> the rule applies to the given node;
+ * <code>false</code> otherwise.
+ */
+ public boolean appliesTo(Tree state) {
+ if (!nodeTypeName.equals(getPrimaryTypeName(state))) {
+ return false;
+ }
+ //TODO Add support for condition
+ //return condition == null || condition.evaluate(state);
+ return true;
+ }
+
+ public boolean appliesTo(String nodeTypeName) {
+ if (!this.nodeTypeName.equals(nodeTypeName)) {
+ return false;
+ }
+
+ //TODO Once condition support is done return false
+ //return condition == null || condition.evaluate(state);
+ return true;
+
+ }
+
+ public boolean isFulltextEnabled() {
+ return fullTextEnabled;
+ }
+ /**
+ * @param propertyName name of a property.
+ * @return the property configuration or <code>null</code> if this
+ * indexing rule does not contain a configuration for the given
+ * property.
+ */
+ @CheckForNull
+ public PropertyDefinition getConfig(String propertyName) {
+ PropertyDefinition config = propConfigs.get(propertyName);
+ if (config != null) {
+ return config;
+ } else if (namePatterns.size() > 0) {
+ // check patterns
+ for (NamePattern np : namePatterns) {
+ if (np.matches(propertyName)) {
+ return np.getConfig();
+ }
+ }
+ }
+ return null;
+ }
+
+ public boolean includePropertyType(int type){
+ //TODO Rules related to type inclusion need to be synced
+ //Login IndexEditor evaluates differently compared to IndexDefinition
+ if(propertyTypes == TYPES_ALLOW_ALL){
+ return true;
+ }
+
+ if (propertyTypes == TYPES_ALLOW_NONE){
+ return false;
+ }
+
+ return (propertyTypes & (1 << type)) != 0;
+ }
+
+ public Collection<RelativeProperty> getRelativeProps() {
+ return relativeProps.values();
+ }
+
+ private Map<String, PropertyDefinition> collectPropConfigs(NodeState config, List<NamePattern> patterns,
+ Map<String,RelativeProperty> relativeProps) {
+ Map<String, PropertyDefinition> propDefns = newHashMap();
+ NodeState propNode = config.getChildNode(LuceneIndexConstants.PROP_NODE);
+
+ if (!propNode.exists()){
+ return Collections.emptyMap();
+ }
+
+ if (!hasOrderableChildren(propNode)){
+ log.warn("Properties node for [{}] does not have orderable " +
+ "children in [{}]", this, IndexDefinition.this);
+ }
+
+ //Include all immediate child nodes to 'properties' node by default
+ Tree propTree = new ImmutableTree(propNode);
+ for (Tree prop : propTree.getChildren()) {
+ String propName = prop.getName();
+ NodeState propDefnNode = propNode.getChildNode(propName);
+ if (propDefnNode.exists() && !propDefns.containsKey(propName)) {
+ PropertyDefinition pd = new PropertyDefinition(this, propName, propDefnNode);
+ if(pd.isRegexp){
+ patterns.add(new NamePattern(pd.name, pd));
+ } else {
+ propDefns.put(pd.name, pd);
+ }
+
+ if (RelativeProperty.isRelativeProperty(pd.name)){
+ relativeProps.put(pd.name, new RelativeProperty(pd.name, pd));
+ }
+ }
+ }
+ return ImmutableMap.copyOf(propDefns);
+ }
+
+ private boolean hasAnyFullTextEnabledProperty() {
+ for (PropertyDefinition pd : propConfigs.values()){
+ if (pd.fulltextEnabled()){
+ return true;
+ }
+ }
+
+ for (NamePattern np : namePatterns){
+ if (np.getConfig().fulltextEnabled()){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean hasAnyPropertyIndexConfigured() {
+ for (PropertyDefinition pd : propConfigs.values()){
+ if (pd.propertyIndex){
+ return true;
+ }
+ }
+
+ for (NamePattern np : namePatterns){
+ if (np.getConfig().propertyIndex){
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
+ /**
+ * A property name pattern.
+ */
+ private static final class NamePattern {
+ /**
+ * The pattern to match.
+ */
+ private final Pattern pattern;
+
+ /**
+ * The associated configuration.
+ */
+ private final PropertyDefinition config;
+
+ /**
+ * Creates a new name pattern.
+ *
+ * @param pattern the pattern as defined by the property definition
+ * @param config the associated configuration.
+ */
+ private NamePattern(String pattern,
+ PropertyDefinition config){
+
+ this.pattern = Pattern.compile(pattern);
+ this.config = config;
+ }
+
+ /**
+ * @param propertyPath property name to match
+ * @return <code>true</code> if <code>property name</code> matches this name
+ * pattern; <code>false</code> otherwise.
+ */
+ boolean matches(String propertyPath) {
+ return pattern.matcher(propertyPath).matches();
+ }
+
+ PropertyDefinition getConfig() {
+ return config;
+ }
+ }
+
+ //~---------------------------------------------< compatibility >
+
+ public void updateDefinition(NodeBuilder indexDefn){
+ //TODO Remove existing config once transformed config is persisted
+ NodeState rulesState = definition.getChildNode(LuceneIndexConstants.INDEX_RULES);
+ if (!rulesState.exists()){
+ rulesState = createIndexRules(definition).getNodeState();
+ indexDefn.setChildNode(LuceneIndexConstants.INDEX_RULES, rulesState);
+ log.info("Updated index definition for {}", this);
+ //indexDefn.removeProperty(DECLARING_NODE_TYPES);
+ //indexDefn.removeProperty(INCLUDE_PROPERTY_NAMES);
+ //indexDefn.removeProperty(EXCLUDE_PROPERTY_NAMES);
+ //indexDefn.removeProperty(ORDERED_PROP_NAMES);
+ }
+ }
+
+ /**
+ * Constructs IndexingRule based on earlier format of index configuration
+ */
+ private NodeBuilder createIndexRules(NodeState defn){
+ NodeBuilder builder = EMPTY_NODE.builder();
+ Set<String> declaringNodeTypes = getMultiProperty(defn, DECLARING_NODE_TYPES);
+ Set<String> includes = getMultiProperty(defn, INCLUDE_PROPERTY_NAMES);
+ Set<String> excludes = toLowerCase(getMultiProperty(defn, EXCLUDE_PROPERTY_NAMES));
+ Set<String> orderedProps = getMultiProperty(defn, ORDERED_PROP_NAMES);
+ boolean fullTextEnabled = getOptionalValue(defn, FULL_TEXT_ENABLED, true);
+ boolean storageEnabled = getOptionalValue(defn, EXPERIMENTAL_STORAGE, true);
+ NodeState propNodeState = defn.getChildNode(LuceneIndexConstants.PROP_NODE);
+
+ //If no explicit nodeType defined then all config applies for nt:base
+ if (declaringNodeTypes.isEmpty()){
+ declaringNodeTypes = Collections.singleton(NT_BASE);
+ }
+
+ Set<String> propNamesSet = Sets.newHashSet();
+ propNamesSet.addAll(includes);
+ propNamesSet.addAll(excludes);
+ propNamesSet.addAll(orderedProps);
+
+ //Also include all immediate leaf propNode names
+ for (ChildNodeEntry cne : propNodeState.getChildNodeEntries()){
+ if (!propNamesSet.contains(cne.getName())
+ && Iterables.isEmpty(cne.getNodeState().getChildNodeNames())){
+ propNamesSet.add(cne.getName());
+ }
+ }
+
+ List<String> propNames = new ArrayList<String>(propNamesSet);
+
+ final String includeAllProp = ".*";
+ if (fullTextEnabled
+ && includes.isEmpty()){
+ //Add the regEx for including all properties at the end
+ //for fulltext index and when no explicit includes are defined
+ propNames.add(includeAllProp);
+ }
+
+ for (String typeName : declaringNodeTypes){
+ NodeBuilder rule = builder.child(typeName);
+ markAsNtUnstructured(rule);
+ List<String> propNodeNames = newArrayListWithCapacity(propNamesSet.size());
+ NodeBuilder propNodes = rule.child(PROP_NODE);
+ int i = 0;
+ for (String propName : propNames){
+ String propNodeName = propName;
+
+ //For proper propName use the propName as childNode name
+ if(RelativeProperty.isRelativeProperty(propName)
+ || propName.equals(includeAllProp)){
+ propNodeName = "prop" + i++;
+ }
+ propNodeNames.add(propNodeName);
+
+ NodeBuilder prop = propNodes.child(propNodeName);
+ markAsNtUnstructured(prop);
+ prop.setProperty(LuceneIndexConstants.PROP_NAME, propName);
+
+ if (excludes.contains(propName)){
+ prop.setProperty(LuceneIndexConstants.PROP_INDEX, false);
+ } else if (fullTextEnabled){
+ prop.setProperty(LuceneIndexConstants.PROP_ANALYZED, true);
+ prop.setProperty(LuceneIndexConstants.PROP_NODE_SCOPE_INDEX, true);
+ prop.setProperty(LuceneIndexConstants.PROP_USE_IN_EXCERPT, storageEnabled);
+ prop.setProperty(LuceneIndexConstants.PROP_PROPERTY_INDEX, false);
+ } else {
+ prop.setProperty(LuceneIndexConstants.PROP_PROPERTY_INDEX, true);
+
+ if (orderedProps.contains(propName)){
+ prop.setProperty(LuceneIndexConstants.PROP_ORDERED, true);
+ }
+ }
+
+ if (propName.equals(includeAllProp)){
+ prop.setProperty(LuceneIndexConstants.PROP_IS_REGEX, true);
+ }
+
+ //Copy over the property configuration
+ NodeState propDefNode = getPropDefnNode(defn, propName);
+ if (propDefNode != null){
+ for (PropertyState ps : propDefNode.getProperties()){
+ prop.setProperty(ps);
+ }
+ }
+ }
+
+ //If no propertyType defined then default to UNKNOWN such that none
+ //of the properties get indexed
+ PropertyState supportedTypes = defn.getProperty(INCLUDE_PROPERTY_TYPES);
+ if (supportedTypes == null){
+ supportedTypes = PropertyStates.createProperty(INCLUDE_PROPERTY_TYPES, TYPES_ALLOW_ALL_NAME);
+ }
+ rule.setProperty(supportedTypes);
+
+ if (!NT_BASE.equals(typeName)) {
+ rule.setProperty(LuceneIndexConstants.RULE_INHERITED, false);
+ }
+
+ propNodes.setProperty(OAK_CHILD_ORDER, propNodeNames ,NAMES);
+ markAsNtUnstructured(propNodes);
+ }
+
+ markAsNtUnstructured(builder);
+ builder.setProperty(OAK_CHILD_ORDER, declaringNodeTypes ,NAMES);
+ return builder;
+ }
+
+ private NodeState getPropDefnNode(NodeState defn, String propName){
+ NodeState propNode = defn.getChildNode(LuceneIndexConstants.PROP_NODE);
+ NodeState propDefNode;
+ if (RelativeProperty.isRelativeProperty(propName)) {
+ propDefNode = new RelativeProperty(propName).getPropDefnNode(propNode);
+ } else {
+ propDefNode = propNode.getChildNode(propName);
+ }
+ return propDefNode.exists() ? propDefNode : null;
+ }
+
+
+ //~---------------------------------------------< utility >
+
+ private static String determineIndexName(NodeState defn, String indexPath) {
+ String indexName = defn.getString(PROP_NAME);
+ if (indexName == null){
+ if (indexPath != null) {
+ return indexPath;
+ }
+ return "<No 'name' property defined>";
+ }
+
+ if (indexPath != null){
+ return indexName + "(" + indexPath + ")";
+ }
+ return indexName;
}
private static Set<String> getMultiProperty(NodeState definition, String propName){
@@ -362,4 +833,106 @@ class IndexDefinition {
}
return ImmutableSet.copyOf(result);
}
+
+ private static List<String> getAllNodeTypes(ReadOnlyNodeTypeManager ntReg) {
+ try {
+ List<String> typeNames = newArrayList();
+ NodeTypeIterator ntItr = ntReg.getAllNodeTypes();
+ while (ntItr.hasNext()){
+ typeNames.add(ntItr.nextNodeType().getName());
+ }
+ return typeNames;
+ } catch (RepositoryException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static ReadOnlyNodeTypeManager createNodeTypeManager(final Tree root) {
+ return new ReadOnlyNodeTypeManager() {
+ @Override
+ protected Tree getTypes() {
+ return TreeUtil.getTree(root,NODE_TYPES_PATH);
+ }
+
+ @Nonnull
+ @Override
+ protected NamePathMapper getNamePathMapper() {
+ return NamePathMapper.DEFAULT;
+ }
+ };
+ }
+
+ private static String getPrimaryTypeName(Tree state) {
+ String primaryType = TreeUtil.getPrimaryTypeName(state);
+ //In case not a proper JCR assume nt:base TODO return null and ignore indexing such nodes
+ //at all
+ return primaryType != null ? primaryType : "nt:base";
+ }
+
+ private static Iterable<String> getMixinTypeNames(Tree tree) {
+ PropertyState property = tree.getProperty(JcrConstants.JCR_MIMETYPE);
+ return property != null ? property.getValue(Type.NAMES) : Collections.<String>emptyList();
+ }
+
+ private static boolean hasOrderableChildren(NodeState state){
+ return state.hasProperty(OAK_CHILD_ORDER);
+ }
+
+ private static int getSupportedTypes(NodeState defn, int defaultVal) {
+ PropertyState pst = defn.getProperty(INCLUDE_PROPERTY_TYPES);
+ if (pst != null) {
+ int types = 0;
+ for (String inc : pst.getValue(Type.STRINGS)) {
+ if (TYPES_ALLOW_ALL_NAME.equals(inc)){
+ return TYPES_ALLOW_ALL;
+ }
+
+ try {
+ types |= 1 << PropertyType.valueFromName(inc);
+ } catch (IllegalArgumentException e) {
+ log.warn("Unknown property type: " + inc);
+ }
+ }
+ return types;
+ }
+ return defaultVal;
+ }
+
+ private static boolean hasFulltextEnabledIndexRule(List<IndexingRule> rules) {
+ for (IndexingRule rule : rules){
+ if (rule.fulltextEnabled){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean hasPropertyIndexEnabledIndexRule(List<IndexingRule> rules) {
+ for (IndexingRule rule : rules){
+ if (rule.propertyIndexEnabled){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static Set<String> collectRelPropertyNames(Collection<RelativeProperty> props) {
+ Set<String> propNames = newHashSet();
+ for (RelativeProperty prop : props) {
+ propNames.add(prop.name);
+ }
+ return ImmutableSet.copyOf(propNames);
+ }
+
+ private static List<RelativeProperty> collectRelativeProperties(List<IndexingRule> indexRules) {
+ List<RelativeProperty> relProps = newArrayList();
+ for (IndexingRule rule : indexRules){
+ relProps.addAll(rule.getRelativeProps());
+ }
+ return ImmutableList.copyOf(relProps);
+ }
+
+ private static void markAsNtUnstructured(NodeBuilder nb){
+ nb.setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
+ }
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java?rev=1642092&r1=1642091&r2=1642092&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java Thu Nov 27 11:25:17 2014
@@ -43,7 +43,7 @@ class IndexNode {
static IndexNode open(String indexPath, NodeState root, NodeState defnNodeState,@Nullable IndexCopier cloner)
throws IOException {
Directory directory = null;
- IndexDefinition definition = new IndexDefinition(root, defnNodeState);
+ IndexDefinition definition = new IndexDefinition(root, defnNodeState, indexPath);
NodeState data = defnNodeState.getChildNode(INDEX_DATA_CHILD_NAME);
if (data.exists()) {
directory = new OakDirectory(new ReadOnlyBuilder(data), definition);
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java?rev=1642092&r1=1642091&r2=1642092&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java Thu Nov 27 11:25:17 2014
@@ -21,16 +21,20 @@ package org.apache.jackrabbit.oak.plugin
import java.util.Collections;
import java.util.List;
-import java.util.Set;
+import java.util.Map;
+import javax.annotation.CheckForNull;
-import com.google.common.collect.Sets;
+import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule;
import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.lucene.index.IndexReader;
+import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Lists.newArrayListWithCapacity;
+import static com.google.common.collect.Maps.newHashMap;
import static org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction;
import static org.apache.jackrabbit.oak.spi.query.QueryIndex.IndexPlan;
import static org.apache.jackrabbit.oak.spi.query.QueryIndex.OrderEntry;
@@ -41,6 +45,7 @@ class IndexPlanner {
private final String indexPath;
private final List<OrderEntry> sortOrder;
private IndexNode indexNode;
+ private PlanResult result;
public IndexPlanner(IndexNode indexNode,
String indexPath,
@@ -69,12 +74,23 @@ class IndexPlanner {
private IndexPlan.Builder getPlanBuilder() {
FullTextExpression ft = filter.getFullTextConstraint();
- //IndexPlanner is currently for property indexes only and does not
- //support full text indexes
- if (ft != null) {
+ //Query Fulltext and Index does not support fulltext
+ if (ft != null && !defn.isFullTextEnabled()) {
return null;
}
+ IndexingRule indexingRule = getApplicableRule();
+ if (indexingRule == null){
+ return null;
+ }
+
+ //Query Fulltext and indexing rule does not support fulltext
+ if (ft != null && !indexingRule.isFulltextEnabled()){
+ return null;
+ }
+
+ result = new PlanResult(indexPath, defn, indexingRule);
+
if (defn.hasFunctionDefined()
&& filter.getPropertyRestriction(defn.getFunctionName()) != null) {
//If native function is handled by this index then ensure
@@ -83,32 +99,32 @@ class IndexPlanner {
}
List<String> indexedProps = newArrayListWithCapacity(filter.getPropertyRestrictions().size());
- for (PropertyRestriction pr : filter.getPropertyRestrictions()) {
- //Only those properties which are included and not tokenized
- //can be managed by lucene for property restrictions
- if (defn.includeProperty(pr.propertyName)
- && defn.skipTokenization(pr.propertyName)) {
- indexedProps.add(pr.propertyName);
+
+ //Optimization - Go further only if any of the property is configured
+ //for property index
+ if (indexingRule.propertyIndexEnabled) {
+ for (PropertyRestriction pr : filter.getPropertyRestrictions()) {
+ PropertyDefinition pd = indexingRule.getConfig(pr.propertyName);
+ if (pd != null && pd.propertyIndex) {
+ indexedProps.add(pr.propertyName);
+ result.propDefns.put(pr.propertyName, pd);
+ }
}
}
- List<OrderEntry> sortOrder = createSortOrder();
- if (!indexedProps.isEmpty() || !sortOrder.isEmpty()) {
+ //TODO For the full text case need to determine if all field names
+ //used in fulltext expression are fulltext indexed or not
+
+ //Fulltext expression can also be like jcr:contains(jcr:content/metadata/@format, 'image')
+
+ List<OrderEntry> sortOrder = createSortOrder(indexingRule);
+ if (!indexedProps.isEmpty() || !sortOrder.isEmpty() || ft != null) {
//TODO Need a way to have better cost estimate to indicate that
//this index can evaluate more propertyRestrictions natively (if more props are indexed)
//For now we reduce cost per entry
int costPerEntryFactor = indexedProps.size();
costPerEntryFactor += sortOrder.size();
- // Restrict matching index when declaringNodeTypes declared
- if (defn.hasDeclaredNodeTypes()) {
- if (supportsNodeTypes(filter)) {
- // Reduce cost per entry by a small factor because number of nodes would be less
- costPerEntryFactor += 1;
- } else {
- return null;
- }
- }
//this index can evaluate more propertyRestrictions natively (if more props are indexed)
//For now we reduce cost per entry
IndexPlan.Builder plan = defaultPlan();
@@ -123,14 +139,6 @@ class IndexPlanner {
return null;
}
- /**
- * Determines if NodeTypes as defined in Filter are supported by current index
- */
- private boolean supportsNodeTypes(Filter filter) {
- return !Sets
- .intersection(defn.getDeclaringNodeTypes(), getSuperTypes(filter)).isEmpty();
- }
-
private IndexPlan.Builder defaultPlan() {
return new IndexPlan.Builder()
.setCostPerExecution(1) // we're local. Low-cost
@@ -140,8 +148,19 @@ class IndexPlanner {
.setFilter(filter)
.setPathPrefix(getPathPrefix())
.setDelayed(true) //Lucene is always async
- .setAttribute(LuceneIndex.ATTR_INDEX_PATH, indexPath)
- .setEstimatedEntryCount(Math.min(defn.getEntryCount(), getReader().numDocs()));
+ .setAttribute(LucenePropertyIndex.ATTR_PLAN_RESULT, result)
+ .setEstimatedEntryCount(estimatedEntryCount());
+ }
+
+ private long estimatedEntryCount() {
+ //Other index only compete in case of property indexes. For fulltext
+ //index return true count so as to allow multiple property indexes
+ //to be compared fairly
+ FullTextExpression ft = filter.getFullTextConstraint();
+ if (ft != null && defn.isFullTextEnabled()){
+ return getReader().numDocs();
+ }
+ return Math.min(defn.getEntryCount(), getReader().numDocs());
}
private String getPathPrefix() {
@@ -153,31 +172,68 @@ class IndexPlanner {
return indexNode.getSearcher().getIndexReader();
}
- private List<OrderEntry> createSortOrder() {
- //TODO Refine later once we make mixed indexes having both
- //full text and property index
- if (defn.isFullTextEnabled()) {
- return Collections.emptyList();
- }
-
+ private List<OrderEntry> createSortOrder(IndexingRule rule) {
if (sortOrder == null) {
return Collections.emptyList();
}
List<OrderEntry> orderEntries = newArrayListWithCapacity(sortOrder.size());
for (OrderEntry o : sortOrder) {
- //sorting can only be done for known/configured properties
- // and whose types are known
- if (defn.includeProperty(o.getPropertyName()) || defn.isOrdered(o.getPropertyName())
+ PropertyDefinition pd = rule.getConfig(o.getPropertyName());
+ if (pd != null
+ && pd.ordered
&& o.getPropertyType() != null
&& !o.getPropertyType().isArray()) {
orderEntries.add(o); //Lucene can manage any order desc/asc
+ result.sortedProperties.add(pd);
}
}
+ //TODO Should we return order entries only when all order clauses are satisfied
return orderEntries;
}
- private static Set<String> getSuperTypes(Filter filter) {
- return filter.matchesAllTypes() ? Collections.<String>emptySet() : filter.getSupertypes();
+ @CheckForNull
+ private IndexingRule getApplicableRule() {
+ if (filter.matchesAllTypes()){
+ return defn.getApplicableIndexingRule(JcrConstants.NT_BASE);
+ } else {
+ //TODO May be better if filter.getSuperTypes returned a list which maintains
+ //inheritance order and then we iterate over that
+ for (IndexingRule rule : defn.getDefinedRules()){
+ if (filter.getSupertypes().contains(rule.getNodeTypeName())){
+ //Theoretically there may be multiple rules for same nodeType with
+ //some condition defined. So again find a rule which applies
+ IndexingRule matchingRule = defn.getApplicableIndexingRule(rule.getNodeTypeName());
+ if (matchingRule != null){
+ return rule;
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ //~--------------------------------------------------------< PlanResult >
+
+ public static class PlanResult {
+ final String indexPath;
+ final IndexDefinition indexDefinition;
+ final IndexingRule indexingRule;
+ private List<PropertyDefinition> sortedProperties = newArrayList();
+ private Map<String, PropertyDefinition> propDefns = newHashMap();
+
+ public PlanResult(String indexPath, IndexDefinition defn, IndexingRule indexingRule) {
+ this.indexPath = indexPath;
+ this.indexDefinition = defn;
+ this.indexingRule = indexingRule;
+ }
+
+ public PropertyDefinition getPropDefn(PropertyRestriction pr){
+ return propDefns.get(pr.propertyName);
+ }
+
+ public PropertyDefinition getOrderedProperty(int index){
+ return sortedProperties.get(index);
+ }
}
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java?rev=1642092&r1=1642091&r2=1642092&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java Thu Nov 27 11:25:17 2014
@@ -53,8 +53,10 @@ import com.google.common.collect.Abstrac
import com.google.common.collect.Queues;
import com.google.common.collect.Sets;
+import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.PropertyValue;
import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.MoreLikeThisHelper;
import org.apache.jackrabbit.oak.query.QueryEngineSettings;
import org.apache.jackrabbit.oak.query.QueryImpl;
@@ -529,6 +531,9 @@ public class LuceneIndex implements Adva
break;
}
+ //Fulltext index definition used by LuceneIndex only works with old format
+ //which is not nodeType based. So just use the nt:base index
+ IndexingRule rule = indexDefinition.getApplicableIndexingRule(JcrConstants.NT_BASE);
for (PropertyRestriction pr : filter.getPropertyRestrictions()) {
if (pr.first == null && pr.last == null) {
@@ -538,7 +543,7 @@ public class LuceneIndex implements Adva
}
// check excluded properties and types
- if (isExcludedProperty(pr, indexDefinition)) {
+ if (isExcludedProperty(pr, rule)) {
continue;
}
@@ -630,15 +635,16 @@ public class LuceneIndex implements Adva
}
private static boolean isExcludedProperty(PropertyRestriction pr,
- IndexDefinition definition) {
+ IndexingRule rule) {
String name = pr.propertyName;
if (name.contains("/")) {
// lucene cannot handle child-level property restrictions
return true;
}
+ PropertyDefinition pd = rule.getConfig(name);
// check name
- if(!definition.includeProperty(name)){
+ if(pd == null || !pd.index){
return true;
}
@@ -652,13 +658,20 @@ public class LuceneIndex implements Adva
type = pr.list.get(0).getType().tag();
}
if (type != null) {
- if (!definition.includePropertyType(type)) {
+ if (!includePropertyType(type, rule)) {
return true;
}
}
return false;
}
+ private static boolean includePropertyType(int type, IndexingRule rule){
+ if(rule.propertyTypes < 0){
+ return false;
+ }
+ return (rule.propertyTypes & (1 << type)) != 0;
+ }
+
private static void addReferenceConstraint(String uuid, List<Query> qs,
IndexReader reader) {
if (reader == null) {
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1642092&r1=1642091&r2=1642092&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java Thu Nov 27 11:25:17 2014
@@ -101,9 +101,31 @@ public interface LuceneIndexConstants {
*/
String PROP_NODE = "properties";
+ String INDEX_RULES = "indexRules";
+
/**
* Field boost factor
*/
String FIELD_BOOST = "boost";
+ /**
+ * Property name defined explicitly. Mostly used in case of relative property names
+ */
+ String PROP_NAME = "name";
+
+ String PROP_IS_REGEX = "isRegexp";
+
+ String PROP_INDEX = "index";
+
+ String PROP_USE_IN_EXCERPT = "useInExcerpt";
+
+ String PROP_NODE_SCOPE_INDEX = "nodeScopeIndex";
+
+ String PROP_PROPERTY_INDEX = "propertyIndex";
+
+ String PROP_ANALYZED = "analyzed";
+
+ String RULE_INHERITED = "inherited";
+
+ String PROP_ORDERED = "ordered";
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1642092&r1=1642091&r2=1642092&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java Thu Nov 27 11:25:17 2014
@@ -29,8 +29,6 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
import com.google.common.collect.Lists;
import org.apache.jackrabbit.JcrConstants;
@@ -41,7 +39,7 @@ import org.apache.jackrabbit.oak.api.Typ
import org.apache.jackrabbit.oak.plugins.index.IndexEditor;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
-import org.apache.jackrabbit.oak.plugins.nodetype.TypePredicate;
+import org.apache.jackrabbit.oak.plugins.tree.ImmutableTree;
import org.apache.jackrabbit.oak.spi.commit.Editor;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
@@ -87,9 +85,7 @@ public class LuceneIndexEditor implement
private boolean propertiesChanged = false;
- private NodeState root;
-
- private final Predicate<NodeState> typePredicate;
+ private final NodeState root;
private List<RelativeProperty> changedRelativeProps;
@@ -100,6 +96,10 @@ public class LuceneIndexEditor implement
private final int deletedMaxLevels;
+ private ImmutableTree current;
+
+ private IndexDefinition.IndexingRule indexingRule;
+
LuceneIndexEditor(NodeState root, NodeBuilder definition, Analyzer analyzer,
IndexUpdateCallback updateCallback) throws CommitFailedException {
this.parent = null;
@@ -108,11 +108,6 @@ public class LuceneIndexEditor implement
this.context = new LuceneIndexEditorContext(root, definition, analyzer,
updateCallback);
this.root = root;
- if (context.getDefinition().hasDeclaredNodeTypes()) {
- typePredicate = new TypePredicate(root, context.getDefinition().getDeclaringNodeTypes());
- } else {
- typePredicate = Predicates.alwaysTrue();
- }
this.isDeleted = false;
this.deletedMaxLevels = -1;
}
@@ -124,12 +119,12 @@ public class LuceneIndexEditor implement
this.path = null;
this.context = parent.context;
this.root = parent.root;
- this.typePredicate = parent.typePredicate;
this.isDeleted = isDeleted;
this.deletedMaxLevels = deletedMaxLevels;
}
public String getPath() {
+ //TODO Use the tree instance to determine path
if (path == null) { // => parent != null
path = concat(parent.getPath(), name);
}
@@ -142,6 +137,16 @@ public class LuceneIndexEditor implement
if (EmptyNodeState.MISSING_NODE == before && parent == null){
context.enableReindexMode();
}
+
+ //For traversal in deleted sub tree before state has to be used
+ NodeState currentState = after.exists() ? after : before;
+ if (parent == null){
+ current = new ImmutableTree(currentState);
+ } else {
+ current = new ImmutableTree(parent.current, name, currentState);
+ }
+
+ indexingRule = getDefinition().getApplicableIndexingRule(current);
}
@Override
@@ -223,12 +228,12 @@ public class LuceneIndexEditor implement
}
}
- if (context.getDefinition().hasRelativeProperties()) {
+ if (getDefinition().hasRelativeProperties()) {
int maxLevelsDown;
if (isDeleted) {
maxLevelsDown = deletedMaxLevels - 1;
} else {
- maxLevelsDown = context.getDefinition()
+ maxLevelsDown = getDefinition()
.getRelPropertyMaxLevels();
}
if (maxLevelsDown > 0) {
@@ -258,11 +263,7 @@ public class LuceneIndexEditor implement
}
private Document makeDocument(String path, NodeState state, boolean isUpdate) throws CommitFailedException {
- //TODO Possibly we can add support for compound properties like foo/bar
- //i.e. support for relative path restrictions
-
- // Check for declaringNodeType validity
- if (!typePredicate.apply(state)) {
+ if (!isIndexable()) {
return null;
}
@@ -275,23 +276,17 @@ public class LuceneIndexEditor implement
continue;
}
- if (context.getDefinition().isOrdered(pname)) {
- dirty |= addTypedOrderedFields(fields, property, pname);
- }
+ PropertyDefinition pd = indexingRule.getConfig(pname);
- if (context.includeProperty(pname)) {
- //In case of fulltext we also check if given type is enabled for indexing
- //TODO Use context.includePropertyType however that cause issue. Need
- //to make filtering based on type consistent both on indexing side and
- //query size
- if(context.isFullTextEnabled()
- && (context.getPropertyTypes() & (1 << property.getType()
- .tag())) == 0){
- continue;
- }
+ if (pd == null || !pd.index){
+ continue;
+ }
- dirty |= indexProperty(path, fields, state, property, pname);
+ if (pd.ordered) {
+ dirty |= addTypedOrderedFields(fields, property, pname, pd);
}
+
+ dirty |= indexProperty(path, fields, state, property, pname, pd);
}
dirty |= indexRelativeProperties(path, fields, state);
@@ -303,7 +298,7 @@ public class LuceneIndexEditor implement
//For property index no use making an empty document if
//none of the properties are indexed
- if(!context.isFullTextEnabled() && !dirty){
+ if(!indexingRule.isFulltextEnabled() && !dirty){
return null;
}
@@ -312,41 +307,60 @@ public class LuceneIndexEditor implement
String name = getName(path);
//TODO Possibly index nodeName without tokenization for node name based queries
- if (context.isFullTextEnabled()) {
+ if (indexingRule.isFulltextEnabled()) {
document.add(newFulltextField(name));
}
for (Field f : fields) {
document.add(f);
}
+
+ //TODO Boost at document level
+
return document;
}
- private boolean indexProperty(String path, List<Field> fields, NodeState state,
- PropertyState property, String pname) throws CommitFailedException {
- if (Type.BINARY.tag() == property.getType().tag()) {
+ private boolean indexProperty(String path,
+ List<Field> fields, NodeState state,
+ PropertyState property,
+ String pname,
+ PropertyDefinition pd) throws CommitFailedException {
+ //In case of fulltext we also check if given type is enabled for indexing
+ //TODO Use context.includePropertyType however that cause issue. Need
+ //to make filtering based on type consistent both on indexing side and
+ //query side
+ //TODO Replace with indexRule.includeType
+ boolean includeTypeForFullText = (indexingRule.propertyTypes & (1 << property.getType().tag())) != 0;
+ if (Type.BINARY.tag() == property.getType().tag()
+ && includeTypeForFullText) {
this.context.indexUpdate();
fields.addAll(newBinary(property, state, path + "@" + pname));
return true;
- } else if(!context.isFullTextEnabled()
- && FieldFactory.canCreateTypedField(property.getType())){
- return addTypedFields(fields, property, pname);
- } else {
+ } else {
boolean dirty = false;
- for (String value : property.getValue(Type.STRINGS)) {
- this.context.indexUpdate();
- fields.add(newPropertyField(pname, value,
- !context.skipTokenization(pname),
- context.isStored(pname)));
- if (context.isFullTextEnabled()) {
- Field field = newFulltextField(value);
- boolean hasBoost = context.getDefinition().getPropDefn(pname) != null &&
- context.getDefinition().getPropDefn(pname).hasFieldBoost();
- if (hasBoost) {
- field.setBoost((float)context.getDefinition().getPropDefn(pname).fieldBoost());
+
+ if (pd.propertyIndex){
+ dirty |= addTypedFields(fields, property, pname);
+ }
+
+ if (pd.fulltextEnabled() && includeTypeForFullText) {
+ for (String value : property.getValue(Type.STRINGS)) {
+ this.context.indexUpdate();
+ //TODO Analyzed field should be stored against different field name
+ //as term field use the same name. For compatibility this should be done
+ //for newer index versions only
+ if (pd.analyzed) {
+ fields.add(newPropertyField(pname, value, !pd.skipTokenization(pname), pd.stored));
+ //TODO Property field uses OakType which has omitNorms set hence
+ //cannot be boosted
+ }
+
+ if (pd.nodeScopeIndex) {
+ Field field = newFulltextField(value);
+ field.setBoost(pd.boost);
+ fields.add(field);
}
- fields.add(field);
+ dirty = true;
}
- dirty = true;
}
return dirty;
}
@@ -356,7 +370,7 @@ public class LuceneIndexEditor implement
int tag = property.getType().tag();
boolean fieldAdded = false;
for (int i = 0; i < property.count(); i++) {
- Field f = null;
+ Field f;
if (tag == Type.LONG.tag()) {
f = new LongField(pname, property.getValue(Type.LONG, i), Field.Store.NO);
} else if (tag == Type.DATE.tag()) {
@@ -366,21 +380,23 @@ public class LuceneIndexEditor implement
f = new DoubleField(pname, property.getValue(Type.DOUBLE, i), Field.Store.NO);
} else if (tag == Type.BOOLEAN.tag()) {
f = new StringField(pname, property.getValue(Type.BOOLEAN, i).toString(), Field.Store.NO);
+ } else {
+ f = new StringField(pname, property.getValue(Type.STRING, i), Field.Store.NO);
}
- if (f != null) {
- this.context.indexUpdate();
- fields.add(f);
- fieldAdded = true;
- }
+ this.context.indexUpdate();
+ fields.add(f);
+ fieldAdded = true;
}
return fieldAdded;
}
- private boolean addTypedOrderedFields(List<Field> fields, PropertyState property, String pname) throws CommitFailedException {
+ private boolean addTypedOrderedFields(List<Field> fields,
+ PropertyState property,
+ String pname,
+ PropertyDefinition pd) throws CommitFailedException {
int tag = property.getType().tag();
-
- int idxDefinedTag = getIndexDefinitionType(pname);
+ int idxDefinedTag = pd.getType();
// Try converting type to the defined type in the index definition
if (tag != idxDefinedTag) {
log.debug(
@@ -429,18 +445,6 @@ public class LuceneIndexEditor implement
return fieldAdded;
}
- private int getIndexDefinitionType(String pname) {
- int type = Type.UNDEFINED.tag();
- if (context.getDefinition().hasPropertyDefinition(pname)) {
- type = context.getDefinition().getPropDefn(pname).getPropertyType();
- }
- if (type == Type.UNDEFINED.tag()) {
- //If no explicit type is defined we assume it to be string
- type = Type.STRING.tag();
- }
- return type;
- }
-
private static boolean isVisible(String name) {
return name.charAt(0) != ':';
}
@@ -467,9 +471,8 @@ public class LuceneIndexEditor implement
}
private boolean indexRelativeProperties(String path, List<Field> fields, NodeState state) throws CommitFailedException {
- IndexDefinition defn = context.getDefinition();
boolean dirty = false;
- for (RelativeProperty rp : defn.getRelativeProps()){
+ for (RelativeProperty rp : indexingRule.getRelativeProps()){
String pname = rp.propertyPath;
PropertyState property = rp.getProperty(state);
@@ -478,18 +481,18 @@ public class LuceneIndexEditor implement
continue;
}
- if (defn.isOrdered(pname)) {
- dirty |= addTypedOrderedFields(fields, property, pname);
+ if (rp.getPropertyDefinition().ordered) {
+ dirty |= addTypedOrderedFields(fields, property, pname, rp.getPropertyDefinition());
}
- dirty |= indexProperty(path, fields, state, property, pname);
+ dirty |= indexProperty(path, fields, state, property, pname, rp.getPropertyDefinition());
}
return dirty;
}
private void checkForRelativePropertyChange(String name) {
- if (context.getDefinition().hasRelativeProperty(name)) {
- context.getDefinition().collectRelPropsForName(name, getChangedRelProps());
+ if (isIndexable() && getDefinition().hasRelativeProperty(name)) {
+ getDefinition().collectRelPropsForName(name, getChangedRelProps());
}
}
@@ -518,11 +521,21 @@ public class LuceneIndexEditor implement
}
private void markPropertyChanged(String name) {
- if (!propertiesChanged && context.getDefinition().includeProperty(name)) {
+ if (isIndexable()
+ && !propertiesChanged
+ && indexingRule.isIndexed(name)) {
propertiesChanged = true;
}
}
+ private IndexDefinition getDefinition() {
+ return context.getDefinition();
+ }
+
+ private boolean isIndexable(){
+ return indexingRule != null;
+ }
+
private void relativePropertyChanged() {
propertiesChanged = true;
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1642092&r1=1642091&r2=1642092&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java Thu Nov 27 11:25:17 2014
@@ -112,14 +112,6 @@ public class LuceneIndexEditorContext {
this.updateCallback = updateCallback;
}
- boolean includeProperty(String name) {
- return definition.includeProperty(name);
- }
-
- boolean includePropertyType(int type){
- return definition.includePropertyType(type);
- }
-
Parser getParser() {
return parser;
}
@@ -172,25 +164,6 @@ public class LuceneIndexEditorContext {
updateCallback.indexUpdate();
}
- /**
- * Checks if a given property should be stored in the lucene index or not
- */
- public boolean isStored(String name) {
- return definition.isStored(name);
- }
-
- public boolean isFullTextEnabled() {
- return definition.isFullTextEnabled();
- }
-
- public boolean skipTokenization(String propertyName){
- return definition.skipTokenization(propertyName);
- }
-
- public int getPropertyTypes() {
- return definition.getPropertyTypes();
- }
-
public IndexDefinition getDefinition() {
return definition;
}