You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2014/11/27 12:40:20 UTC
svn commit: r1642116 - in /jackrabbit/oak/branches/1.0: ./
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/
oak-lucene/src/test/java/org/apache/jackra...
Author: chetanm
Date: Thu Nov 27 11:40:20 2014
New Revision: 1642116
URL: http://svn.apache.org/r1642116
Log:
OAK-2298 - LuceneIndex should provide correct cost based on index size
Merging revisions 1642031
Modified:
jackrabbit/oak/branches/1.0/ (props changed)
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java
jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java
Propchange: jackrabbit/oak/branches/1.0/
------------------------------------------------------------------------------
Merged /jackrabbit/oak/trunk:r1642031
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1642116&r1=1642115&r2=1642116&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java Thu Nov 27 11:40:20 2014
@@ -112,8 +112,6 @@ class IndexDefinition implements Aggrega
private final boolean fullTextEnabled;
- private final boolean propertyIndexEnabled;
-
private final NodeState definition;
private final NodeState root;
@@ -130,6 +128,12 @@ class IndexDefinition implements Aggrega
*/
private final long entryCount;
+ private final boolean entryCountDefined;
+
+ private final double costPerEntry;
+
+ private final double costPerExecution;
+
/**
* The {@link IndexingRule}s inside this configuration. Keys being the NodeType names
*/
@@ -181,8 +185,6 @@ class IndexDefinition implements Aggrega
this.definedRules = ImmutableList.copyOf(definedIndexRules);
this.fullTextEnabled = hasFulltextEnabledIndexRule(definedIndexRules);
- this.propertyIndexEnabled = hasPropertyIndexEnabledIndexRule(definedIndexRules);
-
this.evaluatePathRestrictions = getOptionalValue(defn, EVALUATE_PATH_RESTRICTION, false);
String functionName = getOptionalValue(defn, LuceneIndexConstants.FUNC_NAME, null);
@@ -194,11 +196,15 @@ class IndexDefinition implements Aggrega
this.codec = createCodec();
if (defn.hasProperty(ENTRY_COUNT_PROPERTY_NAME)) {
+ this.entryCountDefined = true;
this.entryCount = defn.getProperty(ENTRY_COUNT_PROPERTY_NAME).getValue(Type.LONG);
} else {
+ this.entryCountDefined = false;
this.entryCount = DEFAULT_ENTRY_COUNT;
}
+ this.costPerEntry = getOptionalValue(defn, LuceneIndexConstants.COST_PER_ENTRY, 1.0);
+ this.costPerExecution = getOptionalValue(defn, LuceneIndexConstants.COST_PER_EXECUTION, 1.0);
this.indexesAllTypes = areAllTypesIndexed();
}
@@ -206,10 +212,6 @@ class IndexDefinition implements Aggrega
return fullTextEnabled;
}
- public boolean isPropertyIndexEnabled() {
- return propertyIndexEnabled;
- }
-
public String getFunctionName(){
return funcName;
}
@@ -241,6 +243,25 @@ class IndexDefinition implements Aggrega
return entryCount;
}
+ public boolean isEntryCountDefined() {
+ return entryCountDefined;
+ }
+
+ public double getCostPerEntry() {
+ return costPerEntry;
+ }
+
+ public double getCostPerExecution() {
+ return costPerExecution;
+ }
+
+ public long getFulltextEntryCount(long numOfDocs){
+ if (isEntryCountDefined()){
+ return Math.min(getEntryCount(), numOfDocs);
+ }
+ return numOfDocs;
+ }
+
public IndexFormatVersion getVersion() {
return version;
}
@@ -974,15 +995,6 @@ class IndexDefinition implements Aggrega
return false;
}
- private static boolean hasPropertyIndexEnabledIndexRule(List<IndexingRule> rules) {
- for (IndexingRule rule : rules){
- if (rule.propertyIndexEnabled){
- return true;
- }
- }
- return false;
- }
-
private static void markAsNtUnstructured(NodeBuilder nb){
nb.setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java?rev=1642116&r1=1642115&r2=1642116&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java Thu Nov 27 11:40:20 2014
@@ -177,7 +177,7 @@ class IndexPlanner {
result.enableNonFullTextConstraints();
}
- return plan.setCostPerEntry(1.0 / costPerEntryFactor);
+ return plan.setCostPerEntry(defn.getCostPerEntry() / costPerEntryFactor);
}
//TODO Support for property existence queries
@@ -267,8 +267,8 @@ class IndexPlanner {
private IndexPlan.Builder defaultPlan() {
return new IndexPlan.Builder()
- .setCostPerExecution(1) // we're local. Low-cost
- .setCostPerEntry(1)
+ .setCostPerExecution(defn.getCostPerExecution())
+ .setCostPerEntry(defn.getCostPerEntry())
.setFulltextIndex(defn.isFullTextEnabled())
.setIncludesNodeData(false) // we should not include node data
.setFilter(filter)
@@ -284,7 +284,7 @@ class IndexPlanner {
//to be compared fairly
FullTextExpression ft = filter.getFullTextConstraint();
if (ft != null && defn.isFullTextEnabled()){
- return getReader().numDocs();
+ return defn.getFulltextEntryCount(getReader().numDocs());
}
return Math.min(defn.getEntryCount(), getReader().numDocs());
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java?rev=1642116&r1=1642115&r2=1642116&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java Thu Nov 27 11:40:20 2014
@@ -202,24 +202,24 @@ public class LuceneIndex implements Adva
// "contains(a/x, 'hello') and contains(b/x, 'world')"
return Collections.emptyList();
}
- String parent = relPaths.iterator().next();
-
- // no relative properties
- double cost = 10;
- if (!parent.isEmpty()) {
- // all relative properties have the same "parent", as in
- // "contains(a/x, 'hello') and contains(a/y, 'world')" or
- // "contains(a/x, 'hello') or contains(a/*, 'world')"
- // TODO: proper cost calculation
- // we assume this will cause more read operations,
- // as we need to read the node and then the parent
- cost = 15;
- }
- return Collections.singletonList(planBuilder(filter)
- .setCostPerExecution(cost)
- .setAttribute(ATTR_INDEX_PATH, indexPath)
- .build());
-
+ IndexNode node = tracker.acquireIndexNode(indexPath);
+ try{
+ if (node != null){
+ IndexDefinition defn = node.getDefinition();
+ return Collections.singletonList(planBuilder(filter)
+ .setEstimatedEntryCount(defn.getFulltextEntryCount(node.getSearcher().getIndexReader().numDocs()))
+ .setCostPerExecution(defn.getCostPerExecution())
+ .setCostPerEntry(defn.getCostPerEntry())
+ .setAttribute(ATTR_INDEX_PATH, indexPath)
+ .build());
+ }
+ //No index node then no plan possible
+ return Collections.emptyList();
+ } finally {
+ if (node != null){
+ node.release();
+ }
+ }
}
@Override
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1642116&r1=1642115&r2=1642116&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java Thu Nov 27 11:40:20 2014
@@ -178,4 +178,8 @@ public interface LuceneIndexConstants {
* node represented by this pattern
*/
String AGG_RELATIVE_NODE = "relativeNode";
+
+ String COST_PER_ENTRY = "costPerEntry";
+
+ String COST_PER_EXECUTION = "costPerExecution";
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java?rev=1642116&r1=1642115&r2=1642116&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java Thu Nov 27 11:40:20 2014
@@ -47,6 +47,11 @@ public class ConfigUtil {
return ps == null ? defaultVal : ps.getValue(Type.DOUBLE).floatValue();
}
+ public static double getOptionalValue(NodeState definition, String propName, double defaultVal){
+ PropertyState ps = definition.getProperty(propName);
+ return ps == null ? defaultVal : ps.getValue(Type.DOUBLE);
+ }
+
public static String getPrimaryTypeName(NodeState nodeState) {
PropertyState ps = nodeState.getProperty(JcrConstants.JCR_PRIMARYTYPE);
return (ps == null) ? JcrConstants.NT_BASE : ps.getValue(Type.NAME);
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java?rev=1642116&r1=1642115&r2=1642116&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java Thu Nov 27 11:40:20 2014
@@ -22,10 +22,10 @@ package org.apache.jackrabbit.oak.plugin
import javax.jcr.PropertyType;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
import org.apache.jackrabbit.oak.plugins.tree.ImmutableTree;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
@@ -409,6 +409,41 @@ public class IndexDefinitionTest {
assertTrue(defn.isFullTextEnabled());
}
+ @Test
+ public void costConfig() throws Exception{
+ NodeBuilder defnb = newLucenePropertyIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME),
+ "lucene", of("foo"), "async");
+ IndexDefinition defn = new IndexDefinition(root, defnb.getNodeState());
+ assertEquals(1.0, defn.getCostPerEntry(), 0);
+ assertEquals(1.0, defn.getCostPerExecution(), 0);
+ assertEquals(IndexDefinition.DEFAULT_ENTRY_COUNT, defn.getEntryCount());
+ assertFalse(defn.isEntryCountDefined());
+
+ defnb.setProperty(LuceneIndexConstants.COST_PER_ENTRY, 2.0);
+ defnb.setProperty(LuceneIndexConstants.COST_PER_EXECUTION, 3.0);
+ defnb.setProperty(IndexConstants.ENTRY_COUNT_PROPERTY_NAME, 500);
+
+ IndexDefinition defn2 = new IndexDefinition(root, defnb.getNodeState());
+ assertEquals(2.0, defn2.getCostPerEntry(), 0);
+ assertEquals(3.0, defn2.getCostPerExecution(), 0);
+ assertEquals(500, defn2.getEntryCount());
+ }
+
+ @Test
+ public void fulltextCost() throws Exception{
+ NodeBuilder defnb = newLucenePropertyIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME),
+ "lucene", of("foo"), "async");
+ IndexDefinition defn = new IndexDefinition(root, defnb.getNodeState());
+ assertEquals(300, defn.getFulltextEntryCount(300));
+ assertEquals(IndexDefinition.DEFAULT_ENTRY_COUNT + 100,
+ defn.getFulltextEntryCount(IndexDefinition.DEFAULT_ENTRY_COUNT + 100));
+
+ //Once count is explicitly defined then it would influence the cost
+ defnb.setProperty(IndexConstants.ENTRY_COUNT_PROPERTY_NAME, 100);
+ defn = new IndexDefinition(root, defnb.getNodeState());
+ assertEquals(100, defn.getFulltextEntryCount(300));
+ assertEquals(50, defn.getFulltextEntryCount(50));
+ }
private static IndexingRule getRule(IndexDefinition defn, String typeName){
return defn.getApplicableIndexingRule(newTree(newNode(typeName)));
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java?rev=1642116&r1=1642115&r2=1642116&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java Thu Nov 27 11:40:20 2014
@@ -57,6 +57,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.nodetype.NodeTypeConstants.JCR_NODE_TYPES;
import static org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent.INITIAL_CONTENT;
import static org.apache.jackrabbit.oak.spi.query.QueryIndex.OrderEntry;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -137,7 +138,6 @@ public class IndexPlannerTest {
assertTrue(pr(plan).evaluateNonFullTextConstraints());
}
-
@Test
public void worksWithIndexFormatV2Onwards() throws Exception{
NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME);
@@ -154,6 +154,66 @@ public class IndexPlannerTest {
assertNull(planner.getPlan());
}
+ @Test
+ public void propertyIndexCost() throws Exception{
+ NodeBuilder defn = newLucenePropertyIndexDefinition(builder, "test", of("foo"), "async");
+ long numofDocs = IndexDefinition.DEFAULT_ENTRY_COUNT + 1000;
+
+ IndexDefinition idxDefn = new IndexDefinition(root, defn.getNodeState());
+ IndexNode node = createIndexNode(idxDefn, numofDocs);
+ FilterImpl filter = createFilter("nt:base");
+ filter.restrictProperty("foo", Operator.EQUAL, PropertyValues.newString("bar"));
+ IndexPlanner planner = new IndexPlanner(node, "/foo", filter, Collections.<OrderEntry>emptyList());
+ QueryIndex.IndexPlan plan = planner.getPlan();
+
+ //For propertyIndex if entry count (default to IndexDefinition.DEFAULT_ENTRY_COUNT) is
+ //less than numOfDoc then that would be preferred
+ assertEquals(idxDefn.getEntryCount(), plan.getEstimatedEntryCount());
+ assertEquals(1.0, plan.getCostPerExecution(), 0);
+ assertEquals(1.0, plan.getCostPerEntry(), 0);
+ }
+
+ @Test
+ public void propertyIndexCost2() throws Exception{
+ NodeBuilder defn = newLucenePropertyIndexDefinition(builder, "test", of("foo"), "async");
+ defn.setProperty(LuceneIndexConstants.COST_PER_ENTRY, 2.0);
+ defn.setProperty(LuceneIndexConstants.COST_PER_EXECUTION, 3.0);
+
+ long numofDocs = IndexDefinition.DEFAULT_ENTRY_COUNT - 100;
+ IndexNode node = createIndexNode(new IndexDefinition(root, defn.getNodeState()), numofDocs);
+ FilterImpl filter = createFilter("nt:base");
+ filter.restrictProperty("foo", Operator.EQUAL, PropertyValues.newString("bar"));
+ IndexPlanner planner = new IndexPlanner(node, "/foo", filter, Collections.<OrderEntry>emptyList());
+ QueryIndex.IndexPlan plan = planner.getPlan();
+
+ assertEquals(numofDocs, plan.getEstimatedEntryCount());
+ assertEquals(3.0, plan.getCostPerExecution(), 0);
+ assertEquals(2.0, plan.getCostPerEntry(), 0);
+ assertNotNull(plan);
+ }
+
+ @Test
+ public void fulltextIndexCost() throws Exception{
+ NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME);
+ NodeBuilder defn = newLuceneIndexDefinition(index, "lucene",
+ of(TYPENAME_STRING));
+ TestUtil.useV2(defn);
+
+ long numofDocs = IndexDefinition.DEFAULT_ENTRY_COUNT + 1000;
+ IndexNode node = createIndexNode(new IndexDefinition(root, defn.getNodeState()), numofDocs);
+ FilterImpl filter = createFilter("nt:base");
+ filter.setFullTextConstraint(FullTextParser.parse(".", "mountain"));
+ IndexPlanner planner = new IndexPlanner(node, "/foo", filter, Collections.<OrderEntry>emptyList());
+
+ QueryIndex.IndexPlan plan = planner.getPlan();
+ assertNotNull(plan);
+ assertEquals(numofDocs, plan.getEstimatedEntryCount());
+ }
+
+ private IndexNode createIndexNode(IndexDefinition defn, long numOfDocs) throws IOException {
+ return new IndexNode("foo", defn, createSampleDirectory(numOfDocs));
+ }
+
private IndexNode createIndexNode(IndexDefinition defn) throws IOException {
return new IndexNode("foo", defn, createSampleDirectory());
}
@@ -167,12 +227,18 @@ public class IndexPlannerTest {
}
private static Directory createSampleDirectory() throws IOException {
+ return createSampleDirectory(1);
+ }
+
+ private static Directory createSampleDirectory(long numOfDocs) throws IOException {
Directory dir = new RAMDirectory();
IndexWriterConfig config = new IndexWriterConfig(VERSION, LuceneIndexConstants.ANALYZER);
IndexWriter writer = new IndexWriter(dir, config);
- Document doc = new Document();
- doc.add(new StringField("foo", "bar", Field.Store.NO));
- writer.addDocument(doc);
+ for (int i = 0; i < numOfDocs; i++) {
+ Document doc = new Document();
+ doc.add(new StringField("foo", "bar" + i, Field.Store.NO));
+ writer.addDocument(doc);
+ }
writer.close();
return dir;
}