You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jackrabbit.apache.org by mr...@apache.org on 2007/09/24 10:47:00 UTC
svn commit: r578711 - in
/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene:
FieldNames.java IndexFormatVersion.java LuceneQueryBuilder.java
MultiIndex.java NodeIndexer.java QueryImpl.java SearchIndex.java
Author: mreutegg
Date: Mon Sep 24 01:46:59 2007
New Revision: 578711
URL: http://svn.apache.org/viewvc?rev=578711&view=rev
Log:
JCR-1064: Optimize queries that check for the existence of a property
Added:
jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexFormatVersion.java (with props)
Modified:
jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/FieldNames.java
jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryBuilder.java
jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java
jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NodeIndexer.java
jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/QueryImpl.java
jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java
Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/FieldNames.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/FieldNames.java?rev=578711&r1=578710&r2=578711&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/FieldNames.java (original)
+++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/FieldNames.java Mon Sep 24 01:46:59 2007
@@ -71,6 +71,12 @@
public static final String PROPERTIES = "_:PROPERTIES".intern();
/**
+ * Name of the field that contains the names of all properties that are set
+ * on an indexed node.
+ */
+ public static final String PROPERTIES_SET = "_:PROPERTIES_SET".intern();
+
+ /**
* Name of the field that contains the UUIDs of the aggregated nodes. The
* terms are not tokenized and not stored, only indexed.
*/
Added: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexFormatVersion.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexFormatVersion.java?rev=578711&view=auto
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexFormatVersion.java (added)
+++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexFormatVersion.java Mon Sep 24 01:46:59 2007
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.query.lucene;
+
+import java.util.Collection;
+
+import org.apache.lucene.index.IndexReader;
+
+/**
+ * This class indicates the lucene index format that is used. Version 1 formats
+ * do not have the <code>PROPERTIES_SET</code> lucene fieldname and queries
+ * assuming this format also run on newer versions. When the index is recreated
+ * from scratch, the Version 2 format will automatically be used. This format is
+ * faster certain queries, so if the index does not contain
+ * <code>PROPERTIES_SET</code> fieldname and re-indexing is an option, this is
+ * advisable. Existing indexes are not automatically upgraded to a newer
+ * version!
+ */
+public class IndexFormatVersion {
+
+ /**
+ * V1 is the index format for Jackrabbit releases 1.0 to 1.3.x.
+ */
+ public static final IndexFormatVersion V1 = new IndexFormatVersion(1);
+
+ /**
+ * V2 is the index format for Jackrabbit releases >= 1.4
+ */
+ public static final IndexFormatVersion V2 = new IndexFormatVersion(2);
+
+ /**
+ * The used version of the index format
+ */
+ private final int version;
+
+ /**
+ * Creates a index format version.
+ *
+ * @param version The version of the index.
+ */
+ private IndexFormatVersion(int version) {
+ this.version = version;
+ }
+
+ /**
+ * Returns the index format version
+ * @return the index format version.
+ */
+ public int getVersion(){
+ return version;
+ }
+
+ /**
+ * @return a string representation of this index format version.
+ */
+ public String toString() {
+ return String.valueOf(getVersion());
+ }
+
+ /**
+ * @return the index format version of the index used by the given
+ * index reader.
+ */
+ public static IndexFormatVersion getVersion(IndexReader indexReader) {
+ Collection fields = indexReader.getFieldNames(IndexReader.FieldOption.ALL);
+ if (fields.contains(FieldNames.PROPERTIES_SET)
+ || indexReader.numDocs() == 0) {
+ return IndexFormatVersion.V2;
+ } else {
+ return IndexFormatVersion.V1;
+ }
+ }
+}
Propchange: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexFormatVersion.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryBuilder.java?rev=578711&r1=578710&r2=578711&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryBuilder.java (original)
+++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryBuilder.java Mon Sep 24 01:46:59 2007
@@ -130,6 +130,11 @@
private SynonymProvider synonymProvider;
/**
+ * Wether the index format is new or old.
+ */
+ private IndexFormatVersion indexFormatVersion;
+
+ /**
* Exceptions thrown during tree translation
*/
private List exceptions = new ArrayList();
@@ -137,16 +142,18 @@
/**
* Creates a new <code>LuceneQueryBuilder</code> instance.
*
- * @param root the root node of the abstract query tree.
- * @param session of the user executing this query.
- * @param sharedItemMgr the shared item state manager of the workspace.
- * @param hmgr a hierarchy manager based on sharedItemMgr.
- * @param nsMappings namespace resolver for internal prefixes.
- * @param analyzer for parsing the query statement of the contains
- * function.
- * @param propReg the property type registry.
- * @param synonymProvider the synonym provider or <code>null</code> if node
- * is configured.
+ * @param root the root node of the abstract query tree.
+ * @param session of the user executing this query.
+ * @param sharedItemMgr the shared item state manager of the
+ * workspace.
+ * @param hmgr a hierarchy manager based on sharedItemMgr.
+ * @param nsMappings namespace resolver for internal prefixes.
+ * @param analyzer for parsing the query statement of the contains
+ * function.
+ * @param propReg the property type registry.
+ * @param synonymProvider the synonym provider or <code>null</code> if
+ * node is configured.
+ * @param indexFormatVersion the index format version for the lucene query.
*/
private LuceneQueryBuilder(QueryRootNode root,
SessionImpl session,
@@ -155,7 +162,8 @@
NamespaceMappings nsMappings,
Analyzer analyzer,
PropertyTypeRegistry propReg,
- SynonymProvider synonymProvider) {
+ SynonymProvider synonymProvider,
+ IndexFormatVersion indexFormatVersion) {
this.root = root;
this.session = session;
this.sharedItemMgr = sharedItemMgr;
@@ -164,6 +172,7 @@
this.analyzer = analyzer;
this.propRegistry = propReg;
this.synonymProvider = synonymProvider;
+ this.indexFormatVersion = indexFormatVersion;
}
/**
@@ -180,6 +189,7 @@
* information.
* @param synonymProvider the synonym provider or <code>null</code> if node
* is configured.
+ * @param indexFormatVersion the index format version to be used
* @return the lucene query tree.
* @throws RepositoryException if an error occurs during the translation.
*/
@@ -189,7 +199,8 @@
NamespaceMappings nsMappings,
Analyzer analyzer,
PropertyTypeRegistry propReg,
- SynonymProvider synonymProvider)
+ SynonymProvider synonymProvider,
+ IndexFormatVersion indexFormatVersion)
throws RepositoryException {
NodeId id = ((NodeImpl) session.getRootNode()).getNodeId();
@@ -197,7 +208,7 @@
id, sharedItemMgr, session);
LuceneQueryBuilder builder = new LuceneQueryBuilder(
root, session, sharedItemMgr, hmgr, nsMappings,
- analyzer, propReg, synonymProvider);
+ analyzer, propReg, synonymProvider, indexFormatVersion);
Query q = builder.createLuceneQuery();
if (builder.exceptions.size() > 0) {
@@ -543,7 +554,7 @@
// todo this will traverse the whole index, optimize!
Query subQuery = null;
try {
- subQuery = new MatchAllQuery(NameFormat.format(QName.JCR_PRIMARYTYPE, nsMappings));
+ subQuery = createMatchAllQuery(NameFormat.format(QName.JCR_PRIMARYTYPE, nsMappings));
} catch (NoPrefixDeclaredException e) {
// will never happen, prefixes are created when unknown
}
@@ -584,7 +595,7 @@
}
if (node.getIncludeDescendants()) {
- Query refPropQuery = new MatchAllQuery(refProperty);
+ Query refPropQuery = createMatchAllQuery(refProperty);
context = new DescendantSelfAxisQuery(context, refPropQuery, false);
}
@@ -789,7 +800,7 @@
// the like operation always has one string value.
// no coercing, see above
if (stringValues[0].equals("%")) {
- query = new MatchAllQuery(field);
+ query = createMatchAllQuery(field);
} else {
query = new WildcardQuery(FieldNames.PROPERTIES, field, stringValues[0], transform[0]);
}
@@ -810,7 +821,7 @@
case QueryConstants.OPERATION_NE_VALUE: // !=
// match nodes with property 'field' that includes svp and mvp
BooleanQuery notQuery = new BooleanQuery();
- notQuery.add(new MatchAllQuery(field), Occur.SHOULD);
+ notQuery.add(createMatchAllQuery(field), Occur.SHOULD);
// exclude all nodes where 'field' has the term in question
for (int i = 0; i < stringValues.length; i++) {
Term t = new Term(FieldNames.PROPERTIES, FieldNames.createNamedValue(field, stringValues[i]));
@@ -836,7 +847,7 @@
// minus the nodes that have a multi-valued property 'field' and
// all values are equal to term in question
notQuery = new BooleanQuery();
- notQuery.add(new MatchAllQuery(field), Occur.SHOULD);
+ notQuery.add(createMatchAllQuery(field), Occur.SHOULD);
for (int i = 0; i < stringValues.length; i++) {
// exclude the nodes that have the term and are single valued
Term t = new Term(FieldNames.PROPERTIES, FieldNames.createNamedValue(field, stringValues[i]));
@@ -859,7 +870,7 @@
query = notQuery;
break;
case QueryConstants.OPERATION_NULL:
- query = new NotQuery(new MatchAllQuery(field));
+ query = new NotQuery(createMatchAllQuery(field));
break;
case QueryConstants.OPERATION_SIMILAR:
String uuid = "x";
@@ -874,7 +885,7 @@
query = new SimilarityQuery(uuid, analyzer);
break;
case QueryConstants.OPERATION_NOT_NULL:
- query = new MatchAllQuery(field);
+ query = createMatchAllQuery(field);
break;
default:
throw new IllegalArgumentException("Unknown relation operation: "
@@ -1081,5 +1092,21 @@
log.debug("Using literal " + literal + " as is.");
}
return (String[]) values.toArray(new String[values.size()]);
+ }
+
+ /**
+ * Depending on the index format this method returns
+ * a query that matches all nodes that have a property named 'field'
+ *
+ * @param field
+ * @return Query that matches all nodes that have a property named 'field'
+ */
+ private final Query createMatchAllQuery(String field) {
+ if (indexFormatVersion.getVersion() >= IndexFormatVersion.V2.getVersion()) {
+ // new index format style
+ return new TermQuery(new Term(FieldNames.PROPERTIES_SET, field));
+ } else {
+ return new MatchAllQuery(field);
+ }
}
}
Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java?rev=578711&r1=578710&r2=578711&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java (original)
+++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java Mon Sep 24 01:46:59 2007
@@ -206,6 +206,11 @@
private boolean reindexing = false;
/**
+ * The index format version of this multi index.
+ */
+ private final IndexFormatVersion version;
+
+ /**
* Creates a new MultiIndex.
*
* @param indexDir the base file system
@@ -285,6 +290,14 @@
// init volatile index
resetVolatileIndex();
+ // set index format version
+ IndexReader reader = getIndexReader();
+ try {
+ version = IndexFormatVersion.getVersion(reader);
+ } finally {
+ reader.close();
+ }
+
redoLogApplied = redoLog.hasEntries();
// run recovery
@@ -337,6 +350,13 @@
}
/**
+ * @return the index format version for this multi index.
+ */
+ IndexFormatVersion getIndexFormatVersion() {
+ return version;
+ }
+
+ /**
* Creates an initial index by traversing the node hierarchy starting at the
* node with <code>rootId</code>.
*
@@ -780,7 +800,7 @@
* workspace.
*/
Document createDocument(NodeState node) throws RepositoryException {
- return handler.createDocument(node, nsMappings);
+ return handler.createDocument(node, nsMappings, version);
}
/**
Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NodeIndexer.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NodeIndexer.java?rev=578711&r1=578710&r2=578711&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NodeIndexer.java (original)
+++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NodeIndexer.java Mon Sep 24 01:46:59 2007
@@ -96,6 +96,11 @@
protected boolean supportHighlighting = false;
/**
+ * Indicates index format for this node indexer.
+ */
+ protected IndexFormatVersion indexFormatVersion = IndexFormatVersion.V1;
+
+ /**
* Creates a new node indexer.
*
* @param node the node state to index.
@@ -132,6 +137,15 @@
}
/**
+ * Sets the index format version
+ *
+ * @param indexFormatVersion the index format version
+ */
+ public void setIndexFormatVersion(IndexFormatVersion indexFormatVersion) {
+ this.indexFormatVersion = indexFormatVersion;
+ }
+
+ /**
* Sets the indexing configuration for this node indexer.
*
* @param config the indexing configuration.
@@ -189,6 +203,14 @@
PropertyId id = new PropertyId(node.getNodeId(), propName);
try {
PropertyState propState = (PropertyState) stateProvider.getItemState(id);
+
+ // add each property to the _PROPERTIES_SET for searching
+ // beginning with V2
+ if (indexFormatVersion.getVersion()
+ >= IndexFormatVersion.V2.getVersion()) {
+ addPropertyName(doc, propState.getName());
+ }
+
InternalValue[] values = propState.getValues();
for (int i = 0; i < values.length; i++) {
addValue(doc, values[i], propState.getName());
@@ -312,6 +334,22 @@
}
}
+ /**
+ * Adds the property name to the lucene _:PROPERTIES_SET field.
+ *
+ * @param doc the document.
+ * @param name the name of the property.
+ */
+ private void addPropertyName(Document doc, QName name) {
+ String fieldName = name.getLocalName();
+ try {
+ fieldName = NameFormat.format(name, mappings);
+ } catch (NoPrefixDeclaredException e) {
+ // will never happen
+ }
+ doc.add(new Field(FieldNames.PROPERTIES_SET, fieldName, Field.Store.NO, Field.Index.NO_NORMS));
+ }
+
/**
* Adds the binary value to the document as the named field.
* <p/>
Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/QueryImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/QueryImpl.java?rev=578711&r1=578710&r2=578711&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/QueryImpl.java (original)
+++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/QueryImpl.java Mon Sep 24 01:46:59 2007
@@ -114,8 +114,10 @@
// build lucene query
Query query = LuceneQueryBuilder.createQuery(root, session,
- index.getContext().getItemStateManager(), index.getNamespaceMappings(),
- index.getTextAnalyzer(), propReg, index.getSynonymProvider());
+ index.getContext().getItemStateManager(),
+ index.getNamespaceMappings(), index.getTextAnalyzer(),
+ propReg, index.getSynonymProvider(),
+ index.getIndexFormatVersion());
OrderQueryNode orderNode = root.getOrderNode();
Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java?rev=578711&r1=578710&r2=578711&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java (original)
+++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java Mon Sep 24 01:46:59 2007
@@ -320,6 +320,16 @@
private SynonymProvider synProvider;
/**
+ * Indicates the index format version which is relevant to a <b>query</b>. This
+ * value may be different from what {@link MultiIndex#getIndexFormatVersion()}
+ * returns because queries may be executed on two physical indexes with
+ * different formats. Index format versions are considered backward
+ * compatible. That is, the lower version of the two physical indexes is
+ * used for querying.
+ */
+ private IndexFormatVersion indexFormatVersion;
+
+ /**
* Indicates if this <code>SearchIndex</code> is closed and cannot be used
* anymore.
*/
@@ -404,7 +414,8 @@
log.warn("Failed to run consistency check on index: " + e);
}
}
- log.info("Index initialized: " + path);
+ log.info("Index initialized: {} Version: {}",
+ new Object[]{path, index.getIndexFormatVersion()});
}
/**
@@ -461,7 +472,8 @@
removedNodeIds.remove(state.getNodeId());
Document doc = null;
try {
- doc = createDocument(state, getNamespaceMappings());
+ doc = createDocument(state, getNamespaceMappings(),
+ index.getIndexFormatVersion());
retrieveAggregateRoot(state, aggregateRoots);
} catch (RepositoryException e) {
log.warn("Exception while creating document for node: "
@@ -489,7 +501,8 @@
public Object next() {
NodeState state = (NodeState) super.next();
try {
- return createDocument(state, getNamespaceMappings());
+ return createDocument(state, getNamespaceMappings(),
+ index.getIndexFormatVersion());
} catch (RepositoryException e) {
log.warn("Exception while creating document for node: "
+ state.getNodeId() + ": " + e.toString());
@@ -687,6 +700,29 @@
}
/**
+ * Returns the index format version that this search index is able to
+ * support when a query is executed on this index.
+ *
+ * @return the index format version for this search index.
+ */
+ public IndexFormatVersion getIndexFormatVersion() {
+ if (indexFormatVersion == null) {
+ if (getContext().getParentHandler() instanceof SearchIndex) {
+ SearchIndex parent = (SearchIndex) getContext().getParentHandler();
+ if (parent.getIndexFormatVersion().getVersion()
+ < index.getIndexFormatVersion().getVersion()) {
+ indexFormatVersion = parent.getIndexFormatVersion();
+ } else {
+ indexFormatVersion = index.getIndexFormatVersion();
+ }
+ } else {
+ indexFormatVersion = index.getIndexFormatVersion();
+ }
+ }
+ return indexFormatVersion;
+ }
+
+ /**
* Returns an index reader for this search index. The caller of this method
* is responsible for closing the index reader when he is finished using
* it.
@@ -750,20 +786,24 @@
* Creates a lucene <code>Document</code> for a node state using the
* namespace mappings <code>nsMappings</code>.
*
- * @param node the node state to index.
- * @param nsMappings the namespace mappings of the search index.
+ * @param node the node state to index.
+ * @param nsMappings the namespace mappings of the search index.
+ * @param indexFormatVersion the index format version that should be used to
+ * index the passed node state.
* @return a lucene <code>Document</code> that contains all properties of
* <code>node</code>.
* @throws RepositoryException if an error occurs while indexing the
* <code>node</code>.
*/
protected Document createDocument(NodeState node,
- NamespaceMappings nsMappings)
+ NamespaceMappings nsMappings,
+ IndexFormatVersion indexFormatVersion)
throws RepositoryException {
NodeIndexer indexer = new NodeIndexer(node,
getContext().getItemStateManager(), nsMappings, extractor);
indexer.setSupportHighlighting(supportHighlighting);
indexer.setIndexingConfiguration(indexingConfig);
+ indexer.setIndexFormatVersion(indexFormatVersion);
Document doc = indexer.createDoc();
mergeAggregatedNodeIndexes(node, doc);
return doc;
@@ -892,7 +932,8 @@
}
for (int j = 0; j < aggregates.length; j++) {
Document aDoc = createDocument(aggregates[j],
- getNamespaceMappings());
+ getNamespaceMappings(),
+ index.getIndexFormatVersion());
// transfer fields to doc if there are any
Field[] fulltextFields = aDoc.getFields(FieldNames.FULLTEXT);
if (fulltextFields != null) {