You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jackrabbit.apache.org by mr...@apache.org on 2005/07/18 14:20:46 UTC
svn commit: r219477 - in /incubator/jackrabbit/trunk/core:
applications/test/ src/conf/
src/java/org/apache/jackrabbit/core/query/lucene/
Author: mreutegg
Date: Mon Jul 18 05:20:20 2005
New Revision: 219477
URL: http://svn.apache.org/viewcvs?rev=219477&view=rev
Log:
JCR-160: Query index not in sync with workspace
Extended consistency check to also include:
- Missing ancestor
- Unknown parent
- Nodes indexed more than once
Added:
incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java (with props)
incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java (with props)
Modified:
incubator/jackrabbit/trunk/core/applications/test/repository.xml
incubator/jackrabbit/trunk/core/src/conf/repository.xml
incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java
incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java
Modified: incubator/jackrabbit/trunk/core/applications/test/repository.xml
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/applications/test/repository.xml?rev=219477&r1=219476&r2=219477&view=diff
==============================================================================
--- incubator/jackrabbit/trunk/core/applications/test/repository.xml (original)
+++ incubator/jackrabbit/trunk/core/applications/test/repository.xml Mon Jul 18 05:20:20 2005
@@ -194,12 +194,19 @@
<!--
Search index and the file system it uses.
class: FQN of class implementing the QueryHandler interface
- Supported Parameters:
+
+ Supported parameters for lucene search index:
- useCompoundFile: advises lucene to use compound files for the index files
- minMergeDocs: minimum number of nodes in an index until segments are merged
- maxMergeDocs: maximum number of nodes in segments that will be merged
- mergeFactor: determines how often segment indices are merged
- - bufferSize: maximum number of documents that are held in a pending queue until added to the index
+ - bufferSize: maximum number of documents that are held in a pending
+ queue until added to the index
+ - forceConsistencyCheck: runs a consistency check on every startup. If
+ false, a consistency check is only performed when the search index
+ detects a prior forced shutdown.
+ - autoRepair: errors detected by a consistency check are automatically
+ repaired. If false, errors are only written to the log.
-->
<SearchIndex class="org.apache.jackrabbit.core.query.lucene.SearchIndex">
<param name="useCompoundFile" value="true"/>
@@ -207,7 +214,8 @@
<param name="maxMergeDocs" value="100000"/>
<param name="mergeFactor" value="10"/>
<param name="bufferSize" value="10"/>
-
+ <param name="forceConsistencyCheck" value="false"/>
+ <param name="autoRepair" value="true"/>
<FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
<param name="path" value="${wsp.home}/index"/>
</FileSystem>
Modified: incubator/jackrabbit/trunk/core/src/conf/repository.xml
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/src/conf/repository.xml?rev=219477&r1=219476&r2=219477&view=diff
==============================================================================
--- incubator/jackrabbit/trunk/core/src/conf/repository.xml (original)
+++ incubator/jackrabbit/trunk/core/src/conf/repository.xml Mon Jul 18 05:20:20 2005
@@ -194,12 +194,19 @@
<!--
Search index and the file system it uses.
class: FQN of class implementing the QueryHandler interface
- Supported Parameters:
+
+ Supported parameters for lucene search index:
- useCompoundFile: advises lucene to use compound files for the index files
- minMergeDocs: minimum number of nodes in an index until segments are merged
- maxMergeDocs: maximum number of nodes in segments that will be merged
- mergeFactor: determines how often segment indices are merged
- - bufferSize: maximum number of documents that are held in a pending queue until added to the index
+ - bufferSize: maximum number of documents that are held in a pending
+ queue until added to the index
+ - forceConsistencyCheck: runs a consistency check on every startup. If
+ false, a consistency check is only performed when the search index
+ detects a prior forced shutdown.
+ - autoRepair: errors detected by a consistency check are automatically
+ repaired. If false, errors are only written to the log.
-->
<SearchIndex class="org.apache.jackrabbit.core.query.lucene.SearchIndex">
<param name="useCompoundFile" value="true"/>
@@ -207,7 +214,8 @@
<param name="maxMergeDocs" value="100000"/>
<param name="mergeFactor" value="10"/>
<param name="bufferSize" value="10"/>
-
+ <param name="forceConsistencyCheck" value="false"/>
+ <param name="autoRepair" value="true"/>
<FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
<param name="path" value="${wsp.home}/index"/>
</FileSystem>
Added: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java?rev=219477&view=auto
==============================================================================
--- incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java (added)
+++ incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java Mon Jul 18 05:20:20 2005
@@ -0,0 +1,367 @@
+/*
+ * Copyright 2004-2005 The Apache Software Foundation or its licensors,
+ * as applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.query.lucene;
+
+import org.apache.jackrabbit.core.state.ItemStateManager;
+import org.apache.jackrabbit.core.state.NodeState;
+import org.apache.jackrabbit.core.state.ItemStateException;
+import org.apache.jackrabbit.core.NodeId;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.document.Document;
+import org.apache.log4j.Logger;
+
+import javax.jcr.RepositoryException;
+import java.io.IOException;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.HashSet;
+
+/**
+ * Implements a consistency check on the search index. Currently the following
+ * checks are implemented:
+ * <ul>
+ * <li>Does not node exist in the ItemStateManager? If it does not exist
+ * anymore the node is deleted from the index.</li>
+ * <li>Is the parent of a node also present in the index? If it is not present it
+ * will be indexed.</li>
+ * <li>Is a node indexed multiple times? If that is the case, all occurrences
+ * in the index for such a node are removed, and the node is re-indexed.</li>
+ * </ul>
+ */
+class ConsistencyCheck {
+
+ /**
+ * Logger instance for this class
+ */
+ private static final Logger log = Logger.getLogger(ConsistencyCheck.class);
+
+ /**
+ * The ItemStateManager of the workspace.
+ */
+ private final ItemStateManager stateMgr;
+
+ /**
+ * The index to check.
+ */
+ private final MultiIndex index;
+
+ /**
+ * All the documents within the index.
+ */
+ private Map documents;
+
+ /**
+ * List of all errors.
+ */
+ private final List errors = new ArrayList();
+
+ /**
+ * Private constructor.
+ */
+ private ConsistencyCheck(MultiIndex index, ItemStateManager mgr) {
+ this.index = index;
+ this.stateMgr = mgr;
+ }
+
+ /**
+ * Runs the consistency check on <code>index</code>.
+ *
+ * @param index the index to check.
+ * @param mgr the ItemStateManager from where to load content.
+ * @return the consistency check with the results.
+ * @throws IOException if an error occurs while checking.
+ */
+ static ConsistencyCheck run(MultiIndex index, ItemStateManager mgr) throws IOException {
+ ConsistencyCheck check = new ConsistencyCheck(index, mgr);
+ check.run();
+ return check;
+ }
+
+ /**
+ * Repairs detected errors during the consistency check.
+ * @param ignoreFailure if <code>true</code> repair failures are ignored
+ * the the repair continues without throwing an exception. If
+ * <code>false</code> the repair procedure is aborted on the first
+ * repair failure.
+ * @throws IOException if a repair failure occurs.
+ */
+ void repair(boolean ignoreFailure) throws IOException {
+ if (errors.size() == 0) {
+ log.info("No errors found.");
+ return;
+ }
+ int notRepairable = 0;
+ for (Iterator it = errors.iterator(); it.hasNext(); ) {
+ ConsistencyCheckError error = (ConsistencyCheckError) it.next();
+ try {
+ if (error.repairable()) {
+ error.repair();
+ } else {
+ notRepairable++;
+ }
+ } catch (IOException e) {
+ if (ignoreFailure) {
+ log.warn("Exception while reparing: " + e);
+ } else {
+ throw e;
+ }
+ }
+ }
+ log.info("Repaired " + (errors.size() - notRepairable) + " errors.");
+ if (notRepairable > 0) {
+ log.warn("" + notRepairable + " error(s) not repairable.");
+ }
+ }
+
+ /**
+ * Returns the errors detected by the consistency check.
+ * @return the errors detected by the consistency check.
+ */
+ List getErrors() {
+ return new ArrayList(errors);
+ }
+
+ /**
+ * Runs the consistency check.
+ * @throws IOException if an error occurs while running the check.
+ */
+ private void run() throws IOException {
+ // UUIDs of multiple nodes in the index
+ Set multipleEntries = new HashSet();
+ // collect all documents
+ documents = new HashMap();
+ IndexReader reader = index.getIndexReader();
+ for (int i = 0; i < reader.maxDoc(); i++) {
+ if (reader.isDeleted(i)) {
+ continue;
+ }
+ Document d = reader.document(i);
+ String uuid = d.get(FieldNames.UUID);
+ if (stateMgr.hasItemState(new NodeId(uuid))) {
+ Document old = (Document) documents.put(uuid, d);
+ if (old != null) {
+ multipleEntries.add(uuid);
+ }
+ } else {
+ errors.add(new NodeDeleted(uuid));
+ }
+ }
+
+ // create multiple entries errors
+ for (Iterator it = multipleEntries.iterator(); it.hasNext(); ) {
+ errors.add(new MultipleEntries((String) it.next()));
+ }
+
+ // run through documents
+ for (Iterator it = documents.values().iterator(); it.hasNext(); ) {
+ Document d = (Document) it.next();
+ String uuid = d.get(FieldNames.UUID);
+ String parentUUID = d.get(FieldNames.PARENT);
+ if (documents.containsKey(parentUUID) || parentUUID.length() == 0) {
+ continue;
+ }
+ // parent is missing
+ NodeId parentId = new NodeId(parentUUID);
+ if (stateMgr.hasItemState(parentId)) {
+ errors.add(new MissingAncestor(uuid, parentUUID));
+ } else {
+ errors.add(new UnknownParent(uuid, parentUUID));
+ }
+ }
+ }
+
+ /**
+ * Returns the path for <code>node</code>. If an error occurs this method
+ * returns the uuid of the node.
+ *
+ * @param node the node to retrieve the path from
+ * @return the path of the node or its uuid.
+ */
+ private String getPath(NodeState node) {
+ // remember as fallback
+ String uuid = node.getUUID();
+ StringBuffer path = new StringBuffer();
+ List elements = new ArrayList();
+ try {
+ while (node.getParentUUID() != null) {
+ NodeId parentId = new NodeId(node.getParentUUID());
+ NodeState parent = (NodeState) stateMgr.getItemState(parentId);
+ NodeState.ChildNodeEntry entry = parent.getChildNodeEntry(node.getUUID());
+ elements.add(entry);
+ node = parent;
+ }
+ for (int i = elements.size() - 1; i > -1; i--) {
+ NodeState.ChildNodeEntry entry = (NodeState.ChildNodeEntry) elements.get(i);
+ path.append('/').append(entry.getName().getLocalName());
+ if (entry.getIndex() > 1) {
+ path.append('[').append(entry.getIndex()).append(']');
+ }
+ }
+ if (path.length() == 0) {
+ path.append('/');
+ }
+ return path.toString();
+ } catch (ItemStateException e) {
+ return uuid;
+ }
+ }
+
+ //-------------------< ConsistencyCheckError classes >----------------------
+
+ /**
+ * One or more ancestors of an indexed node are not available in the index.
+ */
+ private class MissingAncestor extends ConsistencyCheckError {
+
+ private final String parentUUID;
+
+ private MissingAncestor(String uuid, String parentUUID) {
+ super("Parent of " + uuid + " missing in index. Parent: " + parentUUID, uuid);
+ this.parentUUID = parentUUID;
+ }
+
+ /**
+ * Returns <code>true</code>.
+ * @return <code>true</code>.
+ */
+ public boolean repairable() {
+ return true;
+ }
+
+ /**
+ * Repairs the missing node by indexing the missing ancestors.
+ * @throws IOException if an error occurs while repairing.
+ */
+ public void repair() throws IOException {
+ String pUUID = parentUUID;
+ while (!documents.containsKey(pUUID)) {
+ try {
+ NodeState n = (NodeState) stateMgr.getItemState(new NodeId(pUUID));
+ log.info("Reparing missing node " + getPath(n));
+ Document d = index.createDocument(n);
+ index.addDocument(d);
+ documents.put(n.getUUID(), d);
+ pUUID = n.getParentUUID();
+ } catch (ItemStateException e) {
+ throw new IOException(e.toString());
+ } catch (RepositoryException e) {
+ throw new IOException(e.toString());
+ }
+ }
+ }
+ }
+
+ /**
+ * The parent of a node is not available through the ItemStateManager.
+ */
+ private class UnknownParent extends ConsistencyCheckError {
+
+ private UnknownParent(String uuid, String parentUUID) {
+ super("Node " + uuid + " has unknown parent: " + parentUUID, uuid);
+ }
+
+ /**
+ * Not reparable (yet).
+ * @return <code>false</code>.
+ */
+ public boolean repairable() {
+ return false;
+ }
+
+ /**
+ * No operation.
+ */
+ public void repair() throws IOException {
+ log.warn("Unknown parent for " + uuid + " cannot be repaired");
+ }
+ }
+
+ /**
+ * A node is present multiple times in the index.
+ */
+ private class MultipleEntries extends ConsistencyCheckError {
+
+ MultipleEntries(String uuid) {
+ super("Multiple entries found for node " + uuid, uuid);
+ }
+
+ /**
+ * Returns <code>true</code>.
+ * @return <code>true</code>.
+ */
+ public boolean repairable() {
+ return true;
+ }
+
+ /**
+ * Removes the nodes with the identical uuids from the index and
+ * re-index the node.
+ * @throws IOException if an error occurs while repairing.
+ */
+ public void repair() throws IOException {
+ // first remove all occurrences
+ Term id = new Term(FieldNames.UUID, uuid);
+ while (index.removeDocument(id) > 0) {
+ // removed
+ }
+ // then re-index the node
+ try {
+ NodeState node = (NodeState) stateMgr.getItemState(new NodeId(uuid));
+ log.info("Re-indexing duplicate node occurrences in index: " + getPath(node));
+ Document d = index.createDocument(node);
+ index.addDocument(d);
+ documents.put(node.getUUID(), d);
+ } catch (ItemStateException e) {
+ throw new IOException(e.toString());
+ } catch (RepositoryException e) {
+ throw new IOException(e.toString());
+ }
+ }
+ }
+
+ /**
+ * Indicates that a node has been deleted but is still in the index.
+ */
+ private class NodeDeleted extends ConsistencyCheckError {
+
+ NodeDeleted(String uuid) {
+ super("Node " + uuid + " does not longer exist.", uuid);
+ }
+
+ /**
+ * Returns <code>true</code>.
+ * @return <code>true</code>.
+ */
+ public boolean repairable() {
+ return true;
+ }
+
+ /**
+ * Deletes the nodes from the index.
+ * @throws IOException if an error occurs while repairing.
+ */
+ public void repair() throws IOException {
+ log.info("Removing deleted node from index: " + uuid);
+ index.removeDocument(new Term(FieldNames.UUID, uuid));
+ }
+ }
+}
Propchange: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java?rev=219477&view=auto
==============================================================================
--- incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java (added)
+++ incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java Mon Jul 18 05:20:20 2005
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2004-2005 The Apache Software Foundation or its licensors,
+ * as applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.query.lucene;
+
+import java.io.IOException;
+
+/**
+ * Common base class for errors detected during the consistency check.
+ */
+abstract class ConsistencyCheckError {
+
+ /**
+ * Diagnostic message for this error.
+ */
+ protected final String message;
+
+ /**
+ * The UUID of the affected node.
+ */
+ protected final String uuid;
+
+ ConsistencyCheckError(String message, String uuid) {
+ this.message = message;
+ this.uuid = uuid;
+ }
+
+ /**
+ * Returns the diagnostic message.
+ * @return the diagnostic message.
+ */
+ public String toString() {
+ return message;
+ }
+
+ /**
+ * Returns <code>true</code> if this error can be repaired.
+ * @return <code>true</code> if this error can be repaired.
+ */
+ abstract boolean repairable();
+
+ /**
+ * Executes the repair operation.
+ * @throws IOException if an error occurs while repairing.
+ */
+ abstract void repair() throws IOException;
+}
Propchange: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java?rev=219477&r1=219476&r2=219477&view=diff
==============================================================================
--- incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java (original)
+++ incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java Mon Jul 18 05:20:20 2005
@@ -42,7 +42,7 @@
* A <code>MultiIndex</code> consists of a {@link VolatileIndex} and multiple
* {@link PersistentIndex}es. The goal is to keep most parts of the index open
* with index readers and write new index data to the volatile index. When
- * the volatile index reaches a certain size (see {@link #setMinMergeDocs(int)} a
+ * the volatile index reaches a certain size (see {@link SearchIndex#setMinMergeDocs(int)} a
* new persistent index is created with the index data from the volatile index.
* the new persistent index is then added to the list of already existing
* persistent indexes. Furhter operations on the new persistent index will
@@ -50,8 +50,8 @@
* but also for delete operations on the index.
* <p/>
* The persistent indexes are merged from time to time. The merge behaviour
- * is configurable using the methods: {@link #setMaxMergeDocs(int)},
- * {@link #setMergeFactor(int)} and {@link #setMinMergeDocs(int)}. For detailed
+ * is configurable using the methods: {@link SearchIndex#setMaxMergeDocs(int)},
+ * {@link SearchIndex#setMergeFactor(int)} and {@link SearchIndex#setMinMergeDocs(int)}. For detailed
* description of the configuration parameters see also the lucene
* <code>IndexWriter</code> class.
* <p/>
@@ -85,37 +85,6 @@
private static final String REDO_LOG = "redo.log";
/**
- * Compound file flag
- */
- private boolean useCompoundFile = true;
-
- /**
- * minMergeDocs config parameter
- */
- private int minMergeDocs = 1000;
-
- /**
- * maxMergeDocs config parameter
- */
- private int maxMergeDocs = 100000;
-
- /**
- * mergeFactor config parameter
- */
- private int mergeFactor = 10;
-
- /**
- * Default merge size: 1000
- */
- private static final long DEFAULT_MERGE_SIZE = 1000;
-
- /**
- * The maximum number of entries in the redo log until the volatile index
- * is merged into the persistent one.
- */
- private long mergeSize = DEFAULT_MERGE_SIZE;
-
- /**
* Names of active persistent index directories.
*/
private final IndexInfos indexNames = new IndexInfos();
@@ -152,6 +121,11 @@
private IndexReader multiReader;
/**
+ * <code>true</code> if the redo log contained entries on startup.
+ */
+ private boolean redoLogApplied = false;
+
+ /**
* Creates a new MultiIndex.
*
* @param fs the base file system
@@ -186,10 +160,10 @@
FileSystem sub = new BasedFileSystem(fs, indexNames.getName(i));
sub.init();
PersistentIndex index = new PersistentIndex(indexNames.getName(i), sub, false, handler.getAnalyzer());
- index.setMaxMergeDocs(maxMergeDocs);
- index.setMergeFactor(mergeFactor);
- index.setMinMergeDocs(minMergeDocs);
- index.setUseCompoundFile(useCompoundFile);
+ index.setMaxMergeDocs(handler.getMaxMergeDocs());
+ index.setMergeFactor(handler.getMergeFactor());
+ index.setMinMergeDocs(handler.getMinMergeDocs());
+ index.setUseCompoundFile(handler.getUseCompoundFile());
indexes.add(index);
}
@@ -198,6 +172,9 @@
RedoLog redoLog = new RedoLog(new FileSystemResource(fs, REDO_LOG));
if (redoLog.hasEntries()) {
+ // when we have entries in the redo log there is no need to reindex
+ doInitialIndex = false;
+
log.warn("Found uncommitted redo log. Applying changes now...");
// apply changes to persistent index
Iterator it = redoLog.getEntries().iterator();
@@ -217,19 +194,12 @@
maybeMergeIndexes();
log.warn("Redo changes applied.");
redoLog.clear();
- }
-
- // execute integrity check on persistent indexes with locks on startup
- for (Iterator it = indexes.iterator(); it.hasNext(); ) {
- PersistentIndex index = (PersistentIndex) it.next();
- if (index.getLockEncountered()) {
- log.info("Running integrity check on index: " + index.getName());
- index.integrityCheck(stateMgr);
- }
+ redoLogApplied = true;
}
volatileIndex = new VolatileIndex(handler.getAnalyzer(), redoLog);
volatileIndex.setUseCompoundFile(false);
+ volatileIndex.setBufferSize(handler.getBufferSize());
if (doInitialIndex) {
// index root node
@@ -255,7 +225,7 @@
synchronized void addDocument(Document doc) throws IOException {
multiReader = null;
volatileIndex.addDocument(doc);
- if (volatileIndex.getRedoLog().getSize() >= mergeSize) {
+ if (volatileIndex.getRedoLog().getSize() >= handler.getMinMergeDocs()) {
log.info("Committing in-memory index");
commit();
}
@@ -335,54 +305,24 @@
return nsMappings;
}
- //-----------------------< properties >-------------------------------------
-
- /**
- * The lucene index writer property: useCompoundFile
- */
- void setUseCompoundFile(boolean b) {
- useCompoundFile = b;
- // apply to all persistent indexes
- for (int i = 0; i < indexes.size(); i++) {
- ((PersistentIndex) indexes.get(i)).setUseCompoundFile(b);
- }
- }
-
- /**
- * The lucene index writer property: minMergeDocs
- */
- void setMinMergeDocs(int minMergeDocs) {
- this.minMergeDocs = minMergeDocs;
- // apply to all persistent indexes
- for (int i = 0; i < indexes.size(); i++) {
- ((PersistentIndex) indexes.get(i)).setMinMergeDocs(minMergeDocs);
- }
- }
-
/**
- * The lucene index writer property: maxMergeDocs
+ * Returns a lucene Document for the <code>node</code>.
+ * @param node the node to index.
+ * @return the index document.
+ * @throws RepositoryException if an error occurs while reading from the
+ * workspace.
*/
- void setMaxMergeDocs(int maxMergeDocs) {
- this.maxMergeDocs = maxMergeDocs;
- // apply to all persistent indexes
- for (int i = 0; i < indexes.size(); i++) {
- ((PersistentIndex) indexes.get(i)).setMaxMergeDocs(maxMergeDocs);
- }
+ Document createDocument(NodeState node) throws RepositoryException {
+ return handler.createDocument(node, nsMappings);
}
/**
- * The lucene index writer property: mergeFactor
+ * Returns <code>true</code> if the redo log contained entries while
+ * this index was instantiated; <code>false</code> otherwise.
+ * @return <code>true</code> if the redo log contained entries.
*/
- void setMergeFactor(int mergeFactor) {
- this.mergeFactor = mergeFactor;
- // apply to all persistent indexes
- for (int i = 0; i < indexes.size(); i++) {
- ((PersistentIndex) indexes.get(i)).setMergeFactor(mergeFactor);
- }
- }
-
- public void setBufferSize(int size) {
- volatileIndex.setBufferSize(size);
+ boolean getRedoLogApplied() {
+ return redoLogApplied;
}
//-------------------------< internal >-------------------------------------
@@ -401,10 +341,10 @@
try {
sub.init();
index = new PersistentIndex(name, sub, true, handler.getAnalyzer());
- index.setMaxMergeDocs(maxMergeDocs);
- index.setMergeFactor(mergeFactor);
- index.setMinMergeDocs(minMergeDocs);
- index.setUseCompoundFile(useCompoundFile);
+ index.setMaxMergeDocs(handler.getMaxMergeDocs());
+ index.setMergeFactor(handler.getMergeFactor());
+ index.setMinMergeDocs(handler.getMinMergeDocs());
+ index.setUseCompoundFile(handler.getUseCompoundFile());
indexes.add(index);
indexNames.addName(name);
indexNames.write(fs);
@@ -427,6 +367,7 @@
// create new volatile index
volatileIndex = new VolatileIndex(handler.getAnalyzer(), volatileIndex.getRedoLog());
volatileIndex.setUseCompoundFile(false);
+ volatileIndex.setBufferSize(handler.getBufferSize());
maybeMergeIndexes();
}
@@ -444,7 +385,7 @@
*/
private void createIndex(NodeState node, ItemStateManager stateMgr)
throws IOException, ItemStateException, RepositoryException {
- addDocument(handler.createDocument(node, nsMappings));
+ addDocument(createDocument(node));
List children = node.getChildNodeEntries();
for (Iterator it = children.iterator(); it.hasNext();) {
NodeState.ChildNodeEntry child = (NodeState.ChildNodeEntry) it.next();
@@ -468,7 +409,7 @@
throws IOException {
Document doc;
try {
- doc = handler.createDocument(node, nsMappings);
+ doc = createDocument(node);
} catch (RepositoryException e) {
log.warn("RepositoryException: " + e.getMessage());
return;
@@ -480,10 +421,10 @@
FileSystem sub = new BasedFileSystem(fs, name);
sub.init();
PersistentIndex index = new PersistentIndex(name, sub, true, handler.getAnalyzer());
- index.setMaxMergeDocs(maxMergeDocs);
- index.setMergeFactor(mergeFactor);
- index.setMinMergeDocs(minMergeDocs);
- index.setUseCompoundFile(useCompoundFile);
+ index.setMaxMergeDocs(handler.getMaxMergeDocs());
+ index.setMergeFactor(handler.getMergeFactor());
+ index.setMinMergeDocs(handler.getMinMergeDocs());
+ index.setUseCompoundFile(handler.getUseCompoundFile());
indexes.add(index);
indexNames.addName(name);
indexNames.write(fs);
@@ -518,8 +459,8 @@
/**
* Merges multiple persistent index into a single one according to the
- * properties: {@link #setMaxMergeDocs(int)}, {@link #setMergeFactor(int)}
- * and {@link #setMinMergeDocs(int)}.
+ * properties: {@link SearchIndex#setMaxMergeDocs(int)}, {@link
+ * SearchIndex#setMergeFactor(int)} and {@link SearchIndex#setMinMergeDocs(int)}.
*
* @throws IOException if an error occurs during the merge.
*/
@@ -546,9 +487,9 @@
}
// only check for merge if there are more than mergeFactor indexes
- if (indexes.size() >= mergeFactor) {
- long targetMergeDocs = minMergeDocs;
- while (targetMergeDocs <= maxMergeDocs) {
+ if (indexes.size() >= handler.getMergeFactor()) {
+ long targetMergeDocs = handler.getMinMergeDocs();
+ while (targetMergeDocs <= handler.getMaxMergeDocs()) {
// find index smaller or equal than current target size
int minIndex = indexes.size();
int mergeDocs = 0;
@@ -561,14 +502,15 @@
mergeDocs += numDocs;
}
- if (indexes.size() - (minIndex + 1) >= mergeFactor && mergeDocs < maxMergeDocs) {
+ if (indexes.size() - (minIndex + 1) >= handler.getMergeFactor()
+ && mergeDocs < handler.getMaxMergeDocs()) {
// found a merge to do
mergeIndex(minIndex + 1);
} else {
break;
}
// increase target size
- targetMergeDocs *= mergeFactor;
+ targetMergeDocs *= handler.getMergeFactor();
}
}
}
@@ -588,10 +530,10 @@
try {
sub.init();
index = new PersistentIndex(name, sub, true, handler.getAnalyzer());
- index.setMaxMergeDocs(maxMergeDocs);
- index.setMergeFactor(mergeFactor);
- index.setMinMergeDocs(minMergeDocs);
- index.setUseCompoundFile(useCompoundFile);
+ index.setMaxMergeDocs(handler.getMaxMergeDocs());
+ index.setMergeFactor(handler.getMergeFactor());
+ index.setMinMergeDocs(handler.getMinMergeDocs());
+ index.setUseCompoundFile(handler.getUseCompoundFile());
} catch (FileSystemException e) {
throw new IOException(e.getMessage());
}
@@ -627,7 +569,7 @@
}
/**
- * <b>Note: This check will be removed when Jackrabbit 1.0 is final.</b>
+ * <b>todo: This check will be removed when Jackrabbit 1.0 is final.</b>
* <p/>
* Checks if an old index format is present and moves it to the new
* subindex structure.
Modified: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java?rev=219477&r1=219476&r2=219477&view=diff
==============================================================================
--- incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java (original)
+++ incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java Mon Jul 18 05:20:20 2005
@@ -41,6 +41,8 @@
import javax.jcr.RepositoryException;
import javax.jcr.query.InvalidQueryException;
import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
/**
* Implements a {@link org.apache.jackrabbit.core.query.QueryHandler} using
@@ -67,11 +69,53 @@
private final FIFOReadWriteLock readWriteLock = new FIFOReadWriteLock();
/**
+ * minMergeDocs config parameter.
+ */
+ private int minMergeDocs = 1000;
+
+ /**
+ * maxMergeDocs config parameter
+ */
+ private int maxMergeDocs = 100000;
+
+ /**
+ * mergeFactor config parameter
+ */
+ private int mergeFactor = 10;
+
+ /**
+ * Number of documents that are buffered before they are added to the index.
+ */
+ private int bufferSize = 10;
+
+ /**
+ * Compound file flag
+ */
+ private boolean useCompoundFile = true;
+
+ /**
* Flag indicating whether document order is enable as the default ordering.
*/
private boolean documentOrder = true;
/**
+ * If set <code>true</code> the index is checked for consistency on startup.
+ * If <code>false</code> a consistency check is only performed when there
+ * are entries in the redo log on startup.
+ * <p/>
+ * Default value is: <code>false</code>.
+ */
+ private boolean forceConsistencyCheck = false;
+
+ /**
+ * If set <code>true</code> errors detected by the consistency check are
+ * repaired. If <code>false</code> the errors are only reported in the log.
+ * <p/>
+ * Default value is: <code>true</code>.
+ */
+ private boolean autoRepair = true;
+
+ /**
* Default constructor.
*/
public SearchIndex() {
@@ -87,6 +131,27 @@
QueryHandlerContext context = getContext();
index = new MultiIndex(context.getFileSystem(), this,
context.getItemStateManager(), context.getRootUUID());
+ if (index.getRedoLogApplied() || forceConsistencyCheck) {
+ log.info("Running consistency check...");
+ try {
+ ConsistencyCheck check = ConsistencyCheck.run(index,
+ context.getItemStateManager());
+ if (autoRepair) {
+ check.repair(true);
+ } else {
+ List errors = check.getErrors();
+ if (errors.size() == 0) {
+ log.info("No errors detected.");
+ }
+ for (Iterator it = errors.iterator(); it.hasNext(); ) {
+ ConsistencyCheckError err = (ConsistencyCheckError) it.next();
+ log.info(err.toString());
+ }
+ }
+ } catch (IOException e) {
+ log.warn("Failed to run consistency check on index: " + e);
+ }
+ }
} catch (FileSystemException e) {
throw new IOException(e.getMessage());
}
@@ -257,34 +322,107 @@
//--------------------------< properties >----------------------------------
+ /**
+ * The lucene index writer property: useCompoundFile
+ */
public void setUseCompoundFile(boolean b) {
- index.setUseCompoundFile(b);
+ useCompoundFile = b;
+ }
+
+ /**
+ * Returns the current value for useCompoundFile.
+ *
+ * @return the current value for useCompoundFile.
+ */
+ public boolean getUseCompoundFile() {
+ return useCompoundFile;
}
+ /**
+ * The lucene index writer property: minMergeDocs
+ */
public void setMinMergeDocs(int minMergeDocs) {
- index.setMinMergeDocs(minMergeDocs);
+ this.minMergeDocs = minMergeDocs;
}
+ /**
+ * Returns the current value for minMergeDocs.
+ *
+ * @return the current value for minMergeDocs.
+ */
+ public int getMinMergeDocs() {
+ return minMergeDocs;
+ }
+
+ /**
+ * The lucene index writer property: maxMergeDocs
+ */
public void setMaxMergeDocs(int maxMergeDocs) {
- index.setMaxMergeDocs(maxMergeDocs);
+ this.maxMergeDocs = maxMergeDocs;
+ }
+
+ /**
+ * Returns the current value for maxMergeDocs.
+ *
+ * @return the current value for maxMergeDocs.
+ */
+ public int getMaxMergeDocs() {
+ return maxMergeDocs;
}
+ /**
+ * The lucene index writer property: mergeFactor
+ */
public void setMergeFactor(int mergeFactor) {
- index.setMergeFactor(mergeFactor);
+ this.mergeFactor = mergeFactor;
}
/**
- * @deprecated redo size is equivalent to minMergeDocs. Use
- * {@link #setMinMergeDocs(int)} to set the size of the redo log.
+ * Returns the current value for the merge factor.
+ *
+ * @return the current value for the merge factor.
*/
- public void setRedoSize(int size) {
+ public int getMergeFactor() {
+ return mergeFactor;
}
+ /**
+ * @see VolatileIndex#setBufferSize(int)
+ */
public void setBufferSize(int size) {
- index.setBufferSize(size);
+ bufferSize = size;
+ }
+
+ /**
+ * Returns the current value for the buffer size.
+ *
+ * @return the current value for the buffer size.
+ */
+ public int getBufferSize() {
+ return bufferSize;
}
public void setRespectDocumentOrder(boolean docOrder) {
documentOrder = docOrder;
+ }
+
+ public boolean getRespectDocumentOrder() {
+ return documentOrder;
+ }
+
+ public void setForceConsistencyCheck(boolean b) {
+ forceConsistencyCheck = b;
+ }
+
+ public boolean getForceConsistencyCheck() {
+ return forceConsistencyCheck;
+ }
+
+ public void setAutoRepair(boolean b) {
+ autoRepair = b;
+ }
+
+ public boolean getAutoRepair() {
+ return autoRepair;
}
}