You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jackrabbit.apache.org by ch...@apache.org on 2017/03/24 05:58:12 UTC
svn commit: r1788363 [10/17] - in /jackrabbit/site/live/oak/docs: ./
architecture/ coldstandby/ features/ nodestore/ nodestore/document/
nodestore/segment/ oak-mongo-js/ oak-mongo-js/scripts/
oak-mongo-js/scripts/prettify/ oak-mongo-js/styles/ oak_api/...
Modified: jackrabbit/site/live/oak/docs/query/indexing.html
URL: http://svn.apache.org/viewvc/jackrabbit/site/live/oak/docs/query/indexing.html?rev=1788363&r1=1788362&r2=1788363&view=diff
==============================================================================
--- jackrabbit/site/live/oak/docs/query/indexing.html (original)
+++ jackrabbit/site/live/oak/docs/query/indexing.html Fri Mar 24 05:58:10 2017
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2017-03-21
+ | Generated by Apache Maven Doxia at 2017-03-24
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20170321" />
+ <meta name="Date-Revision-yyyymmdd" content="20170324" />
<meta http-equiv="Content-Language" content="en" />
<title>Jackrabbit Oak - <a name="indexing"></a> Indexing</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" />
@@ -65,6 +65,9 @@
<li> <a href="../downloads.html" title="Downloads">Downloads</a>
</li>
+
+ <li> <a href="../articles.html" title="Articles">Articles</a>
+</li>
</ul>
</li>
<li class="dropdown">
@@ -93,31 +96,42 @@
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Features and Plugins <b class="caret"></b></a>
<ul class="dropdown-menu">
- <li> <a href="../features/atomic-counter.html" title="Atomic Counter">Atomic Counter</a>
+ <li class="dropdown-submenu">
+ <a href="../nodestore/overview.html" title="Node Storage">Node Storage</a>
+ <ul class="dropdown-menu">
+ <li> <a href="../nodestore/documentmk.html" title="Document NodeStore">Document NodeStore</a>
</li>
+ <li> <a href="../nodestore/segment/overview.html" title="Segment NodeStore">Segment NodeStore</a>
+</li>
+ </ul>
+ </li>
<li> <a href="../plugins/blobstore.html" title="Blob Storage">Blob Storage</a>
</li>
- <li> <a href="../clustering.html" title="Clustering">Clustering</a>
+ <li class="dropdown-submenu">
+ <a href="../query/query.html" title="Query">Query</a>
+ <ul class="dropdown-menu">
+ <li> <a href="../query/query-engine.html" title="Query Engine">Query Engine</a>
</li>
-
- <li> <a href="../nodestore/documentmk.html" title="DocumentNodeStore">DocumentNodeStore</a>
+ <li> <a href="../query/indexing.html" title="Indexing">Indexing</a>
</li>
-
- <li> <a href="../nodestore/overview.html" title="Node Storage">Node Storage</a>
+ <li> <a href="../query/lucene.html" title="Lucene Index">Lucene Index</a>
</li>
-
- <li> <a href="../nodestore/persistent-cache.html" title="Persistent Cache">Persistent Cache</a>
+ <li> <a href="../query/property-index.html" title="Property Index">Property Index</a>
</li>
-
- <li> <a href="../query/query.html" title="Query">Query</a>
+ <li> <a href="../query/solr.html" title="Solr Index">Solr Index</a>
</li>
+ </ul>
+ </li>
<li> <a href="../security/overview.html" title="Security">Security</a>
</li>
- <li> <a href="../nodestore/segment/overview.html" title="Segment Node Store">Segment Node Store</a>
+ <li> <a href="../features/atomic-counter.html" title="Atomic Counter">Atomic Counter</a>
+</li>
+
+ <li> <a href="../features/observation.html" title="Observation">Observation</a>
</li>
</ul>
</li>
@@ -216,7 +230,7 @@
<ul class="breadcrumb">
- <li id="publishDate">Last Published: 2017-03-21</li>
+ <li id="publishDate">Last Published: 2017-03-24</li>
<li class="divider">|</li> <li id="projectVersion">Version: 1.8-SNAPSHOT</li>
@@ -254,6 +268,13 @@
<i class="none"></i>
Downloads</a>
</li>
+
+ <li>
+
+ <a href="../articles.html" title="Articles">
+ <i class="none"></i>
+ Articles</a>
+ </li>
<li class="nav-header">Concepts and Architecture</li>
<li>
@@ -285,55 +306,108 @@
Oak API</a>
</li>
<li class="nav-header">Features and Plugins</li>
-
+
<li>
- <a href="../features/atomic-counter.html" title="Atomic Counter">
+ <a href="../nodestore/overview.html" title="Node Storage">
+ <i class="icon-chevron-down"></i>
+ Node Storage</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../nodestore/documentmk.html" title="Document NodeStore">
+ <i class="icon-chevron-down"></i>
+ Document NodeStore</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../nodestore/document/node-bundling.html" title="Node Bundling">
<i class="none"></i>
- Atomic Counter</a>
+ Node Bundling</a>
</li>
-
+
<li>
- <a href="../plugins/blobstore.html" title="Blob Storage">
+ <a href="../nodestore/document/secondary-store.html" title="Secondary Store">
<i class="none"></i>
- Blob Storage</a>
+ Secondary Store</a>
</li>
-
+
+ <li>
+
+ <a href="../nodestore/persistent-cache.html" title="Persistent Cache">
+ <i class="none"></i>
+ Persistent Cache</a>
+ </li>
+
<li>
<a href="../clustering.html" title="Clustering">
<i class="none"></i>
Clustering</a>
</li>
-
+ </ul>
+ </li>
+
<li>
- <a href="../nodestore/documentmk.html" title="DocumentNodeStore">
+ <a href="../nodestore/segment/overview.html" title="Segment NodeStore">
<i class="none"></i>
- DocumentNodeStore</a>
+ Segment NodeStore</a>
</li>
+ </ul>
+ </li>
<li>
- <a href="../nodestore/overview.html" title="Node Storage">
+ <a href="../plugins/blobstore.html" title="Blob Storage">
<i class="none"></i>
- Node Storage</a>
+ Blob Storage</a>
</li>
-
+
<li>
- <a href="../nodestore/persistent-cache.html" title="Persistent Cache">
+ <a href="../query/query.html" title="Query">
+ <i class="icon-chevron-down"></i>
+ Query</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../query/query-engine.html" title="Query Engine">
<i class="none"></i>
- Persistent Cache</a>
+ Query Engine</a>
</li>
-
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>Indexing</a>
+ </li>
+
<li>
- <a href="../query/query.html" title="Query">
+ <a href="../query/lucene.html" title="Lucene Index">
<i class="none"></i>
- Query</a>
+ Lucene Index</a>
</li>
+
+ <li>
+
+ <a href="../query/property-index.html" title="Property Index">
+ <i class="none"></i>
+ Property Index</a>
+ </li>
+
+ <li>
+
+ <a href="../query/solr.html" title="Solr Index">
+ <i class="none"></i>
+ Solr Index</a>
+ </li>
+ </ul>
+ </li>
<li>
@@ -344,9 +418,16 @@
<li>
- <a href="../nodestore/segment/overview.html" title="Segment Node Store">
+ <a href="../features/atomic-counter.html" title="Atomic Counter">
+ <i class="none"></i>
+ Atomic Counter</a>
+ </li>
+
+ <li>
+
+ <a href="../features/observation.html" title="Observation">
<i class="none"></i>
- Segment Node Store</a>
+ Observation</a>
</li>
<li class="nav-header">Using Oak</li>
@@ -539,7 +620,7 @@
<ul>
-<li><a href="#oak-index-nodes">oak:index node </a></li>
+<li><a href="#oak-index-nodes">Index Definition Location</a></li>
</ul></li>
<li><a href="#sync-indexing">Synchronous Indexing</a></li>
@@ -567,14 +648,32 @@
<li><a href="#corrupt-index-handling">Isolating Corrupt Indexes</a></li>
</ul></li>
+
+<li><a href="#nrt-indexing">Near Real Time Indexing</a>
+
+<ul>
+
+<li><a href="#nrt-indexing-usage">Usage</a>
+
+<ul>
+
+<li><a href="#nrt-indexing-mode-nrt">NRT Indexing Mode - nrt</a></li>
+
+<li><a href="#nrt-indexing-mode-sync">NRT Indexing Mode - sync</a></li>
+ </ul></li>
+
+<li><a href="#nrt-indexing-cluster-setup">Cluster Setup</a></li>
+
+<li><a href="#nrt-indexing-config">Configuration</a></li>
+ </ul></li>
</ul></li>
-<li><a href="#nrt-indexing">Near Real Time Indexing</a></li>
+<li><a href="#reindexing">Reindexing</a></li>
</ul></li>
</ul>
<div class="section">
<h2><a name="overview"></a> Overview<a name="Overview"></a></h2>
-<p>For queries to perform well Oak supports indexing content stored in repository. Indexing works on diff between the base NodeState and modified NodeState. Depending on how diff is performed and when the index content gets updated there are 3 types of indexing modes</p>
+<p>For queries to perform well, Oak supports indexing of content that is stored in the repository. Indexing works by comparing different versions of the node data (technically, “diff” between the base <tt>NodeState</tt> and the modified <tt>NodeState</tt>). The indexing mode defines how comparing is performed, and when the index content gets updated:</p>
<ol style="list-style-type: decimal">
@@ -582,9 +681,9 @@
<li>Asynchronous Indexing</li>
-<li>Near real time indexing</li>
+<li>Near Real Time (NRT) Indexing</li>
</ol>
-<p>Indexing makes use of <a href="../architecture/nodestate.html#commit-editors">Commit Editors</a>. Some of the editors are <tt>IndexEditor</tt> which are responsible for updating index content based on changes in main content. Currently Oak has following in built <tt>IndexEditor</tt>s</p>
+<p>Indexing uses <a href="../architecture/nodestate.html#commit-editors">Commit Editors</a>. Some of the editors are of type <tt>IndexEditor</tt>, which are responsible for updating index content based on changes in main content. Currently, Oak has following in built editors:</p>
<ol style="list-style-type: decimal">
@@ -601,70 +700,96 @@
<ul>
+<li><a href="#nrt-indexing">Near Real Time (NRT) Indexing</a></li>
+
<li><a href="#async-index-setup">Multiple Async indexers setup via OSGi config</a></li>
<li><a href="#corrupt-index-handling">Isolating Corrupt Indexes</a></li>
</ul></div></div>
<div class="section">
<h2><a name="indexing-flow"></a> Indexing Flow<a name="Indexing_Flow"></a></h2>
-<p><tt>IndexEditor</tt> are invoked as part of commit or as part of asynchronous diff process. For both cases at some stage diff is performed between <i>before</i> and <i>after</i> state and passed to <tt>IndexUpdate</tt> which is responsible for invoking <tt>IndexEditor</tt> based on <i>discovered</i> index definitions.</p>
+<p>The <tt>IndexEditor</tt> is invoked as part of a commit (<tt>Session.save()</tt>), or as part of the asynchronous “diff” process. For both cases, at some stage “diff” is performed between the <i>before</i> and the <i>after</i> state, and passed to <tt>IndexUpdate</tt>, which is responsible for invoking the <tt>IndexEditor</tt> based on the <i>discovered</i> index definitions.</p>
<div class="section">
<h3><a name="index-defnitions"></a> Index Definitions<a name="Index_Definitions"></a></h3>
-<p>Index definitions are nodes of type <tt>oak:QueryIndexDefinition</tt> which are stored under a special node named <tt>oak:index</tt>. As part of diff traversal at each level <tt>IndexUpdate</tt> would look for <tt>oak:index</tt> nodes. The index definitions nodes have following properties</p>
+<p>Index definitions are nodes of type <tt>oak:QueryIndexDefinition</tt>, which are stored under a special node named <tt>oak:index</tt>. As part of diff traversal, at each level, <tt>IndexUpdate</tt> looks for <tt>oak:index</tt> nodes. Below is the canonical index definition structure:</p>
+
+<div class="source">
+<pre>/oak:index/indexName
+ - jcr:primaryType = "oak:QueryIndexDefinition"
+ - type (string) mandatory
+ - async (string) multiple
+ - reindex (boolean)
+</pre></div>
+<p>The index definitions nodes have the following properties:</p>
<ol style="list-style-type: decimal">
-<li><tt>type</tt> - It determines the <i>type</i> of index. For e.g. it can be <tt>property</tt>, <tt>lucene</tt>, <tt>solr</tt> etc. Based on the <tt>type</tt> <tt>IndexUpdate</tt> would look for <tt>IndexEditor</tt> of given type from registered <tt>IndexEditorProvider</tt></li>
+<li><tt>type</tt> - It determines the <i>type</i> of index. <tt>IndexUpdate</tt> looks for an <tt>IndexEditor</tt> of the given type from the registered <tt>IndexEditorProvider</tt>. For an out-of-the-box Oak setup, it can have one of the following values:
+
+<ul>
+
+<li><tt>reference</tt> - Configured with the out-of-the-box setup</li>
+
+<li><tt>counter</tt> - Configured with the out-of-the-box setup</li>
+
+<li><tt>property</tt></li>
+
+<li><tt>lucene</tt></li>
+
+<li><tt>solr</tt></li>
+ </ul></li>
-<li><tt>async</tt> - It determines if the index is to be updated synchronously or asynchronously. It can have following values
+<li><tt>async</tt> - This determines if the index is to be updated synchronously or asynchronously. It can have following values:
<ul>
-<li><tt>sync</tt> - Also the default value. It indicates that index is meant to be updated as part of commit</li>
+<li><tt>sync</tt> - The default value. It indicates that index is meant to be updated as part of each commit.</li>
<li><tt>nrt</tt> - Indicates that index is a <a href="#nrt-indexing">near real time</a> index.</li>
-<li><tt>async</tt> - Indicates that index is to be updated asynchronously. In such a case this value is used to determine the <a href="#indexing-lane">indexing lane</a></li>
+<li><tt>async</tt> - Indicates that index is to be updated asynchronously. In such a case, this value is used to determine the <a href="#indexing-lane">indexing lane</a></li>
<li>Any other value which ends in <tt>async</tt>.</li>
</ul></li>
+
+<li><tt>reindex</tt> - If set to <tt>true</tt>, reindexing is performed for that index. After reindexing is done, the property value is set to <tt>false</tt>. See <a href="#reindexing">reindexing</a> for more details.</li>
</ol>
-<p>Based on above 2 properties <tt>IndexUpdate</tt> creates <tt>IndexEditor</tt> instances as it traverses the diff and registers them with itself passing on the callbacks for various changes</p>
+<p>Based on the above two properties, the <tt>IndexUpdate</tt> creates an <tt>IndexEditor</tt> instances as it traverses the “diff”, and registers them with itself, passing on the callbacks for changes.</p>
<div class="section">
-<h4><a name="oak-index-nodes"></a>oak:index node<a name="oak:index_node"></a></h4>
-<p>Indexing logic supports placing <tt>oak:index</tt> nodes at any path. Depending on the location such indexes would only index content which are present under those paths. So for e.g. if ‘oak:index’ is present at <i>‘/content/oak:index’</i> then indexes defined under that node would only index repository state present under <i>‘/content’</i></p>
-<p>Depending on type of index one can create these index definitions under root path (‘/’) or non root paths. Currently only <tt>lucene</tt> indexes support creating index definitions at non root paths. <tt>property</tt> indexes can only be created under root path i.e. under ‘/’</p></div></div>
+<h4><a name="oak-index-nodes"></a> Index Definition Location<a name="Index_Definition_Location"></a></h4>
+<p>Indexing logic supports placing <tt>oak:index</tt> nodes at any path. Depending on the location, such indexes only index content which are present under those paths. So, for example if ‘oak:index’ is present at <i>‘/content/oak:index’</i>, then indexes defined under that node only index repository data present under <i>‘/content’</i>.</p>
+<p>Depending on the type of the index, one can create these index definitions under the root path (‘/’), or non-root paths. Currently only <tt>lucene</tt> indexes support creating index definitions at non-root paths. <tt>property</tt> indexes can only be created under the root path, that is, under ‘/’.</p></div></div>
<div class="section">
<h3><a name="sync-indexing"></a> Synchronous Indexing<a name="Synchronous_Indexing"></a></h3>
-<p>Under synchronous indexing the index content gets updates as part of commit itself. Changes to both index content and main content are done atomically in single commit. </p>
-<p>This mode is currently supported by <tt>property</tt> and <tt>reference</tt> indexes</p></div>
+<p>Under synchronous indexing, the index content gets updates as part of the commit itself. Changes to both the main content, as well as the index content, are done atomically in a single commit. </p>
+<p>This mode is currently supported by <tt>property</tt> and <tt>reference</tt> indexes.</p></div>
<div class="section">
<h3><a name="async-indexing"></a> Asynchronous Indexing<a name="Asynchronous_Indexing"></a></h3>
-<p>Asynchronous Indexing (also referred as async indexing) is performed using periodic scheduled jobs. As part of setup Oak would schedule certain periodic jobs which would perform diff of the repository content and update the index content based on that diff. </p>
-<p>Each periodic job i.e. <tt>AsyncIndexUpdate</tt> is assigned to an <a href="#indexing-lane">indexing lane</a> and is scheduled to run at certain interval. At time of execution the job would perform work</p>
+<p>Asynchronous indexing (also called async indexing) is performed using periodic scheduled jobs. As part of the setup, Oak schedules certain periodic jobs which perform diff of the repository content, and update the index content based on that. </p>
+<p>Each periodic <tt>AsyncIndexUpdate</tt> job is assigned to an <a href="#indexing-lane">indexing lane</a>, and is scheduled to run at a certain interval. At time of execution, the job performs its work:</p>
<ol style="list-style-type: decimal">
-<li>Look for last indexed state via stored checkpoint data. If such a checkpoint exist then resolve the <tt>NodeState</tt> for that checkpoint. If no such state exist or no such checkpoint is present then it treats it as initial indexing case where base state is set to empty. This state is considered as <tt>before</tt> state</li>
+<li>Look for the last indexed state via stored checkpoint data. If such a checkpoint exists, then read the <tt>NodeState</tt> for that checkpoint. If no such state exists, or no such checkpoint is present, then it treats it as initial indexing, in which case the base state is empty. This state is considered the <tt>before</tt> state.</li>
-<li>Create a checkpoint for <i>current</i> state and refer to this as <tt>after</tt> state</li>
+<li>Create a checkpoint for <i>current</i> state and refer to this as <tt>after</tt> state.</li>
-<li>Create an <tt>IndexUpdate</tt> instance bound to current <i>indexing lane</i> and trigger a diff between the <tt>before</tt> and <tt>after</tt> state</li>
+<li>Create an <tt>IndexUpdate</tt> instance bound to the current <i>indexing lane</i>, and trigger a diff between the <tt>before</tt> and the <tt>after</tt> state.</li>
-<li><tt>IndexUpdate</tt> would then pick up index definitions which are bound to current indexing lane and would create <tt>IndexEditor</tt> instances for them and pass them the diff callbacks</li>
+<li><tt>IndexUpdate</tt> will then pick up index definitions that are bound to the current indexing lane, will create <tt>IndexEditor</tt> instances for them, and pass them the diff callbacks.</li>
-<li>The diff traverses in a depth first manner and at the end of diff the <tt>IndexEditor</tt> would do final changes for current indexing run. Depending on index implementation the index data can be either stored in NodeStore itself (e.g. lucene) or in any remote store (e.g. solr)</li>
+<li>The diff traverses in a depth-first manner, and at the end of diff, the <tt>IndexEditor</tt> will do final changes for the current indexing run. Depending on the index implementation, the index data can be either stored in the NodeStore itself (for indexes of type <tt>lucene</tt>, <tt>property</tt>, and so on), or in any remote store (for type <tt>solr</tt>).</li>
-<li><tt>AsyncIndexUpdate</tt> would then update the last indexed checkpoint to current checkpoint and do a commit.</li>
+<li><tt>AsyncIndexUpdate</tt> will then update the last indexed checkpoint to the current checkpoint and do a commit.</li>
</ol>
-<p>Such async indexes are <i>eventually consistent</i> with the repository state and lag behind the latest repository state by some time. However the index content would be eventually consistent and never end up in wrong state with respect to repository state.</p>
+<p>Such async indexes are <i>eventually consistent</i> with the repository state, and lag behind the latest repository state by some time. However, the index content is eventually consistent, and never ends up in wrong state with respect to repository state.</p>
<div class="section">
<h4><a name="checkpoint"></a> Checkpoint<a name="Checkpoint"></a></h4>
-<p>Checkpoint is a mechanism whereby a client of NodeStore can request it to ensure that repository state at that time can be preserved and not garbage collected by revision garbage collection process. Later that state can be retrieved back from NodeStore by passing the checkpoint back. You can treat checkpoint like a named revision or a tag in git repo. </p>
+<p>A checkpoint is a mechanism, whereby a client of the <tt>NodeStore</tt> can request Oak to ensure that the repository state (snapshot) at that time can be preserved, and not removed by the revision garbage collection process. Later, that state can be retrieved from the NodeStore by passing the checkpoint. You can think of a checkpoint as a tag in a git repository, or as a named revision. </p>
<p>Async indexing makes use of checkpoint support to access older repository state. </p></div>
<div class="section">
<h4><a name="indexing-lane"></a> Indexing Lane<a name="Indexing_Lane"></a></h4>
-<p>Indexing lane refers to a set of indexes which are to be indexed by given async indexer. Each index definition meant for async indexing defines an <tt>async</tt> property whose value is the name of indexing lane. For e.g. consider following 2 index definitions</p>
+<p>The term “indexing lane” refers to a set of indexes which are to be updated by a given async indexer. Each index definition meant for async indexing defines an <tt>async</tt> property, whose value is the name of the indexing lane. For example, consider following two index definitions:</p>
<div class="source">
<pre>/oak:index/userIndex
@@ -675,96 +800,210 @@
- jcr:primaryType = "oak:QueryIndexDefinition"
- async = "fulltext-async"
</pre></div>
-<p>Here <i>userIndex</i> is bound to “async” indexing lane while <i>assetIndex</i> is bound to “fulltext-async” lane. Oak <a href="#async-index-setup">setup</a> would configure 2 <tt>AsyncIndexUpdate</tt> jobs one for “async” and one for “fulltext-async”. When job for “async” would run it would only process index definition where <tt>async</tt> value is <tt>async</tt> while when job for “fulltext-async” would run it would pick up index definitions where <tt>async</tt> value is <tt>fulltext-async</tt>.</p>
-<p>These jobs can be scheduled to run at different intervals and also on different cluster nodes. Each job would keep its own bookkeeping of checkpoint state and can be <a href="#async-index-mbean">paused and resumed</a> separately.</p>
-<p>Prior to Oak 1.4 there was only one indexing lane <tt>async</tt>. In Oak 1.4 support was added to create 2 lanes <tt>async</tt> and <tt>fulltext-async</tt>. With 1.6 its possible to <a href="#async-index-setup">create multiple lanes</a>. </p></div>
+<p>Here, <i>userIndex</i> is bound to the “async” indexing lane, while <i>assetIndex</i> is bound to the “fulltext-async” lane. Oak <a href="#async-index-setup">setup</a> configures two <tt>AsyncIndexUpdate</tt> jobs: one for “async”, and one for “fulltext-async”. When the job for “async” is run, it only processes index definition where the <tt>async</tt> value is <tt>async</tt>, while when the job for “fulltext-async” is run, it only pick up index definitions where the <tt>async</tt> value is <tt>fulltext-async</tt>.</p>
+<p>These jobs can be scheduled to run at different intervals, and also on different cluster nodes. Each job keeps its own bookkeeping of checkpoint state, and can be <a href="#async-index-mbean">paused and resumed</a> separately.</p>
+<p>Prior to Oak 1.4, there was only one indexing lane: <tt>async</tt>. In Oak 1.4, support was added to create two lanes: <tt>async</tt> and <tt>fulltext-async</tt>. With 1.6, it is possible to <a href="#async-index-setup">create multiple lanes</a>. </p></div>
<div class="section">
<h4><a name="cluster"></a> Clustered Setup<a name="Clustered_Setup"></a></h4>
-<p>In a clustered setup it needs to be ensured by the host application that async indexing jobs for specific lanes are to be run as singleton in the cluster. If <tt>AsyncIndexUpdate</tt> for same lane gets executed concurrently on different cluster nodes then it can lead to race conditions where old checkpoint gets lost leading to reindexing of the indexes.</p>
-<p>Refer to <a href="../clustering.html#scheduled-jobs">clustering</a> for more details on how the host application should schedule such indexing jobs</p>
+<p>In a clustered setup, one needs to ensure in the host application that the async indexing jobs for all lanes are run as singleton in the cluster. If <tt>AsyncIndexUpdate</tt> for the same lane is executed concurrently on different cluster nodes, it leads to race conditions, where an old checkpoint gets lost, leading to reindexing.</p>
+<p>See also <a href="../clustering.html#scheduled-jobs">clustering</a> for more details on how the host application should schedule such indexing jobs.</p>
<div class="section">
<h5><a name="async-index-lease"></a> Indexing Lease<a name="Indexing_Lease"></a></h5>
-<p><tt>AsyncIndexUpdate</tt> has an inbuilt lease logic to ensure that even if the jobs gets scheduled to run on different cluster nodes then also only one of them runs. This is done by keeping a lease property which gets periodically updated as indexing progresses. </p>
-<p>An <tt>AsyncIndexUpdate</tt> run would skip indexing if current lease has not expired i.e. if the last update of lease was done long ago (default 15 mins) then it would be assumed that cluster node doing indexing is not available and some other node would take over.</p>
-<p>The lease logic can delay start of indexing if the system is not stopped cleanly. As of Oak 1.6 this does not affect non clustered setup like those based on SegmentNodeStore but only <a class="externalLink" href="https://issues.apache.org/jira/browse/OAK-5159">affects DocumentNodeStore</a> based setups </p></div></div>
+<p><tt>AsyncIndexUpdate</tt> has an in-built “lease” logic to ensure that even if the jobs gets scheduled to run on different cluster nodes, only one of them runs. This is done by keeping a lease property, which gets periodically updated as indexing progresses. </p>
+<p>An <tt>AsyncIndexUpdate</tt> run skips indexing if the current lease has not expired. If the last update of the lease was done too long ago (default: more than 15 minutes), it is assumed that cluster node that is supposed to index is not available, and some other node will take over.</p>
+<p>The lease logic can delay the start of indexing if the system is not stopped cleanly. As of Oak 1.6, this does not affect non-clustered setups like those based on SegmentNodeStore, but only <a class="externalLink" href="https://issues.apache.org/jira/browse/OAK-5159">affects DocumentNodeStore</a> based setups.</p></div></div>
<div class="section">
<h4><a name="async-index-lag"></a> Indexing Lag<a name="Indexing_Lag"></a></h4>
-<p>Async indexing jobs are by default configured to run at interval of 5 secs. Depending on the system load and diff size of content to be indexed the indexing may start lagging by longer time intervals. Due to this the indexing results would lag behind the repository state and may become stale i.e. new content added would show up in result after some time.</p>
-<p><tt>IndexStats</tt> MBean keeps a time series and metrics stats for the indexing frequency. This can be used to track the indexing state</p>
-<p><a href="#nrt-indexing">NRT Indexing</a> introduced in Oak 1.6 would help in such situations and can keep the results more upto date</p></div>
+<p>Async indexing jobs are by default configured to run at an interval of 5 seconds. Depending on the system load and diff size of content to be indexed, the indexing may start lagging by a longer time interval. Due to this, the indexing results can lag behind the repository state, and may become stale, that means new content added will only show up in query results after a longer time.</p>
+<p>The <tt>IndexStats</tt> MBean keeps a time series and metrics stats for the indexing frequency. This can be used to track the indexing state.</p>
+<p><a href="#nrt-indexing">NRT Indexing</a> introduced in Oak 1.6 helps in such situations, and can keep the results more up to date.</p></div>
<div class="section">
<h4><a name="async-index-setup"></a> Setup<a name="Setup"></a></h4>
-<p><tt>Since 1.6</tt></p>
-<p>Async indexers can be configure via OSGi config for <tt>org.apache.jackrabbit.oak.plugins.index.AsyncIndexerService</tt></p>
+<p><tt>@since Oak 1.6</tt></p>
+<p>Async indexers can be configure via the OSGi config for <tt>org.apache.jackrabbit.oak.plugins.index.AsyncIndexerService</tt>.</p>
<p><img src="async-index-config.png" alt="Async Indexing Config" /></p>
-<p>Different lanes can be configured by adding more rows of <i>Async Indexer Configs</i>. Prior to 1.6 the indexers were created programatically while constructing Oak.</p></div>
+<p>Different lanes can be configured by adding more rows of <i>Async Indexer Configs</i>. Prior to 1.6, the indexers were created programatically while constructing Oak.</p></div>
<div class="section">
<h4><a name="async-index-mbean"></a> Async Indexing MBean<a name="Async_Indexing_MBean"></a></h4>
-<p>For each configured async indexer in the setup the indexer exposes a <tt>IndexStatsMBean</tt> which provides various stats around current indexing state. </p>
+<p>For each configured async indexer in the setup, the indexer exposes a <tt>IndexStatsMBean</tt>, which provides various stats around the current indexing state:</p>
<div class="source">
<pre>org.apache.jackrabbit.oak: async (IndexStats)
org.apache.jackrabbit.oak: fulltext-async (IndexStats)
</pre></div>
-<p>It provide details like</p>
+<p>It provide the following details:</p>
<ul>
-<li>FailingIndexStats - Stats around indexes which are <a href="#corrupt-index-handling">failing and marked as corrupt</a></li>
+<li>FailingIndexStats - Stats around indexes which are <a href="#corrupt-index-handling">failing and marked as corrupt</a>.</li>
-<li>LastIndexedTime - Time upto which repository state has been indexed</li>
+<li>LastIndexedTime - Time up to which the repository state has been indexed.</li>
-<li>Status - running, done, failing etc</li>
+<li>Status - running, done, failing etc.</li>
-<li>Failing - boolean flag indicating that indexing has been failing due to some issue. This can be monitored for detecting if indexer is healthy or not</li>
+<li>Failing - boolean flag indicating that indexing has been failing due to some issue. This can be monitored for detecting if indexer is healthy or not.</li>
-<li>ExecutionCount - Time series data around when number of execution for various time intervals</li>
+<li>ExecutionCount - Time series data around the number of runs for various time intervals.</li>
</ul>
-<p>Further it provides operations like</p>
+<p>Further it provides the following operations:</p>
<ul>
-<li>pause - Pauses the indexer</li>
+<li>pause - Pauses the indexer.</li>
-<li>abortAndPause - Aborts any running indexing cycle and pauses the indexer. Invoke ‘resume’ once you are ready to resume indexing again</li>
+<li>abortAndPause - Aborts any running indexing cycle and pauses the indexer. Invoke ‘resume’ once you are ready to resume indexing again.</li>
-<li>resume - Resume the indexing</li>
+<li>resume - Resume indexing.</li>
</ul></div>
<div class="section">
<h4><a name="corrupt-index-handling"></a> Isolating Corrupt Indexes<a name="Isolating_Corrupt_Indexes"></a></h4>
<p><tt>Since 1.6</tt></p>
-<p>AsyncIndexerService would now mark any index which fails to update for 30 mins (configurable) as <tt>corrupt</tt> and ignore such indexes from further indexing. </p>
-<p>When any index is marked as corrupt following log entry would be made</p>
+<p>The <tt>AsyncIndexerService</tt> marks any index which fails to update for 30 minutes (configurable) as <tt>corrupt</tt>, and ignore such indexes from further indexing. </p>
+<p>When any index is marked as corrupt, the following log entry is made:</p>
<div class="source">
-<pre>2016-11-22 12:52:35,484 INFO NA [async-index-update-fulltext-async] o.a.j.o.p.i.AsyncIndexUpdate - Marking
-[/oak:index/lucene] as corrupt. The index is failing since Tue Nov 22 12:51:25 IST 2016 ,1 indexing cycles, failed
-7 times, skipped 0 time
+<pre>2016-11-22 12:52:35,484 INFO NA [async-index-update-fulltext-async] o.a.j.o.p.i.AsyncIndexUpdate -
+Marking [/oak:index/lucene] as corrupt. The index is failing since Tue Nov 22 12:51:25 IST 2016,
+1 indexing cycles, failed 7 times, skipped 0 time
</pre></div>
-<p>Post this when any new content gets indexed and any such corrupt index is skipped then following warn entry would be made</p>
+<p>Post this, when any new content gets indexed and any such corrupt index is skipped, the following warn entry is made:</p>
<div class="source">
-<pre>2016-11-22 12:52:35,485 WARN NA [async-index-update-fulltext-async] o.a.j.o.p.index.IndexUpdate - Ignoring corrupt
-index [/oak:index/lucene] which has been marked as corrupt since [2016-11-22T12:51:25.492+05:30]. This index MUST be
-reindexed for indexing to work properly
+<pre>2016-11-22 12:52:35,485 WARN NA [async-index-update-fulltext-async] o.a.j.o.p.index.IndexUpdate -
+Ignoring corrupt index [/oak:index/lucene] which has been marked as corrupt since
+[2016-11-22T12:51:25.492+05:30]. This index MUST be reindexed for indexing to work properly
</pre></div>
-<p>This info would also be seen in MBean</p>
+<p>This info is also seen in the MBean</p>
<p><img src="corrupt-index-mbean.png" alt="Corrupt Index stats in IndexStatsMBean" /></p>
-<p>Later once the index is reindexed following log entry would be made</p>
+<p>Later, once the index is reindexed, the following log entry is made</p>
+
+<div class="source">
+<pre>2016-11-22 12:56:25,486 INFO NA [async-index-update-fulltext-async] o.a.j.o.p.index.IndexUpdate -
+Removing corrupt flag from index [/oak:index/lucene] which has been marked as corrupt since
+[corrupt = 2016-11-22T12:51:25.492+05:30]
+</pre></div>
+<p>This feature can be disabled by setting <tt>failingIndexTimeoutSeconds</tt> to 0 in the <tt>AsyncIndexService</tt> config. See also <a class="externalLink" href="https://issues.apache.org/jira/browse/OAK-4939">OAK-4939</a> for more details.</p></div></div>
+<div class="section">
+<h3><a name="nrt-indexing"></a> Near Real Time Indexing<a name="Near_Real_Time_Indexing"></a></h3>
+<p><tt>@since Oak 1.6</tt></p>
+<p><i>This mode is only supported for <tt>lucene</tt> indexes</i></p>
+<p>Lucene indexes perform well for evaluating complex queries, and have the benefit of being evaluated locally with copy-on-read support. However, they are <tt>async</tt>, and depending on system load can lag behind the repository state. For cases where such lag (which can be in the order of minutes) is not acceptable, one must use <tt>property</tt> indexes. To avoid that, Oak 1.6 has <a class="externalLink" href="https://issues.apache.org/jira/browse/OAK-4412">added support for near real time indexing</a></p>
+<p><img src="index-nrt.png" alt="NRT Index Flow" /></p>
+<p>In this mode, the indexing happen in two modes, and a query will consult multiple indexes. The diagram above shows the indexing flow with time. In the above flow:</p>
+
+<ul>
+
+<li>T1, T3 and T5 - Time instances at which checkpoints are created.</li>
+
+<li>T2 and T4 - Time instance when async indexer runs completed and indexes were updated.</li>
+
+<li>Persisted Index:
+
+<ul>
+
+<li>v2 - Index version v2, which has repository state indexed up to T1.</li>
+
+<li>v3 - Index version v2, which has repository state indexed up to T3.</li>
+ </ul></li>
+
+<li>Local Index:
+
+<ul>
+
+<li>NRT1 - Local index, which has repository state indexed between T2 and T4.</li>
+
+<li>NRT2 - Local index, which has repository state indexed between T4 and T6.</li>
+ </ul></li>
+</ul>
+<p>As the repository state changes with time, the Async indexer will run and index the changes between the last known checkpoint and current state when that run started. So when async run 1 completed, the persisted index has the repository state indexed up to T3.</p>
+<p>Now without NRT index support, if any query is performed between T2 and T4, it can only see index results for the repository state at T1, as that is the state where the persisted indexes have data for. Any change after that cannot be seen until the next async indexing cycle is complete (at T4). </p>
+<p>With NRT indexing support, indexing will happen at two places:</p>
+
+<ul>
+
+<li>Persisted Index - This is the index which is updated via the async indexer run. This flow remains the same, it will be periodically updated by the indexer run.</li>
+
+<li>Local Index - In addition to persisted index, each cluster node will also maintain a local index. This index only keeps data between two async indexer runs. Post each run, the previous index is discarded, and a new index is built (actually, the previous index is retained for one cycle).</li>
+</ul>
+<p>Any query making use of such an index will automatically make use of both the persisted and the local indexes. With this, new content added in the repository after the last async index run will also show up quickly.</p>
+<div class="section">
+<h4><a name="nrt-indexing-usage"></a> Usage<a name="Usage"></a></h4>
+<p>NRT (Near real time) indexing can be enabled for an index by configuring the <tt>async</tt> property:</p>
<div class="source">
-<pre>2016-11-22 12:56:25,486 INFO NA [async-index-update-fulltext-async] o.a.j.o.p.index.IndexUpdate - Removing corrupt
-flag from index [/oak:index/lucene] which has been marked as corrupt since [corrupt = 2016-11-22T12:51:25.492+05:30]
+<pre>/oak:index/assetIndex
+ - jcr:primaryType = "oak:QueryIndexDefinition"
+ - async = ['fulltext-async', 'nrt']
</pre></div>
-<p>This feature can be disabled by setting <tt>failingIndexTimeoutSeconds</tt> to 0 in AsyncIndexService config. Refer to <a class="externalLink" href="https://issues.apache.org/jira/browse/OAK-4939">OAK-4939</a> for more details</p></div></div></div>
+<p>Here, <tt>async</tt> has been set to a multi-valued property, with the</p>
+
+<ul>
+
+<li>Indexing lane - For example <tt>async</tt> or <tt>fulltext-async</tt>,</li>
+
+<li>NRT Indexing Mode - <tt>nrt</tt> or <tt>sync</tt>.</li>
+</ul>
<div class="section">
-<h2><a name="nrt-indexing"></a> Near Real Time Indexing<a name="Near_Real_Time_Indexing"></a></h2></div>
+<h5><a name="nrt-indexing-mode-nrt"></a> NRT Indexing Mode - nrt<a name="NRT_Indexing_Mode_-_nrt"></a></h5>
+<p>In this mode, the local index is updated asynchronously on that cluster nodes post each commit, and the index reader is refreshed each second. So, any change done should show up on that cluster node within 1 to 2 seconds.</p>
+
+<div class="source">
+<pre>/oak:index/userIndex
+ - jcr:primaryType = "oak:QueryIndexDefinition"
+ - async = ['async', 'nrt']
+</pre></div></div>
<div class="section">
-<h2>Index Types<a name="Index_Types"></a></h2>
+<h5><a name="nrt-indexing-mode-sync"></a> NRT Indexing Mode - sync<a name="NRT_Indexing_Mode_-_sync"></a></h5>
+<p>In this mode, the local index is updated synchronously on that cluster nodes post each commit, and the index reader is refreshed immediately. This mode indexes more slowly compared to the “nrt” mode.</p>
+
+<div class="source">
+<pre>/oak:index/userIndex
+ - jcr:primaryType = "oak:QueryIndexDefinition"
+ - async = ['async', 'sync']
+</pre></div>
+<p>For a single node setup (for example with the <tt>SegmentNodeStore</tt>), this mode effectively makes async lucene index perform same as synchronous property indexes. However, the ‘nrt’ mode performs better, so using that is preferable.</p></div></div>
+<div class="section">
+<h4><a name="nrt-indexing-cluster-setup"></a> Cluster Setup<a name="Cluster_Setup"></a></h4>
+<p>In cluster setup, each cluster node maintains its own local index for changes happening in that cluster node. In addition to that, it also indexes changes from other cluster nodes by relying on <a class="externalLink" href="https://issues.apache.org/jira/browse/OAK-4808">Oak observation for external changes</a>. This depends on how frequently external changes are delivered. Due to this, even with NRT indexing changes from other cluster nodes will take some more time to be reflected in query results compared to local changes.</p></div>
<div class="section">
-<h3>Property Indexes<a name="Property_Indexes"></a></h3></div>
+<h4><a name="nrt-indexing-config"></a> Configuration<a name="Configuration"></a></h4>
+<p>NRT indexing expose a few configuration options as part of the <a href="lucene.html#osgi-config">LuceneIndexProviderService</a>:</p>
+
+<ul>
+
+<li><tt>enableHybridIndexing</tt> - Boolean property, defaults to <tt>true</tt>. Can be set to <tt>false</tt> to disable the NRT indexing feature completely.</li>
+
+<li><tt>hybridQueueSize</tt> - The size of the in-memory queue used to hold Lucene documents for indexing in the <tt>nrt</tt> mode. The default size is 10000.</li>
+</ul></div></div></div>
<div class="section">
-<h3>Lucene Indexes<a name="Lucene_Indexes"></a></h3></div></div>
+<h2><a name="reindexing"></a> Reindexing<a name="Reindexing"></a></h2>
+<p>Reindexing of existing indexes is required in the following scenarios:</p>
+
+<ul>
+
+<li>Incompatible changes in the index definition - Needed after adding a property to an index definition, if content nodes with this property are already present.</li>
+
+<li>Corrupted Index - If the index is corrupt, and <tt>AsyncIndexUpdate</tt> run fails with an exception pointing to index being corrupt.</li>
+</ul>
+<p>Reindexing does not resolve other problems, such that queries not returning data. For such cases, it is <i>not</i> recommended to reindex (also because this can be very slow and use a lot of temporary disk space). If queries don’t return the right data, then possibly the index is <a class="externalLink" href="https://issues.apache.org/jira/browse/OAK-5159">not yet up-to-date</a>, or the query is incorrect, or included/excluded path settings are wrong (for Lucene indexes). Instead of reindexing, it is suggested to first check the log file, modify the query so it uses a different index or traversal and run the query again. One case where reindexing can help is if the query engine picks a very slow index for some queries because the counter index <a class="externalLink" href="https://issues.apache.org/jira/browse/OAK-4065">got out of sync after adding and removing lots of nodes many times (fixed in recent version)</a>. For this case, it is recommended to verify the contents o
f the counter index first, and upgrade Oak before reindexing.</p>
+<p>Also, note that with Oak 1.6, for Lucene indexes, changes in the index definition are only effective <a href="lucene.html#stored-index-definition">post reindexing</a>.</p>
+<p>To reindex, set the <tt>reindex</tt> property to <tt>true</tt> in the respective index definition:</p>
+
+<div class="source">
+<pre>/oak:index/userIndex
+ - jcr:primaryType = "oak:QueryIndexDefinition"
+ - async = ['async']
+ - reindex = true
+</pre></div>
+<p>Once changes are saved, the index is reindexed. For synchronous indexes, the reindexing is done as part of save (or commit) itself. While for asynchronous indexes, reindex starts with the next async indexing cycle. Once reindexing starts, the following log entries can be seen in the log:</p>
+
+<div class="source">
+<pre>[async-index-update-async] o.a.j.o.p.i.IndexUpdate Reindexing will be performed for following indexes: [/oak:index/userIndex]
+[async-index-update-async] o.a.j.o.p.i.IndexUpdate Reindexing Traversed #100000 /home/user/admin
+[async-index-update-async] o.a.j.o.p.i.AsyncIndexUpdate [async] Reindexing completed for indexes: [/oak:index/userIndex*(4407016)] in 30 min
+</pre></div>
+<p>In both cases, once reindexing is complete, the <tt>reindex</tt> flag is removed.</p>
+<p>For a property index, you can also make use of the <tt>PropertyIndexAsyncReindexMBean</tt>. See also the <a href="property-index.html#reindexing">reindeinxing property indexes</a> section for more details on that.</p></div>
</div>
</div>
</div>
Modified: jackrabbit/site/live/oak/docs/query/lucene-old.html
URL: http://svn.apache.org/viewvc/jackrabbit/site/live/oak/docs/query/lucene-old.html?rev=1788363&r1=1788362&r2=1788363&view=diff
==============================================================================
--- jackrabbit/site/live/oak/docs/query/lucene-old.html (original)
+++ jackrabbit/site/live/oak/docs/query/lucene-old.html Fri Mar 24 05:58:10 2017
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2017-03-21
+ | Generated by Apache Maven Doxia at 2017-03-24
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20170321" />
+ <meta name="Date-Revision-yyyymmdd" content="20170324" />
<meta http-equiv="Content-Language" content="en" />
<title>Jackrabbit Oak - Lucene Index</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" />
@@ -65,6 +65,9 @@
<li> <a href="../downloads.html" title="Downloads">Downloads</a>
</li>
+
+ <li> <a href="../articles.html" title="Articles">Articles</a>
+</li>
</ul>
</li>
<li class="dropdown">
@@ -93,31 +96,42 @@
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Features and Plugins <b class="caret"></b></a>
<ul class="dropdown-menu">
- <li> <a href="../features/atomic-counter.html" title="Atomic Counter">Atomic Counter</a>
+ <li class="dropdown-submenu">
+ <a href="../nodestore/overview.html" title="Node Storage">Node Storage</a>
+ <ul class="dropdown-menu">
+ <li> <a href="../nodestore/documentmk.html" title="Document NodeStore">Document NodeStore</a>
</li>
+ <li> <a href="../nodestore/segment/overview.html" title="Segment NodeStore">Segment NodeStore</a>
+</li>
+ </ul>
+ </li>
<li> <a href="../plugins/blobstore.html" title="Blob Storage">Blob Storage</a>
</li>
- <li> <a href="../clustering.html" title="Clustering">Clustering</a>
+ <li class="dropdown-submenu">
+ <a href="../query/query.html" title="Query">Query</a>
+ <ul class="dropdown-menu">
+ <li> <a href="../query/query-engine.html" title="Query Engine">Query Engine</a>
</li>
-
- <li> <a href="../nodestore/documentmk.html" title="DocumentNodeStore">DocumentNodeStore</a>
+ <li> <a href="../query/indexing.html" title="Indexing">Indexing</a>
</li>
-
- <li> <a href="../nodestore/overview.html" title="Node Storage">Node Storage</a>
+ <li> <a href="../query/lucene.html" title="Lucene Index">Lucene Index</a>
</li>
-
- <li> <a href="../nodestore/persistent-cache.html" title="Persistent Cache">Persistent Cache</a>
+ <li> <a href="../query/property-index.html" title="Property Index">Property Index</a>
</li>
-
- <li> <a href="../query/query.html" title="Query">Query</a>
+ <li> <a href="../query/solr.html" title="Solr Index">Solr Index</a>
</li>
+ </ul>
+ </li>
<li> <a href="../security/overview.html" title="Security">Security</a>
</li>
- <li> <a href="../nodestore/segment/overview.html" title="Segment Node Store">Segment Node Store</a>
+ <li> <a href="../features/atomic-counter.html" title="Atomic Counter">Atomic Counter</a>
+</li>
+
+ <li> <a href="../features/observation.html" title="Observation">Observation</a>
</li>
</ul>
</li>
@@ -216,7 +230,7 @@
<ul class="breadcrumb">
- <li id="publishDate">Last Published: 2017-03-21</li>
+ <li id="publishDate">Last Published: 2017-03-24</li>
<li class="divider">|</li> <li id="projectVersion">Version: 1.8-SNAPSHOT</li>
@@ -254,6 +268,13 @@
<i class="none"></i>
Downloads</a>
</li>
+
+ <li>
+
+ <a href="../articles.html" title="Articles">
+ <i class="none"></i>
+ Articles</a>
+ </li>
<li class="nav-header">Concepts and Architecture</li>
<li>
@@ -285,55 +306,110 @@
Oak API</a>
</li>
<li class="nav-header">Features and Plugins</li>
-
+
<li>
- <a href="../features/atomic-counter.html" title="Atomic Counter">
+ <a href="../nodestore/overview.html" title="Node Storage">
+ <i class="icon-chevron-down"></i>
+ Node Storage</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../nodestore/documentmk.html" title="Document NodeStore">
+ <i class="icon-chevron-down"></i>
+ Document NodeStore</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../nodestore/document/node-bundling.html" title="Node Bundling">
<i class="none"></i>
- Atomic Counter</a>
+ Node Bundling</a>
</li>
-
+
<li>
- <a href="../plugins/blobstore.html" title="Blob Storage">
+ <a href="../nodestore/document/secondary-store.html" title="Secondary Store">
<i class="none"></i>
- Blob Storage</a>
+ Secondary Store</a>
</li>
-
+
+ <li>
+
+ <a href="../nodestore/persistent-cache.html" title="Persistent Cache">
+ <i class="none"></i>
+ Persistent Cache</a>
+ </li>
+
<li>
<a href="../clustering.html" title="Clustering">
<i class="none"></i>
Clustering</a>
</li>
-
+ </ul>
+ </li>
+
<li>
- <a href="../nodestore/documentmk.html" title="DocumentNodeStore">
+ <a href="../nodestore/segment/overview.html" title="Segment NodeStore">
<i class="none"></i>
- DocumentNodeStore</a>
+ Segment NodeStore</a>
</li>
+ </ul>
+ </li>
<li>
- <a href="../nodestore/overview.html" title="Node Storage">
+ <a href="../plugins/blobstore.html" title="Blob Storage">
<i class="none"></i>
- Node Storage</a>
+ Blob Storage</a>
</li>
-
+
<li>
- <a href="../nodestore/persistent-cache.html" title="Persistent Cache">
+ <a href="../query/query.html" title="Query">
+ <i class="icon-chevron-down"></i>
+ Query</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../query/query-engine.html" title="Query Engine">
<i class="none"></i>
- Persistent Cache</a>
+ Query Engine</a>
</li>
-
+
<li>
- <a href="../query/query.html" title="Query">
+ <a href="../query/indexing.html" title="Indexing">
<i class="none"></i>
- Query</a>
+ Indexing</a>
+ </li>
+
+ <li>
+
+ <a href="../query/lucene.html" title="Lucene Index">
+ <i class="none"></i>
+ Lucene Index</a>
+ </li>
+
+ <li>
+
+ <a href="../query/property-index.html" title="Property Index">
+ <i class="none"></i>
+ Property Index</a>
+ </li>
+
+ <li>
+
+ <a href="../query/solr.html" title="Solr Index">
+ <i class="none"></i>
+ Solr Index</a>
</li>
+ </ul>
+ </li>
<li>
@@ -344,9 +420,16 @@
<li>
- <a href="../nodestore/segment/overview.html" title="Segment Node Store">
+ <a href="../features/atomic-counter.html" title="Atomic Counter">
+ <i class="none"></i>
+ Atomic Counter</a>
+ </li>
+
+ <li>
+
+ <a href="../features/observation.html" title="Observation">
<i class="none"></i>
- Segment Node Store</a>
+ Observation</a>
</li>
<li class="nav-header">Using Oak</li>
Modified: jackrabbit/site/live/oak/docs/query/lucene.html
URL: http://svn.apache.org/viewvc/jackrabbit/site/live/oak/docs/query/lucene.html?rev=1788363&r1=1788362&r2=1788363&view=diff
==============================================================================
--- jackrabbit/site/live/oak/docs/query/lucene.html (original)
+++ jackrabbit/site/live/oak/docs/query/lucene.html Fri Mar 24 05:58:10 2017
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2017-03-21
+ | Generated by Apache Maven Doxia at 2017-03-24
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20170321" />
+ <meta name="Date-Revision-yyyymmdd" content="20170324" />
<meta http-equiv="Content-Language" content="en" />
<title>Jackrabbit Oak - Lucene Index</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" />
@@ -65,6 +65,9 @@
<li> <a href="../downloads.html" title="Downloads">Downloads</a>
</li>
+
+ <li> <a href="../articles.html" title="Articles">Articles</a>
+</li>
</ul>
</li>
<li class="dropdown">
@@ -93,31 +96,42 @@
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Features and Plugins <b class="caret"></b></a>
<ul class="dropdown-menu">
- <li> <a href="../features/atomic-counter.html" title="Atomic Counter">Atomic Counter</a>
+ <li class="dropdown-submenu">
+ <a href="../nodestore/overview.html" title="Node Storage">Node Storage</a>
+ <ul class="dropdown-menu">
+ <li> <a href="../nodestore/documentmk.html" title="Document NodeStore">Document NodeStore</a>
</li>
+ <li> <a href="../nodestore/segment/overview.html" title="Segment NodeStore">Segment NodeStore</a>
+</li>
+ </ul>
+ </li>
<li> <a href="../plugins/blobstore.html" title="Blob Storage">Blob Storage</a>
</li>
- <li> <a href="../clustering.html" title="Clustering">Clustering</a>
+ <li class="dropdown-submenu">
+ <a href="../query/query.html" title="Query">Query</a>
+ <ul class="dropdown-menu">
+ <li> <a href="../query/query-engine.html" title="Query Engine">Query Engine</a>
</li>
-
- <li> <a href="../nodestore/documentmk.html" title="DocumentNodeStore">DocumentNodeStore</a>
+ <li> <a href="../query/indexing.html" title="Indexing">Indexing</a>
</li>
-
- <li> <a href="../nodestore/overview.html" title="Node Storage">Node Storage</a>
+ <li> <a href="../query/lucene.html" title="Lucene Index">Lucene Index</a>
</li>
-
- <li> <a href="../nodestore/persistent-cache.html" title="Persistent Cache">Persistent Cache</a>
+ <li> <a href="../query/property-index.html" title="Property Index">Property Index</a>
</li>
-
- <li> <a href="../query/query.html" title="Query">Query</a>
+ <li> <a href="../query/solr.html" title="Solr Index">Solr Index</a>
</li>
+ </ul>
+ </li>
<li> <a href="../security/overview.html" title="Security">Security</a>
</li>
- <li> <a href="../nodestore/segment/overview.html" title="Segment Node Store">Segment Node Store</a>
+ <li> <a href="../features/atomic-counter.html" title="Atomic Counter">Atomic Counter</a>
+</li>
+
+ <li> <a href="../features/observation.html" title="Observation">Observation</a>
</li>
</ul>
</li>
@@ -216,7 +230,7 @@
<ul class="breadcrumb">
- <li id="publishDate">Last Published: 2017-03-21</li>
+ <li id="publishDate">Last Published: 2017-03-24</li>
<li class="divider">|</li> <li id="projectVersion">Version: 1.8-SNAPSHOT</li>
@@ -254,6 +268,13 @@
<i class="none"></i>
Downloads</a>
</li>
+
+ <li>
+
+ <a href="../articles.html" title="Articles">
+ <i class="none"></i>
+ Articles</a>
+ </li>
<li class="nav-header">Concepts and Architecture</li>
<li>
@@ -285,55 +306,108 @@
Oak API</a>
</li>
<li class="nav-header">Features and Plugins</li>
-
+
<li>
- <a href="../features/atomic-counter.html" title="Atomic Counter">
+ <a href="../nodestore/overview.html" title="Node Storage">
+ <i class="icon-chevron-down"></i>
+ Node Storage</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../nodestore/documentmk.html" title="Document NodeStore">
+ <i class="icon-chevron-down"></i>
+ Document NodeStore</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../nodestore/document/node-bundling.html" title="Node Bundling">
<i class="none"></i>
- Atomic Counter</a>
+ Node Bundling</a>
</li>
-
+
<li>
- <a href="../plugins/blobstore.html" title="Blob Storage">
+ <a href="../nodestore/document/secondary-store.html" title="Secondary Store">
<i class="none"></i>
- Blob Storage</a>
+ Secondary Store</a>
</li>
-
+
+ <li>
+
+ <a href="../nodestore/persistent-cache.html" title="Persistent Cache">
+ <i class="none"></i>
+ Persistent Cache</a>
+ </li>
+
<li>
<a href="../clustering.html" title="Clustering">
<i class="none"></i>
Clustering</a>
</li>
-
+ </ul>
+ </li>
+
<li>
- <a href="../nodestore/documentmk.html" title="DocumentNodeStore">
+ <a href="../nodestore/segment/overview.html" title="Segment NodeStore">
<i class="none"></i>
- DocumentNodeStore</a>
+ Segment NodeStore</a>
</li>
+ </ul>
+ </li>
<li>
- <a href="../nodestore/overview.html" title="Node Storage">
+ <a href="../plugins/blobstore.html" title="Blob Storage">
<i class="none"></i>
- Node Storage</a>
+ Blob Storage</a>
</li>
-
+
<li>
- <a href="../nodestore/persistent-cache.html" title="Persistent Cache">
+ <a href="../query/query.html" title="Query">
+ <i class="icon-chevron-down"></i>
+ Query</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../query/query-engine.html" title="Query Engine">
<i class="none"></i>
- Persistent Cache</a>
+ Query Engine</a>
</li>
-
+
<li>
- <a href="../query/query.html" title="Query">
+ <a href="../query/indexing.html" title="Indexing">
<i class="none"></i>
- Query</a>
+ Indexing</a>
+ </li>
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>Lucene Index</a>
+ </li>
+
+ <li>
+
+ <a href="../query/property-index.html" title="Property Index">
+ <i class="none"></i>
+ Property Index</a>
</li>
+
+ <li>
+
+ <a href="../query/solr.html" title="Solr Index">
+ <i class="none"></i>
+ Solr Index</a>
+ </li>
+ </ul>
+ </li>
<li>
@@ -344,9 +418,16 @@
<li>
- <a href="../nodestore/segment/overview.html" title="Segment Node Store">
+ <a href="../features/atomic-counter.html" title="Atomic Counter">
<i class="none"></i>
- Segment Node Store</a>
+ Atomic Counter</a>
+ </li>
+
+ <li>
+
+ <a href="../features/observation.html" title="Observation">
+ <i class="none"></i>
+ Observation</a>
</li>
<li class="nav-header">Using Oak</li>
@@ -558,8 +639,12 @@
<li><a href="#boost">Boost and Search Relevancy</a></li>
<li><a href="#stored-index-definition">Effective Index Definition</a></li>
+
+<li><a href="#generate-index-definition">Generating Index Definition</a></li>
</ul></li>
+<li><a href="#nrt-indexing">Near Real Time Indexing</a></li>
+
<li><a href="#osgi-config">LuceneIndexProvider Configuration</a></li>
<li><a href="#tika-config">Tika Config</a>
@@ -684,6 +769,8 @@
<ul>
+<li><a href="#nrt-indexing">Near Real Time Indexing</a></li>
+
<li><a href="#stored-index-definition">Effective Index Definition</a></li>
</ul></div>
<div class="section">
@@ -1283,7 +1370,15 @@ WHERE
<p>To simplify troubleshooting the stored index definition can be accessed from <tt>LuceneIndexMBean</tt> via <tt>getStoredIndexDefinition</tt> operation. It would dump the string representation of stored NodeState</p>
<p><img src="lucene-index-mbean-dump-index.png" alt="Dump Stored Index Definition" /></p>
<p>This feature can be disabled by setting OSGi property <tt>disableStoredIndexDefinition</tt> for <tt>LuceneIndexProviderService</tt> to true. Once disable any change in index definition would start effecting the query plans</p>
-<p>Refer to <a class="externalLink" href="https://issues.apache.org/jira/browse/OAK-4400">OAK-4400</a> for more details.</p></div></div>
+<p>Refer to <a class="externalLink" href="https://issues.apache.org/jira/browse/OAK-4400">OAK-4400</a> for more details.</p></div>
+<div class="section">
+<h4><a name="generate-index-definition"></a> Generating Index Definition<a name="Generating_Index_Definition"></a></h4>
+<p>To simplify generating index definition suitable for evaluating certain set of queries you can make use of <a class="externalLink" href="http://oakutils.appspot.com/generate/index">http://oakutils.appspot.com/generate/index</a>. Here you can provide a set of queries and then it would generate the suitable index definitions for those queries.</p>
+<p>Note that you would still need to tweak the definition for aggregation, path include exclude etc as that data cannot be inferred from the query</p></div></div>
+<div class="section">
+<h3><a name="nrt-indexing"></a> Near Real Time Indexing<a name="Near_Real_Time_Indexing"></a></h3>
+<p><tt>@since Oak 1.6</tt></p>
+<p>Refer to <a href="indexing.html#nrt-indexing">Near realtime indexing</a> for more details</p></div>
<div class="section">
<h3><a name="osgi-config"></a>LuceneIndexProvider Configuration<a name="LuceneIndexProvider_Configuration"></a></h3>
<p>Some of the runtime aspects of the Oak Lucene support can be configured via OSGi configuration. The configuration needs to be done for PID <tt>org.apache
@@ -1643,13 +1738,14 @@ SELECT rep:suggest() FROM [nt:base] WHER
<li>
<p>Lucene index is Asynchronous - Lucene indexing is done asynchronously with a default interval of 5 secs. If there are lots of writes and those writes are related to what is being indexed then it might cause further delay. Compared to this the property index are always synchronous and upto date.</p>
-<p>So if in your usecase you need the latest result then prefer <i>Property Indexes</i> over <i>Lucene Index</i></p></li>
+<p>So if in your usecase you need the latest result then prefer <i>Property Indexes</i> over <i>Lucene Index</i>. Oak 1.6 supports <a href="indexing.html#nrt-indexing">Near Realtime Indexing</a> which reduce the lag considerably. With this you should be able to use lucene indexing for most cases</p></li>
<li>
<p>Lucene index cannot enforce uniqueness constraint - By virtue of it being asynchronous it cannot enforce uniqueness constraint.</p></li>
</ol></div>
<div class="section">
<h3><a name="examples"></a>Examples<a name="Examples"></a></h3>
+<p>Have a look at <a href="#generate-index-definition">generating index definition</a> for some tooling details which simplify generating index definition for given set of queries</p>
<div class="section">
<h4><a name="simple-queries"></a>A - Simple queries<a name="A_-_Simple_queries"></a></h4>
<p>In many cases the query is purely based on some specific property and is not restricted to any specific nodeType</p>
Modified: jackrabbit/site/live/oak/docs/query/ootb-index-change.html
URL: http://svn.apache.org/viewvc/jackrabbit/site/live/oak/docs/query/ootb-index-change.html?rev=1788363&r1=1788362&r2=1788363&view=diff
==============================================================================
--- jackrabbit/site/live/oak/docs/query/ootb-index-change.html (original)
+++ jackrabbit/site/live/oak/docs/query/ootb-index-change.html Fri Mar 24 05:58:10 2017
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2017-03-21
+ | Generated by Apache Maven Doxia at 2017-03-24
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20170321" />
+ <meta name="Date-Revision-yyyymmdd" content="20170324" />
<meta http-equiv="Content-Language" content="en" />
<title>Jackrabbit Oak - Changing Out-Of-The-Box Index Definitions</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" />
@@ -65,6 +65,9 @@
<li> <a href="../downloads.html" title="Downloads">Downloads</a>
</li>
+
+ <li> <a href="../articles.html" title="Articles">Articles</a>
+</li>
</ul>
</li>
<li class="dropdown">
@@ -93,31 +96,42 @@
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Features and Plugins <b class="caret"></b></a>
<ul class="dropdown-menu">
- <li> <a href="../features/atomic-counter.html" title="Atomic Counter">Atomic Counter</a>
+ <li class="dropdown-submenu">
+ <a href="../nodestore/overview.html" title="Node Storage">Node Storage</a>
+ <ul class="dropdown-menu">
+ <li> <a href="../nodestore/documentmk.html" title="Document NodeStore">Document NodeStore</a>
</li>
+ <li> <a href="../nodestore/segment/overview.html" title="Segment NodeStore">Segment NodeStore</a>
+</li>
+ </ul>
+ </li>
<li> <a href="../plugins/blobstore.html" title="Blob Storage">Blob Storage</a>
</li>
- <li> <a href="../clustering.html" title="Clustering">Clustering</a>
+ <li class="dropdown-submenu">
+ <a href="../query/query.html" title="Query">Query</a>
+ <ul class="dropdown-menu">
+ <li> <a href="../query/query-engine.html" title="Query Engine">Query Engine</a>
</li>
-
- <li> <a href="../nodestore/documentmk.html" title="DocumentNodeStore">DocumentNodeStore</a>
+ <li> <a href="../query/indexing.html" title="Indexing">Indexing</a>
</li>
-
- <li> <a href="../nodestore/overview.html" title="Node Storage">Node Storage</a>
+ <li> <a href="../query/lucene.html" title="Lucene Index">Lucene Index</a>
</li>
-
- <li> <a href="../nodestore/persistent-cache.html" title="Persistent Cache">Persistent Cache</a>
+ <li> <a href="../query/property-index.html" title="Property Index">Property Index</a>
</li>
-
- <li> <a href="../query/query.html" title="Query">Query</a>
+ <li> <a href="../query/solr.html" title="Solr Index">Solr Index</a>
</li>
+ </ul>
+ </li>
<li> <a href="../security/overview.html" title="Security">Security</a>
</li>
- <li> <a href="../nodestore/segment/overview.html" title="Segment Node Store">Segment Node Store</a>
+ <li> <a href="../features/atomic-counter.html" title="Atomic Counter">Atomic Counter</a>
+</li>
+
+ <li> <a href="../features/observation.html" title="Observation">Observation</a>
</li>
</ul>
</li>
@@ -216,7 +230,7 @@
<ul class="breadcrumb">
- <li id="publishDate">Last Published: 2017-03-21</li>
+ <li id="publishDate">Last Published: 2017-03-24</li>
<li class="divider">|</li> <li id="projectVersion">Version: 1.8-SNAPSHOT</li>
@@ -254,6 +268,13 @@
<i class="none"></i>
Downloads</a>
</li>
+
+ <li>
+
+ <a href="../articles.html" title="Articles">
+ <i class="none"></i>
+ Articles</a>
+ </li>
<li class="nav-header">Concepts and Architecture</li>
<li>
@@ -285,55 +306,110 @@
Oak API</a>
</li>
<li class="nav-header">Features and Plugins</li>
-
+
<li>
- <a href="../features/atomic-counter.html" title="Atomic Counter">
+ <a href="../nodestore/overview.html" title="Node Storage">
+ <i class="icon-chevron-down"></i>
+ Node Storage</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../nodestore/documentmk.html" title="Document NodeStore">
+ <i class="icon-chevron-down"></i>
+ Document NodeStore</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../nodestore/document/node-bundling.html" title="Node Bundling">
<i class="none"></i>
- Atomic Counter</a>
+ Node Bundling</a>
</li>
-
+
<li>
- <a href="../plugins/blobstore.html" title="Blob Storage">
+ <a href="../nodestore/document/secondary-store.html" title="Secondary Store">
<i class="none"></i>
- Blob Storage</a>
+ Secondary Store</a>
</li>
-
+
+ <li>
+
+ <a href="../nodestore/persistent-cache.html" title="Persistent Cache">
+ <i class="none"></i>
+ Persistent Cache</a>
+ </li>
+
<li>
<a href="../clustering.html" title="Clustering">
<i class="none"></i>
Clustering</a>
</li>
-
+ </ul>
+ </li>
+
<li>
- <a href="../nodestore/documentmk.html" title="DocumentNodeStore">
+ <a href="../nodestore/segment/overview.html" title="Segment NodeStore">
<i class="none"></i>
- DocumentNodeStore</a>
+ Segment NodeStore</a>
</li>
+ </ul>
+ </li>
<li>
- <a href="../nodestore/overview.html" title="Node Storage">
+ <a href="../plugins/blobstore.html" title="Blob Storage">
<i class="none"></i>
- Node Storage</a>
+ Blob Storage</a>
</li>
-
+
<li>
- <a href="../nodestore/persistent-cache.html" title="Persistent Cache">
+ <a href="../query/query.html" title="Query">
+ <i class="icon-chevron-down"></i>
+ Query</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../query/query-engine.html" title="Query Engine">
<i class="none"></i>
- Persistent Cache</a>
+ Query Engine</a>
</li>
-
+
<li>
- <a href="../query/query.html" title="Query">
+ <a href="../query/indexing.html" title="Indexing">
<i class="none"></i>
- Query</a>
+ Indexing</a>
+ </li>
+
+ <li>
+
+ <a href="../query/lucene.html" title="Lucene Index">
+ <i class="none"></i>
+ Lucene Index</a>
+ </li>
+
+ <li>
+
+ <a href="../query/property-index.html" title="Property Index">
+ <i class="none"></i>
+ Property Index</a>
+ </li>
+
+ <li>
+
+ <a href="../query/solr.html" title="Solr Index">
+ <i class="none"></i>
+ Solr Index</a>
</li>
+ </ul>
+ </li>
<li>
@@ -344,9 +420,16 @@
<li>
- <a href="../nodestore/segment/overview.html" title="Segment Node Store">
+ <a href="../features/atomic-counter.html" title="Atomic Counter">
+ <i class="none"></i>
+ Atomic Counter</a>
+ </li>
+
+ <li>
+
+ <a href="../features/observation.html" title="Observation">
<i class="none"></i>
- Segment Node Store</a>
+ Observation</a>
</li>
<li class="nav-header">Using Oak</li>