You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/08/13 08:35:42 UTC
svn commit: r1617667 - in /phoenix: phoenix-docs/src/main/org/h2/
phoenix-docs/src/main/org/h2/jdbc/ phoenix-docs/src/main/org/h2/jdbcx/
phoenix-docs/src/main/org/h2/tools/ site/publish/
site/source/src/site/markdown/
Author: jamestaylor
Date: Wed Aug 13 06:35:42 2014
New Revision: 1617667
URL: http://svn.apache.org/r1617667
Log:
Update secondary index docs
Modified:
phoenix/phoenix-docs/src/main/org/h2/Driver.java
phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcCallableStatement.java
phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcConnection.java
phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcDatabaseMetaData.java
phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcResultSet.java
phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcStatement.java
phoenix/phoenix-docs/src/main/org/h2/jdbcx/JdbcConnectionPool.java
phoenix/phoenix-docs/src/main/org/h2/jdbcx/JdbcDataSource.java
phoenix/phoenix-docs/src/main/org/h2/tools/SimpleResultSet.java
phoenix/site/publish/secondary_indexing.html
phoenix/site/source/src/site/markdown/secondary_indexing.md
Modified: phoenix/phoenix-docs/src/main/org/h2/Driver.java
URL: http://svn.apache.org/viewvc/phoenix/phoenix-docs/src/main/org/h2/Driver.java?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/phoenix-docs/src/main/org/h2/Driver.java (original)
+++ phoenix/phoenix-docs/src/main/org/h2/Driver.java Wed Aug 13 06:35:42 2014
@@ -17,7 +17,7 @@ import org.h2.message.DbException;
import org.h2.message.TraceSystem;
import org.h2.upgrade.DbUpgrade;
-//## Java 1.7 ##
+/*## Java 1.7 ##
import java.util.logging.Logger;
//*/
@@ -138,7 +138,7 @@ public class Driver implements java.sql.
/**
* [Not supported]
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public Logger getParentLogger() {
return null;
}
Modified: phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcCallableStatement.java
URL: http://svn.apache.org/viewvc/phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcCallableStatement.java?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcCallableStatement.java (original)
+++ phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcCallableStatement.java Wed Aug 13 06:35:42 2014
@@ -1440,7 +1440,7 @@ public class JdbcCallableStatement exten
* @param parameterIndex the parameter index (1, 2, ...)
* @param type the class of the returned value
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public <T> T getObject(int parameterIndex, Class<T> type) {
return null;
}
@@ -1452,7 +1452,7 @@ public class JdbcCallableStatement exten
* @param parameterName the parameter name
* @param type the class of the returned value
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public <T> T getObject(String parameterName, Class<T> type) {
return null;
}
Modified: phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcConnection.java
URL: http://svn.apache.org/viewvc/phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcConnection.java?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcConnection.java (original)
+++ phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcConnection.java Wed Aug 13 06:35:42 2014
@@ -49,7 +49,7 @@ import java.sql.SQLXML;
import java.sql.SQLClientInfoException;
//*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
import java.util.concurrent.Executor;
//*/
@@ -1678,7 +1678,7 @@ public class JdbcConnection extends Trac
*
* @param schema the schema
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public void setSchema(String schema) {
// not supported
}
@@ -1687,7 +1687,7 @@ public class JdbcConnection extends Trac
/**
* [Not supported]
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public String getSchema() {
return null;
}
@@ -1698,7 +1698,7 @@ public class JdbcConnection extends Trac
*
* @param executor the executor used by this method
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public void abort(Executor executor) {
// not supported
}
@@ -1710,7 +1710,7 @@ public class JdbcConnection extends Trac
* @param executor the executor used by this method
* @param milliseconds the TCP connection timeout
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public void setNetworkTimeout(Executor executor, int milliseconds) {
// not supported
}
@@ -1719,7 +1719,7 @@ public class JdbcConnection extends Trac
/**
* [Not supported]
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public int getNetworkTimeout() {
return 0;
}
Modified: phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcDatabaseMetaData.java
URL: http://svn.apache.org/viewvc/phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcDatabaseMetaData.java?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcDatabaseMetaData.java (original)
+++ phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcDatabaseMetaData.java Wed Aug 13 06:35:42 2014
@@ -2885,7 +2885,7 @@ public class JdbcDatabaseMetaData extend
/**
* [Not supported]
*/
- //## Java 1.7 ##
+ /*## Java 1.7 ##
public boolean generatedKeyAlwaysReturned() {
return true;
}
@@ -2902,7 +2902,7 @@ public class JdbcDatabaseMetaData extend
* @param columnNamePattern null (to get all objects) or a column name
* (uppercase for unquoted names)
*/
- //## Java 1.7 ##
+ /*## Java 1.7 ##
public ResultSet getPseudoColumns(String catalog, String schemaPattern,
String tableNamePattern, String columnNamePattern) {
return null;
Modified: phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcResultSet.java
URL: http://svn.apache.org/viewvc/phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcResultSet.java?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcResultSet.java (original)
+++ phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcResultSet.java Wed Aug 13 06:35:42 2014
@@ -3431,7 +3431,7 @@ public class JdbcResultSet extends Trace
* @param columnIndex the column index (1, 2, ...)
* @param type the class of the returned value
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public <T> T getObject(int columnIndex, Class<T> type) {
return null;
}
@@ -3443,7 +3443,7 @@ public class JdbcResultSet extends Trace
* @param columnName the column name
* @param type the class of the returned value
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public <T> T getObject(String columnName, Class<T> type) {
return null;
}
Modified: phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcStatement.java
URL: http://svn.apache.org/viewvc/phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcStatement.java?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcStatement.java (original)
+++ phoenix/phoenix-docs/src/main/org/h2/jdbc/JdbcStatement.java Wed Aug 13 06:35:42 2014
@@ -876,7 +876,7 @@ public class JdbcStatement extends Trace
/**
* [Not supported]
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public void closeOnCompletion() {
// not supported
}
@@ -885,7 +885,7 @@ public class JdbcStatement extends Trace
/**
* [Not supported]
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public boolean isCloseOnCompletion() {
return true;
}
Modified: phoenix/phoenix-docs/src/main/org/h2/jdbcx/JdbcConnectionPool.java
URL: http://svn.apache.org/viewvc/phoenix/phoenix-docs/src/main/org/h2/jdbcx/JdbcConnectionPool.java?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/phoenix-docs/src/main/org/h2/jdbcx/JdbcConnectionPool.java (original)
+++ phoenix/phoenix-docs/src/main/org/h2/jdbcx/JdbcConnectionPool.java Wed Aug 13 06:35:42 2014
@@ -35,7 +35,7 @@ import org.h2.util.New;
import org.h2.message.DbException;
//*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
import java.util.logging.Logger;
//*/
@@ -330,7 +330,7 @@ public class JdbcConnectionPool implemen
/**
* [Not supported]
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public Logger getParentLogger() {
return null;
}
Modified: phoenix/phoenix-docs/src/main/org/h2/jdbcx/JdbcDataSource.java
URL: http://svn.apache.org/viewvc/phoenix/phoenix-docs/src/main/org/h2/jdbcx/JdbcDataSource.java?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/phoenix-docs/src/main/org/h2/jdbcx/JdbcDataSource.java (original)
+++ phoenix/phoenix-docs/src/main/org/h2/jdbcx/JdbcDataSource.java Wed Aug 13 06:35:42 2014
@@ -26,7 +26,7 @@ import org.h2.jdbc.JdbcConnection;
import org.h2.message.TraceObject;
import org.h2.util.StringUtils;
-//## Java 1.7 ##
+/*## Java 1.7 ##
import java.util.logging.Logger;
//*/
@@ -381,7 +381,7 @@ public class JdbcDataSource extends Trac
/**
* [Not supported]
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public Logger getParentLogger() {
return null;
}
Modified: phoenix/phoenix-docs/src/main/org/h2/tools/SimpleResultSet.java
URL: http://svn.apache.org/viewvc/phoenix/phoenix-docs/src/main/org/h2/tools/SimpleResultSet.java?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/phoenix-docs/src/main/org/h2/tools/SimpleResultSet.java (original)
+++ phoenix/phoenix-docs/src/main/org/h2/tools/SimpleResultSet.java Wed Aug 13 06:35:42 2014
@@ -853,7 +853,7 @@ public class SimpleResultSet implements
* @param columnIndex the column index (1, 2, ...)
* @param type the class of the returned value
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public <T> T getObject(int columnIndex, Class<T> type) {
return null;
}
@@ -865,7 +865,7 @@ public class SimpleResultSet implements
* @param columnName the column name
* @param type the class of the returned value
*/
-//## Java 1.7 ##
+/*## Java 1.7 ##
public <T> T getObject(String columnName, Class<T> type) {
return null;
}
Modified: phoenix/site/publish/secondary_indexing.html
URL: http://svn.apache.org/viewvc/phoenix/site/publish/secondary_indexing.html?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/site/publish/secondary_indexing.html (original)
+++ phoenix/site/publish/secondary_indexing.html Wed Aug 13 06:35:42 2014
@@ -1,7 +1,7 @@
<!DOCTYPE html>
<!--
- Generated by Apache Maven Doxia at 2014-08-05
+ Generated by Apache Maven Doxia at 2014-08-12
Rendered using Reflow Maven Skin 1.1.0 (http://andriusvelykis.github.io/reflow-maven-skin)
-->
<html xml:lang="en" lang="en">
@@ -124,24 +124,43 @@
<h1>Secondary Indexing</h1>
</div>
<p>Secondary indexes are an orthogonal way to access data from its primary access path. In HBase, you have a single index that is lexicographically sorted on the primary row key. Access to records in any way other than through the primary row requires scanning over potentially all the rows in the table to test them against your filter. With secondary indexing, the columns you index form an alternate row key to allow point lookups and range scans along this new axis. Phoenix is particularly powerful in that we provide <i>covered</i> indexes - we do not need to go back to the primary table once we have found the index entry. Instead, we bundle the data we care about right in the index rows, saving read-time overhead.</p>
-<p>Phoenix supports two main forms of indexing: mutable and immutable indexing. They are useful in different scenarios and have their own failure profiles and performance characteristics. Both indexes are âglobalâ indexes - they live on their own tables and are copies of primary table data, which Phoenix ensures remain in-sync.</p>
-<h1>Mutable Indexing</h1>
-<p>Often, the rows you are inserting are changing - pretty much any time you are not doing time-series data. In this case, use mutable indexing to ensure that your index is properly maintained as your data changes.</p>
-<p>All the performance penalties for indexes occur at write time. We intercept the primary table updates on write (<a href="language/index.html#delete">DELETE</a>, <a href="language/index.html#upsert_values">UPSERT VALUES</a> and <a href="language/index.html#upsert_select">UPSERT SELECT</a>), build the index update and then sent any necessary updates to all interested index tables. At read time, Phoenix will select the index table to use that will produce the fastest query time and directly scan it just like any other HBase table.</p>
+<p>Phoenix supports three types of indexing techniques: immutable, global, and local indexing. They are each useful in different scenarios and have their own failure profiles and performance characteristics.</p>
<div class="section">
- <h2 id="Example">Example</h2>
+ <h2 id="Immutable_Indexing">Immutable Indexing</h2>
+ <p>Immutable indexing targets use cases that are <i>write once</i>, <i>append only</i>; this is common in time-series data, where you log once, but read multiple times. In this case, the indexing is managed entirely on the client - either we successfully write all the primary and index data or we return a failure to the client. Since once written, rows are never updated, no incremental index maintenance is required making them perform very well. This reduces the overhead of secondary indexing at write time. However, keep in mind that immutable indexing are only applicable in a limited set of use cases.</p>
+ <p>One restriction of immutable indexes is that rows from the data table may not be deleted. Instead, the only way to delete rows is to drop the entire data table. This is likely to change once <a class="externalLink" href="https://issues.apache.org/jira/browse/PHOENIX-619">PHOENIX-619</a> is implemented.</p>
+</div>
+<div class="section">
+ <h2 id="Mutable_Indexing">Mutable Indexing</h2>
+ <p>If a column values of the same row changes, you must use mutable indexing to ensure that your index is properly maintained. There are two types of mutable indexes: global and local. From a functional standpoint, these are nearly identical with the difference being in their performance characteristics and tradeoffs.</p>
+ <div class="section">
+ <h3 id="Global_Indexing">Global Indexing</h3>
+ <p>Global indexing targets <i>read heavy</i>, <i>low write</i> uses cases. With global indexes, all the performance penalties for indexes occur at write time. We intercept the data table updates on write (<a href="language/index.html#delete">DELETE</a>, <a href="language/index.html#upsert_values">UPSERT VALUES</a> and <a href="language/index.html#upsert_select">UPSERT SELECT</a>), build the index update and then sent any necessary updates to all interested index tables. At read time, Phoenix will select the index table to use that will produce the fastest query time and directly scan it just like any other HBase table. Note, however, if a column is referenced in a query that isnât part of the index, the index will not be used for that query.</p>
+ </div>
+ <div class="section">
+ <h3 id="Local_Indexing">Local Indexing</h3>
+ <p>Local indexing targets <i>write heavy</i>, <i>space constrained</i> use cases. With local indexes index data and table data are co-reside at same server so no network overhead during writes and reads. Local indexes can be used even when the query isnât fully covered i.e. Phoenix automatically retrieve the columns not in the index through point gets against the data table. Unlike global indexes all local indexes data of a table are stored in a separate shared table. At read time when the local index is used, every region must be examined for the data as the exact region location of index data cannot be predetermined which incurs some overhead.</p>
+ </div>
+</div>
+<div class="section">
+ <h2 id="Examples">Examples</h2>
<p>Given the schema shown here:</p>
<div class="source">
<pre>CREATE TABLE my_table (k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 BIGINT);
</pre>
</div>
- <p>youâd create an index on the v1 column like this:</p>
+ <p>youâd create a global index on the v1 column like this:</p>
<div class="source">
<pre>CREATE INDEX my_index ON my_table (v1);
</pre>
</div>
+ <p>while the equivalent local index would be created like this:</p>
+ <div class="source">
+ <pre>CREATE LOCAL INDEX my_index ON my_table (v1);
+</pre>
+ </div>
<p>A table may contain any number of indexes, but note that your write speed will drop as you add additional indexes.</p>
- <p>We can also include columns from the data table in the index apart from the indexed columns. This allows an index to be used more frequently, as it will only be used if all columns referenced in the query are contained by it.</p>
+ <p>We can also include columns from the data table in the index apart from the indexed columns. This allows a global index to be used more frequently, as a global index will only be used if all columns referenced in the query are contained by it. This is not the case for local indexes.</p>
<div class="source">
<pre>CREATE INDEX my_index ON my_table (v1) INCLUDE (v2);
</pre>
@@ -157,52 +176,21 @@
SALT_BUCKETS=10, DATA_BLOCK_ENCODING='NONE';
</pre>
</div>
- <p>Note that if the primary table is salted, then the index is automatically salted in the same way. In addition, the MAX_FILESIZE for the index is adjusted down, relative to the size of the primary versus index table. For more on salting see <a href="salted.html">here</a>.</p>
- <h1>Immutable Indexing</h1>
- <p>Immutable indexing targets use cases that are <i>write once</i>, <i>append only</i>; this is common in time-series data, where you log once, but read multiple times. In this case, the indexing is managed entirely on the client - either we successfully write all the primary and index data or we return a failure to the client. Since once written, rows are never updated, no incremental index maintenance is required. This reduces the overhead of secondary indexing at write time. However, keep in mind that immutable indexing are only applicable in a limited set of use cases.</p>
-</div>
-<div class="section">
- <h2 id="Example">Example</h2>
+ <p>Note that if the primary table is salted, then the index is automatically salted in the same way for global indexes. In addition, the MAX_FILESIZE for the index is adjusted down, relative to the size of the primary versus index table. For more on salting see <a href="salted.html">here</a>. With local indexes, on the other hand, specifying SALT_BUCKETS is not allowed.</p>
+ <p>To drop an index, youâd issue the following statement: DROP INDEX my_index ON my_table</p>
+ <p>If an indexed column is dropped in the data table, the index will automatically be dropped. In addition, if a covered column is dropped in the data table, it will be automatically dropped from the index as well.</p>
<p>To use immutable indexing, supply an <tt>IMMUTABLE_ROWS=true</tt> property when you create your table like this:</p>
<div class="source">
<pre>CREATE TABLE my_table (k VARCHAR PRIMARY KEY, v VARCHAR) IMMUTABLE_ROWS=true;
</pre>
</div>
- <p>Other than that, all of the previous examples are identical for immutable indexing.</p>
+ <p>In that case, all indexes on the table are immutable indexes and local indexes are not allowed.</p>
<p>If you have an existing table that youâd like to switch from immutable indexing to mutable indexing, use the <tt>ALTER TABLE</tt> command as show below:</p>
<div class="source">
<pre>ALTER TABLE my_table SET IMMUTABLE_ROWS=false;
</pre>
</div>
- <p>For the complete syntax, see our <a href="language/index.html#create_index">Language Reference Guide</a>.</p>
- <h1>Local(Region level) indexing</h1>
- <p>Local indexing targets <i>write heavy</i>, <i>low latency</i> and <i>space constraint</i> use cases. With local indexes index data and table data are co-reside at same server so no network overhead during writes and reads. Local indexes can be used even when the query isnât fully covered i.e. Phoenix automatically retrieve the columns not in the index through point gets against the data table. Unlike global indexes all local indexes data of a table are stored in a separate shared table.</p>
- <p>Reading data via the local index does however require to contact each region until unless query contains equal/range condition(s) on leading primary key column(s) of the data table.</p>
-</div>
-<div class="section">
- <h2 id="Example">Example</h2>
- <p>To use local indexing, just supply a <tt>LOCAL</tt> keyword when you create index like this:</p>
- <div class="source">
- <pre>CREATE LOCAL INDEX my_index ON my_table (v1);
-</pre>
- </div>
- <p>For the complete syntax, see our <a href="language/index.html#create_index">Language Reference Guide</a>.</p>
- <div class="section">
- <h3 id="Setup">Setup</h3>
- <p>Local indexing requires special configurations in the master to ensure data table and local index regions co-location.</p>
- <p>You will need to add the following parameters to <tt>hbase-site.xml</tt>:</p>
- <div class="source">
- <pre><property>
- <name>hbase.master.loadbalancer.class</name>
- <value>org.apache.phoenix.hbase.index.balancer.IndexLoadBalancer</value>
-</property>
-<property>
- <name>hbase.coprocessor.master.classes</name>
- <value>org.apache.phoenix.hbase.index.master.IndexMasterObserver</value>
-</property>
-</pre>
- </div>
- </div>
+ <p>For the complete syntax, see our <a href="language/index.html">Language Reference Guide</a>.</p>
</div>
<div class="section">
<h2 id="Data_Guarantees_and_Failure_Management">Data Guarantees and Failure Management</h2>
@@ -233,70 +221,40 @@
</pre>
</div>
<p>If we cannot disable the index, then the server will be immediately aborted. If the abort fails, we call System.exit on the JVM, forcing the server to die. By killing the server, we ensure that the WAL will be replayed on recovery, replaying the index updates to their appropriate tables.</p>
- <p><b>WARNING: indexing has the potential to bring down your entire cluster very quickly.</b></p>
+ <p><b>WARNING: global indexing has the potential to bring down your entire cluster very quickly.</b></p>
<p>If the index tables are not setup correctly (Phoenix ensures that they are), this failure policy can cause a cascading failure as each region server attempts and fails to write the index update, subsequently killing itself to ensure the visibility concerns outlined above.</p>
</div>
</div>
<div class="section">
<h2 id="Setup">Setup</h2>
- <p>Only mutable indexing requires special configuration options in the region server to run - phoenix ensures that they are setup correctly when you enable mutable indexing on the table; if the correct properties are not set, you will not be able to turn it on.</p>
- <p>You will need to add the following parameters to <tt>hbase-site.xml</tt>:</p>
+ <p>Mutable indexing requires special configuration options on the region server and master to run - Phoenix ensures that they are setup correctly when you enable mutable indexing on the table; if the correct properties are not set, you will not be able to use secondary indexing. After adding these settings to your hbase-site.xml, youâll need to do a rolling restart of your cluster.</p>
+ <p>You will need to add the following parameters to <tt>hbase-site.xml</tt> on each region server:</p>
<div class="source">
<pre><property>
<name>hbase.regionserver.wal.codec</name>
<value>org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec</value>
</property>
-</pre>
- </div>
- <p>This enables custom WAL edits to be written, ensuring proper writing/replay of the index updates. This codec supports the usual host of WALEdit options, most notably WALEdit compression.</p>
- <div class="section">
- <h3 id="Advanced_Setup_-_Removing_Index_Deadlocks_0.98.4">Advanced Setup - Removing Index Deadlocks (0.98.4+)</h3>
- <p>Phoenix releases that include these changes (4.1+, 5.0.0+) are still backwards compatible with older versions of phoenix (to the extent that they are semantically compatible) as well as with older versions of HBase (0.98.1-0.98.3).</p>
- <p>As of HBase 0.98.4 we can finally remove the change of index deadlocks. In HBase you can tune the number of RPC threads to match client writes + index writes, but there is still a chance you could have a deadlock in an unlucky scenario (i.e. Client A -> Server A, Client B -> Server B, each taking the last RPC thread. Then each server attempts to make an index update to the other, Server A -> Server B, and vice versa, but they canât as there are no more available RPC threads).</p>
- <p>As of <a class="externalLink" href="https://issues.apache.org/jira/browse/PHOENIX-938">PHOENIX-938</a> and <a class="externalLink" href="https://issues.apache.org/jira/browse/HBASE-11513">HBASE-11513</a> we can remove these deadlocks by providing a different set of RPC handlers for index updates by giving index updates their own ârpc priorityâ and handling the priorities via a custom Phoenix RPC Handler.</p>
- <p>The properties you need to set to enable this are</p>
- <div class="section">
- <h4 id="Server_Side">Server Side</h4>
- <div class="source">
- <pre><property>
+<property>
<name>hbase.region.server.rpc.scheduler.factory.class</name>
<value>org.apache.phoenix.hbase.index.ipc.PhoenixIndexRpcSchedulerFactory</value>
<description>Factory to create the Phoenix RPC Scheduler that knows to put index updates into index queues</description>
</property>
</pre>
- </div>
- <p>After adding these settings to your hbase-site.xml, you just need to do a rolling restart of your cluster.</p>
- <p>Note that having the configs on both client and server side will not impact correctness or performance.</p>
- </div>
- <div class="section">
- <h4 id="Tuning">Tuning</h4>
- <p>By default, index priority range is between (1000, 1050]. Higher priorites within the index range, at this time, do not means updates are processed sooner. However, we reserve this range to provide that possibility in the future.</p>
- <p>You can specifiy this range however to suit your individual cluster requirements by adjusting the follwing parameters</p>
- <div class="source">
- <pre><property>
- <name>org.apache.phoenix.regionserver.index.priority.min</name>
- <value>1050</value>
- <description>Value to specify to bottom (inclusive) of the range in which index priority may lie</description>
+ </div>
+ <p>The first property enables custom WAL edits to be written, ensuring proper writing/replay of the index updates. This codec supports the usual host of WALEdit options, most notably WALEdit compression.</p>
+ <p>The second property prevents deadlocks from occurring during index maintenance for global indexes (HBase 0.98.4+ only) by ensuring index updates are processed with a higher priority than data updates.</p>
+ <p>Local indexing also requires special configurations in the master to ensure data table and local index regions co-location.</p>
+ <p>You will need to add the following parameters to <tt>hbase-site.xml</tt> on the master:</p>
+ <div class="source">
+ <pre><property>
+ <name>hbase.master.loadbalancer.class</name>
+ <value>org.apache.phoenix.hbase.index.balancer.IndexLoadBalancer</value>
</property>
<property>
- <name>org.apache.phoenix.regionserver.index.priority.max</name>
- <value>1050</value>
- <description>Value to specify to top (exclusive) of the range in which index priority may lie</description>
-</property>
-</pre>
- </div>
- <p>The number of RPC Handler Threads can be specified via:</p>
- <div class="source">
- <pre><property>
- <name>org.apache.phoenix.regionserver.index.handler.count</name>
- <value>30</value>
- <description>Number of threads to use when serving index write requests</description>
+ <name>hbase.coprocessor.master.classes</name>
+ <value>org.apache.phoenix.hbase.index.master.IndexMasterObserver</value>
</property>
</pre>
- </div>
- <p>Though the actual number of threads is dictated by the Max(number of call queues, handler count), where the number of call queues is determined by standard HBase configuration (see below).</p>
- <p>To further tune the queues, you can adjust the standard rpc queue length parameters (currently, there are no special knobs for the index queues), specifically âipc.server.max.callqueue.lengthâ and âipc.server.callqueue.handler.factorâ. See the <a class="externalLink" href="http://hbase.apache.org/book.html">HBase Reference Guide</a> for more details.</p>
- </div>
</div>
</div>
<div class="section">
@@ -304,57 +262,76 @@
<p>Out the box, indexing is pretty fast. However, to optimize for your particular environment and workload, there are several properties you can tune.</p>
<p>All the following parameters must be set in <tt>hbase-site.xml</tt> - they are true for the entire cluster and all index tables, as well as across all regions on the same server (so, for instance, a single server would not write to too many different index tables at once).</p>
<ol style="list-style-type: decimal">
- <li> <p>index.builder.threads.max</p>
+ <li>index.builder.threads.max
<ul>
<li>Number of threads to used to build the index update from the primary table update</li>
<li>Increasing this value overcomes the bottleneck of reading the current row state from the underlying HRegion. Tuning this value too high will just bottleneck at the HRegion as it will not be able to handle too many concurrent scan requests as well as general thread-swapping concerns.</li>
<li><b>Default: 10</b></li>
</ul></li>
- <li> <p>index.builder.threads.keepalivetime</p>
+ <li>index.builder.threads.keepalivetime
<ul>
<li>Amount of time in seconds after we expire threads in the builder thread pool.</li>
<li>Unused threads are immediately released after this amount of time and not core threads are retained (though this last is a small concern as tables are expected to sustain a fairly constant write load), but simultaneously allows us to drop threads if we are not seeing the expected load.</li>
<li><b>Default: 60</b></li>
</ul></li>
- <li> <p>index.writer.threads.max</p>
+ <li>index.writer.threads.max
<ul>
<li>Number of threads to use when writing to the target index tables.</li>
<li>The first level of parallelization, on a per-table basis - it should roughly correspond to the number of index tables</li>
<li><b>Default: 10</b></li>
</ul></li>
- <li> <p>index.writer.threads.keepalivetime</p>
+ <li>index.writer.threads.keepalivetime
<ul>
<li>Amount of time in seconds after we expire threads in the writer thread pool.</li>
<li>Unused threads are immediately released after this amount of time and not core threads are retained (though this last is a small concern as tables are expected to sustain a fairly constant write load), but simultaneously allows us to drop threads if we are not seeing the expected load.</li>
<li><b>Default: 60</b></li>
</ul></li>
- <li> <p>hbase.htable.threads.max</p>
+ <li>hbase.htable.threads.max
<ul>
<li>Number of threads each index HTable can use for writes.</li>
<li>Increasing this allows more concurrent index updates (for instance across batches), leading to high overall throughput.</li>
<li><b>Default: 2,147,483,647</b></li>
</ul></li>
- <li> <p>hbase.htable.threads.keepalivetime</p>
+ <li>hbase.htable.threads.keepalivetime
<ul>
<li>Amount of time in seconds after we expire threads in the HTableâs thread pool.</li>
<li>Using the âdirect handoffâ approach, new threads will only be created if it is necessary and will grow unbounded. This could be bad but HTables only create as many Runnables as there are region servers; therefore, it also scales when new region servers are added.</li>
<li><b>Default: 60</b></li>
</ul></li>
- <li> <p>index.tablefactory.cache.size</p>
+ <li>index.tablefactory.cache.size
<ul>
<li>Number of index HTables we should keep in cache.</li>
<li>Increasing this number ensures that we do not need to recreate an HTable for each attempt to write to an index table. Conversely, you could see memory pressure if this value is set too high.</li>
<li><b>Default: 10</b></li>
</ul></li>
+ <li>org.apache.phoenix.regionserver.index.priority.min
+ <ul>
+ <li>Value to specify to bottom (inclusive) of the range in which index priority may lie.</li>
+ <li><b>Default: 1000</b></li>
+ </ul></li>
+ <li>org.apache.phoenix.regionserver.index.priority.max
+ <ul>
+ <li>Value to specify to top (exclusive) of the range in which index priority may lie.</li>
+ <li>Higher priorites within the index min/max range do not means updates are processed sooner.</li>
+ <li><b>Default: 1050</b></li>
+ </ul></li>
+ <li>org.apache.phoenix.regionserver.index.handler.count
+ <ul>
+ <li>Number of threads to use when serving index write requests for global index maintenance.</li>
+ <li>Though the actual number of threads is dictated by the Max(number of call queues, handler count), where the number of call queues is determined by standard HBase configuration. To further tune the queues, you can adjust the standard rpc queue length parameters (currently, there are no special knobs for the index queues), specifically <tt>ipc.server.max.callqueue.length</tt> and <tt>ipc.server.callqueue.handler.factor</tt>. See the <a class="externalLink" href="http://hbase.apache.org/book.html">HBase Reference Guide</a> for more details.</li>
+ <li><b>Default: 30</b></li>
+ </ul></li>
</ol>
<h1>Performance</h1>
<p>We track secondary index performance via our <a class="externalLink" href="http://phoenix-bin.github.io/client/performance/latest.htm">performance framework</a>. This is a generic test of performance based on defaults - your results will vary based on hardware specs as well as you individual configuration.</p>
<p>That said, we have seen secondary indexing (both immutable and mutable) go as quickly as < 2x the regular write path on a small, (3 node) desktop-based cluster. This is actually a phenomenal as we have to write to multiple tables as well as build the index update.</p>
- <h1>Presentations</h1>
+ <h1>Resources</h1>
<p>There have been several presentations given on how secondary indexing works in Phoenix that have a more in-depth look at how indexing works (with pretty pictures!):</p>
<ul>
<li><a class="externalLink" href="http://files.meetup.com/1350427/PhoenixIndexing-SF-HUG_09-26-13.pptx">San Francisco HBase Meetup</a> - Sept. 26, 2013</li>
<li><a class="externalLink" href="http://www.slideshare.net/jesse_yates/phoenix-secondary-indexing-la-hug-sept-9th-2013">Los Anglees HBase Meetup</a> - Sept, 4th, 2013</li>
+ <li><a class="externalLink" href="https://github.com/Huawei-Hadoop/hindex/blob/master/README.md#how-it-works">Local Indexes</a> by Huawei</li>
+ <li><a class="externalLink" href="https://issues.apache.org/jira/browse/PHOENIX-938">PHOENIX-938</a> and <a class="externalLink" href="https://issues.apache.org/jira/browse/HBASE-11513">HBASE-11513</a> for deadlock prevention during global index maintenance.</li>
</ul>
</div>
</div>
Modified: phoenix/site/source/src/site/markdown/secondary_indexing.md
URL: http://svn.apache.org/viewvc/phoenix/site/source/src/site/markdown/secondary_indexing.md?rev=1617667&r1=1617666&r2=1617667&view=diff
==============================================================================
--- phoenix/site/source/src/site/markdown/secondary_indexing.md (original)
+++ phoenix/site/source/src/site/markdown/secondary_indexing.md Wed Aug 13 06:35:42 2014
@@ -3,25 +3,39 @@
Secondary indexes are an orthogonal way to access data from its primary access path. In HBase, you have a single index that is lexicographically sorted on
the primary row key. Access to records in any way other than through the primary row requires scanning over potentially all the rows in the table to test them against your filter. With secondary indexing, the columns you index form an alternate row key to allow point lookups and range scans along this new axis. Phoenix is particularly powerful in that we provide _covered_ indexes - we do not need to go back to the primary table once we have found the index entry. Instead, we bundle the data we care about right in the index rows, saving read-time overhead.
-Phoenix supports two main forms of indexing: mutable and immutable indexing. They are useful in different scenarios and have their own failure profiles and performance characteristics. Both indexes are 'global' indexes - they live on their own tables and are copies of primary table data, which Phoenix ensures remain in-sync.
+Phoenix supports three types of indexing techniques: immutable, global, and local indexing.
+They are each useful in different scenarios and have their own failure profiles and performance characteristics.
-# Mutable Indexing
+## Immutable Indexing
-Often, the rows you are inserting are changing - pretty much any time you are not doing time-series data. In this case, use mutable indexing to ensure that your index is properly maintained as your data changes.
+Immutable indexing targets use cases that are _write once_, _append only_; this is common in time-series data, where you log once, but read multiple times. In this case, the indexing is managed entirely on the client - either we successfully write all the primary and index data or we return a failure to the client. Since once written, rows are never updated, no incremental index maintenance is required making them perform very well. This reduces the overhead of secondary indexing at write time. However, keep in mind that immutable indexing are only applicable in a limited set of use cases.
-All the performance penalties for indexes occur at write time. We intercept the primary table updates on write ([DELETE](language/index.html#delete), [UPSERT VALUES](language/index.html#upsert_values) and [UPSERT SELECT](language/index.html#upsert_select)), build the index update and then sent any necessary updates to all interested index tables. At read time, Phoenix will select the index table to use that will produce the fastest query time and directly scan it just like any other HBase table.
+One restriction of immutable indexes is that rows from the data table may not be deleted. Instead, the only way to delete rows is to drop the entire data table. This is likely to change once [PHOENIX-619](https://issues.apache.org/jira/browse/PHOENIX-619) is implemented.
-## Example
+## Mutable Indexing
+
+If a column values of the same row changes, you must use mutable indexing to ensure that your index is properly maintained. There are two types of mutable indexes: global and local. From a functional standpoint, these are nearly identical with the difference being in their performance characteristics and tradeoffs.
+
+### Global Indexing
+Global indexing targets _read heavy_, _low write_ uses cases. With global indexes, all the performance penalties for indexes occur at write time. We intercept the data table updates on write ([DELETE](language/index.html#delete), [UPSERT VALUES](language/index.html#upsert_values) and [UPSERT SELECT](language/index.html#upsert_select)), build the index update and then sent any necessary updates to all interested index tables. At read time, Phoenix will select the index table to use that will produce the fastest query time and directly scan it just like any other HBase table. Note, however, if a column is referenced in a query that isn't part of the index, the index will not be used for that query.
+
+### Local Indexing
+Local indexing targets _write heavy_, _space constrained_ use cases. With local indexes index data and table data are co-reside at same server so no network overhead during writes and reads. Local indexes can be used even when the query isn't fully covered i.e. Phoenix automatically retrieve the columns not in the index through point gets against the data table. Unlike global indexes all local indexes data of a table are stored in a separate shared table. At read time when the local index is used, every region must be examined for the data as the exact region location of index data cannot be predetermined which incurs some overhead.
+
+## Examples
Given the schema shown here:
CREATE TABLE my_table (k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 BIGINT);
-you'd create an index on the v1 column like this:
+you'd create a global index on the v1 column like this:
CREATE INDEX my_index ON my_table (v1);
+while the equivalent local index would be created like this:
+
+ CREATE LOCAL INDEX my_index ON my_table (v1);
A table may contain any number of indexes, but note that your write speed will drop as you add additional indexes.
-We can also include columns from the data table in the index apart from the indexed columns. This allows an index to be used more frequently, as it will only be used if all columns referenced in the query are contained by it.
+We can also include columns from the data table in the index apart from the indexed columns. This allows a global index to be used more frequently, as a global index will only be used if all columns referenced in the query are contained by it. This is not the case for local indexes.
CREATE INDEX my_index ON my_table (v1) INCLUDE (v2);
In addition, multiple columns may be indexed and their values may be stored in ascending or descending order.
@@ -31,54 +45,23 @@ Finally, just like with the <code>CREATE
CREATE INDEX my_index ON my_table (v2 DESC, v1) INCLUDE (v3)
SALT_BUCKETS=10, DATA_BLOCK_ENCODING='NONE';
-Note that if the primary table is salted, then the index is automatically salted in the same way. In addition, the MAX_FILESIZE for the index is adjusted down, relative to the size of the primary versus index table. For more on salting see [here](salted.html).
-
-# Immutable Indexing
+Note that if the primary table is salted, then the index is automatically salted in the same way for global indexes. In addition, the MAX_FILESIZE for the index is adjusted down, relative to the size of the primary versus index table. For more on salting see [here](salted.html). With local indexes, on the other hand, specifying SALT_BUCKETS is not allowed.
-Immutable indexing targets use cases that are _write once_, _append only_; this is common in time-series data, where you log once, but read multiple times. In this case, the indexing is managed entirely on the client - either we successfully write all the primary and index data or we return a failure to the client. Since once written, rows are never updated, no incremental index maintenance is required. This reduces the overhead of secondary indexing at write time. However, keep in mind that immutable indexing are only applicable in a limited set of use cases.
+To drop an index, you'd issue the following statement:
+ DROP INDEX my_index ON my_table
-## Example
+If an indexed column is dropped in the data table, the index will automatically be dropped. In addition, if a covered column is dropped in the data table, it will be automatically dropped from the index as well.
To use immutable indexing, supply an <code>IMMUTABLE_ROWS=true</code> property when you create your table like this:
CREATE TABLE my_table (k VARCHAR PRIMARY KEY, v VARCHAR) IMMUTABLE_ROWS=true;
-
-Other than that, all of the previous examples are identical for immutable indexing.
+In that case, all indexes on the table are immutable indexes and local indexes are not allowed.
If you have an existing table that you'd like to switch from immutable indexing to mutable indexing, use the <code>ALTER TABLE</code> command as show below:
ALTER TABLE my_table SET IMMUTABLE_ROWS=false;
-For the complete syntax, see our [Language Reference Guide](language/index.html#create_index).
-
-# Local(Region level) indexing
-
-Local indexing targets _write heavy_, _low latency_ and _space constraint_ use cases. With local indexes index data and table data are co-reside at same server so no network overhead during writes and reads. Local indexes can be used even when the query isn't fully covered i.e. Phoenix automatically retrieve the columns not in the index through point gets against the data table. Unlike global indexes all local indexes data of a table are stored in a separate shared table.
-
-Reading data via the local index does however require to contact each region until unless query contains equal/range condition(s) on leading primary key column(s) of the data table.
-
-## Example
-
-To use local indexing, just supply a <code>LOCAL</code> keyword when you create index like this:
-
- CREATE LOCAL INDEX my_index ON my_table (v1);
-For the complete syntax, see our [Language Reference Guide](language/index.html#create_index).
-
-### Setup
-
-Local indexing requires special configurations in the master to ensure data table and local index regions co-location.
-You will need to add the following parameters to `hbase-site.xml`:
-
-```
-<property>
- <name>hbase.master.loadbalancer.class</name>
- <value>org.apache.phoenix.hbase.index.balancer.IndexLoadBalancer</value>
-</property>
-<property>
- <name>hbase.coprocessor.master.classes</name>
- <value>org.apache.phoenix.hbase.index.master.IndexMasterObserver</value>
-</property>
-```
+For the complete syntax, see our [Language Reference Guide](language/index.html).
## Data Guarantees and Failure Management
@@ -115,38 +98,21 @@ ALTER INDEX my_index ON my_table REBUILD
If we cannot disable the index, then the server will be immediately aborted. If the abort fails, we call System.exit on the JVM, forcing the server to die. By killing the server, we ensure that the WAL will be replayed on recovery, replaying the index updates to their appropriate tables.
-**WARNING: indexing has the potential to bring down your entire cluster very quickly.**
+**WARNING: global indexing has the potential to bring down your entire cluster very quickly.**
If the index tables are not setup correctly (Phoenix ensures that they are), this failure policy can cause a cascading failure as each region server attempts and fails to write the index update, subsequently killing itself to ensure the visibility concerns outlined above.
## Setup
-Only mutable indexing requires special configuration options in the region server to run - phoenix ensures that they are setup correctly when you enable mutable indexing on the table; if the correct properties are not set, you will not be able to turn it on.
+Mutable indexing requires special configuration options on the region server and master to run - Phoenix ensures that they are setup correctly when you enable mutable indexing on the table; if the correct properties are not set, you will not be able to use secondary indexing. After adding these settings to your hbase-site.xml, you'll need to do a rolling restart of your cluster.
-You will need to add the following parameters to `hbase-site.xml`:
+You will need to add the following parameters to `hbase-site.xml` on each region server:
```
<property>
<name>hbase.regionserver.wal.codec</name>
<value>org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec</value>
</property>
-```
-
-This enables custom WAL edits to be written, ensuring proper writing/replay of the index updates. This codec supports the usual host of WALEdit options, most notably WALEdit compression.
-
-### Advanced Setup - Removing Index Deadlocks (0.98.4+)
-
-Phoenix releases that include these changes (4.1+, 5.0.0+) are still backwards compatible with older versions of phoenix (to the extent that they are semantically compatible) as well as with older versions of HBase (0.98.1-0.98.3).
-
-As of HBase 0.98.4 we can finally remove the change of index deadlocks. In HBase you can tune the number of RPC threads to match client writes + index writes, but there is still a chance you could have a deadlock in an unlucky scenario (i.e. Client A -> Server A, Client B -> Server B, each taking the last RPC thread. Then each server attempts to make an index update to the other, Server A -> Server B, and vice versa, but they can't as there are no more available RPC threads).
-
-As of [PHOENIX-938](https://issues.apache.org/jira/browse/PHOENIX-938) and [HBASE-11513](https://issues.apache.org/jira/browse/HBASE-11513) we can remove these deadlocks by providing a different set of RPC handlers for index updates by giving index updates their own 'rpc priority' and handling the priorities via a custom Phoenix RPC Handler.
-
-The properties you need to set to enable this are
-
-#### Server Side
-
-```
<property>
<name>hbase.region.server.rpc.scheduler.factory.class</name>
<value>org.apache.phoenix.hbase.index.ipc.PhoenixIndexRpcSchedulerFactory</value>
@@ -154,93 +120,80 @@ The properties you need to set to enable
</property>
```
-After adding these settings to your hbase-site.xml, you just need to do a rolling restart of your cluster.
-
-
-Note that having the configs on both client and server side will not impact correctness or performance.
+The first property enables custom WAL edits to be written, ensuring proper writing/replay of the index updates. This codec supports the usual host of WALEdit options, most notably WALEdit compression.
-#### Tuning
+The second property prevents deadlocks from occurring during index maintenance for global indexes (HBase 0.98.4+ only) by ensuring index updates are processed with a higher priority than data updates.
-By default, index priority range is between (1000, 1050]. Higher priorites within the index range, at this time, do not means updates are processed sooner. However, we reserve this range to provide that possibility in the future.
+Local indexing also requires special configurations in the master to ensure data table and local index regions co-location.
-You can specifiy this range however to suit your individual cluster requirements by adjusting the follwing parameters
+You will need to add the following parameters to `hbase-site.xml` on the master:
```
<property>
- <name>org.apache.phoenix.regionserver.index.priority.min</name>
- <value>1050</value>
- <description>Value to specify to bottom (inclusive) of the range in which index priority may lie</description>
+ <name>hbase.master.loadbalancer.class</name>
+ <value>org.apache.phoenix.hbase.index.balancer.IndexLoadBalancer</value>
</property>
<property>
- <name>org.apache.phoenix.regionserver.index.priority.max</name>
- <value>1050</value>
- <description>Value to specify to top (exclusive) of the range in which index priority may lie</description>
-</property>
-```
-
-The number of RPC Handler Threads can be specified via:
-
-```
-<property>
- <name>org.apache.phoenix.regionserver.index.handler.count</name>
- <value>30</value>
- <description>Number of threads to use when serving index write requests</description>
+ <name>hbase.coprocessor.master.classes</name>
+ <value>org.apache.phoenix.hbase.index.master.IndexMasterObserver</value>
</property>
```
-Though the actual number of threads is dictated by the Max(number of call queues, handler count), where the number of call queues is determined by standard HBase configuration (see below).
-
-
-To further tune the queues, you can adjust the standard rpc queue length parameters (currently, there are no special knobs for the index queues), specifically "ipc.server.max.callqueue.length" and "ipc.server.callqueue.handler.factor". See the [HBase Reference Guide](http://hbase.apache.org/book.html) for more details.
-
## Tuning
Out the box, indexing is pretty fast. However, to optimize for your particular environment and workload, there are several properties you can tune.
All the following parameters must be set in `hbase-site.xml` - they are true for the entire cluster and all index tables, as well as across all regions on the same server (so, for instance, a single server would not write to too many different index tables at once).
1. index.builder.threads.max
-
* Number of threads to used to build the index update from the primary table update
* Increasing this value overcomes the bottleneck of reading the current row state from the underlying HRegion. Tuning this value too high will just bottleneck at the HRegion as it will not be able to handle too many concurrent scan requests as well as general thread-swapping concerns.
* **Default: 10**
2. index.builder.threads.keepalivetime
-
* Amount of time in seconds after we expire threads in the builder thread pool.
* Unused threads are immediately released after this amount of time and not core threads are retained (though this last is a small concern as tables are expected to sustain a fairly constant write load), but simultaneously allows us to drop threads if we are not seeing the expected load.
* **Default: 60**
-
3. index.writer.threads.max
* Number of threads to use when writing to the target index tables.
* The first level of parallelization, on a per-table basis - it should roughly correspond to the number of index tables
* **Default: 10**
-
4. index.writer.threads.keepalivetime
* Amount of time in seconds after we expire threads in the writer thread pool.
* Unused threads are immediately released after this amount of time and not core threads are retained (though this last is a small concern as tables are expected to sustain a fairly constant write load), but simultaneously allows us to drop threads if we are not seeing the expected load.
* **Default: 60**
-
5. hbase.htable.threads.max
* Number of threads each index HTable can use for writes.
* Increasing this allows more concurrent index updates (for instance across batches), leading to high overall throughput.
* **Default: 2,147,483,647**
-
6. hbase.htable.threads.keepalivetime
* Amount of time in seconds after we expire threads in the HTable's thread pool.
* Using the "direct handoff" approach, new threads will only be created if it is necessary and will grow unbounded. This could be bad but HTables only create as many Runnables as there are region servers; therefore, it also scales when new region servers are added.
* **Default: 60**
-
7. index.tablefactory.cache.size
* Number of index HTables we should keep in cache.
* Increasing this number ensures that we do not need to recreate an HTable for each attempt to write to an index table. Conversely, you could see memory pressure if this value is set too high.
* **Default: 10**
+8. org.apache.phoenix.regionserver.index.priority.min
+ * Value to specify to bottom (inclusive) of the range in which index priority may lie.
+ * **Default: 1000**
+9. org.apache.phoenix.regionserver.index.priority.max
+ * Value to specify to top (exclusive) of the range in which index priority may lie.
+ * Higher priorites within the index min/max range do not means updates are processed sooner.
+ * **Default: 1050**
+10. org.apache.phoenix.regionserver.index.handler.count
+ * Number of threads to use when serving index write requests for global index maintenance.
+ * Though the actual number of threads is dictated by the Max(number of call queues, handler count), where the number of call queues is determined by standard HBase configuration. To further tune the queues, you can adjust the standard rpc queue length parameters (currently, there are no special knobs for the index queues), specifically <code>ipc.server.max.callqueue.length</code> and <code>ipc.server.callqueue.handler.factor</code>. See the [HBase Reference Guide](http://hbase.apache.org/book.html) for more details.
+ * **Default: 30**
+
# Performance
We track secondary index performance via our [performance framework](http://phoenix-bin.github.io/client/performance/latest.htm). This is a generic test of performance based on defaults - your results will vary based on hardware specs as well as you individual configuration.
That said, we have seen secondary indexing (both immutable and mutable) go as quickly as < 2x the regular write path on a small, (3 node) desktop-based cluster. This is actually a phenomenal as we have to write to multiple tables as well as build the index update.
-# Presentations
+# Resources
There have been several presentations given on how secondary indexing works in Phoenix that have a more in-depth look at how indexing works (with pretty pictures!):
* [San Francisco HBase Meetup](http://files.meetup.com/1350427/PhoenixIndexing-SF-HUG_09-26-13.pptx) - Sept. 26, 2013
* [Los Anglees HBase Meetup](http://www.slideshare.net/jesse_yates/phoenix-secondary-indexing-la-hug-sept-9th-2013) - Sept, 4th, 2013
+* [Local Indexes](https://github.com/Huawei-Hadoop/hindex/blob/master/README.md#how-it-works) by Huawei
+* [PHOENIX-938](https://issues.apache.org/jira/browse/PHOENIX-938) and [HBASE-11513](https://issues.apache.org/jira/browse/HBASE-11513) for deadlock prevention during global index maintenance.