You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by je...@apache.org on 2014/07/31 08:11:57 UTC

git commit: PHOENIX-1112: Atomically rebuild index partially when index update fails

Repository: phoenix
Updated Branches:
  refs/heads/3.0 0d28d50f5 -> ddf970a38


PHOENIX-1112: Atomically rebuild index partially when index update fails


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ddf970a3
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ddf970a3
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ddf970a3

Branch: refs/heads/3.0
Commit: ddf970a38ba0abeff197c68d77e25b8ea11fdd2e
Parents: 0d28d50
Author: Jeffrey Zhong <je...@apache.org>
Authored: Mon Jul 21 18:32:56 2014 -0700
Committer: Jeffrey Zhong <je...@apache.org>
Committed: Wed Jul 30 23:12:04 2014 -0700

----------------------------------------------------------------------
 phoenix-core/pom.xml                            |  27 ++-
 phoenix-core/src/build/phoenix-core.xml         |  53 +++++
 .../end2end/index/MutableIndexFailureIT.java    |  71 ++++--
 .../phoenix/compile/StatementContext.java       |  11 +
 .../coprocessor/MetaDataEndpointImpl.java       |  59 ++++-
 .../coprocessor/MetaDataRegionObserver.java     | 224 ++++++++++++++++++-
 .../apache/phoenix/execute/BasicQueryPlan.java  |  19 +-
 .../org/apache/phoenix/hbase/index/Indexer.java |   2 +-
 .../index/PhoenixIndexFailurePolicy.java        |  32 ++-
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |   3 +
 .../phoenix/mapreduce/CsvBulkLoadTool.java      |   2 +-
 .../query/ConnectionQueryServicesImpl.java      |  13 +-
 .../apache/phoenix/query/QueryConstants.java    |   5 +-
 .../org/apache/phoenix/query/QueryServices.java |  11 +
 .../phoenix/query/QueryServicesOptions.java     |   3 +
 .../apache/phoenix/schema/MetaDataClient.java   |  59 ++++-
 .../org/apache/phoenix/schema/TableRef.java     |  23 +-
 .../org/apache/phoenix/util/MetaDataUtil.java   |  52 +++++
 18 files changed, 593 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index e7ff08e..19ffd09 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -168,9 +168,30 @@
       <plugin>
         <!--Make it so assembly:single does nothing in here -->
         <artifactId>maven-assembly-plugin</artifactId>
-        <configuration>
-          <skipAssembly>true</skipAssembly>
-        </configuration>
+        <executions>
+          <execution>
+            <id>core</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+            <configuration>
+              <attach>false</attach>
+              <finalName>phoenix</finalName>
+              <archive>
+                <index>true</index>
+                <manifest>
+                  <addClasspath>true</addClasspath>
+                  <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+                  <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+                </manifest>
+              </archive>
+              <descriptors>
+                <descriptor>src/build/phoenix-core.xml</descriptor>
+              </descriptors>
+            </configuration>
+          </execution>
+        </executions>      
       </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/build/phoenix-core.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/src/build/phoenix-core.xml b/phoenix-core/src/build/phoenix-core.xml
new file mode 100644
index 0000000..7b8df1e
--- /dev/null
+++ b/phoenix-core/src/build/phoenix-core.xml
@@ -0,0 +1,53 @@
+<?xml version='1.0'?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+  <id>core-${parent.version}</id>
+  <!-- All the dependencies (unpacked) necessary to run phoenix from a single, stand-alone jar -->
+  <formats>
+    <format>jar</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  
+  <dependencySets>
+    <dependencySet>
+      <!-- Unpack all the dependencies to class files, since java doesn't support 
+        jar of jars for running -->
+      <unpack>true</unpack>
+      <!-- save these dependencies to the top-level -->
+      <outputDirectory>/</outputDirectory>
+      <includes>
+        <include>org.antlr:antlr-runtime</include>
+      </includes>
+    </dependencySet>
+
+    <dependencySet>
+      <outputDirectory>/</outputDirectory>
+      <unpack>true</unpack>
+      <includes>
+        <include>org.apache.phoenix:phoenix-core*</include>
+      </includes>
+    </dependencySet>
+  </dependencySets>
+</assembly>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 0601c2b..bd9ec52 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -26,7 +26,6 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import java.sql.Connection;
 import java.sql.PreparedStatement;
@@ -37,13 +36,13 @@ import java.util.Properties;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
-import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixTestDriver;
 import org.apache.phoenix.query.BaseTest;
-import org.apache.phoenix.query.ConnectionQueryServices;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.schema.PIndexState;
 import org.apache.phoenix.schema.PTableType;
@@ -81,6 +80,7 @@ public class MutableIndexFailureIT extends BaseTest {
         setUpConfigForMiniCluster(conf);
         conf.setInt("hbase.client.retries.number", 2);
         conf.setInt("hbase.client.pause", 5000);
+        conf.setLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB, 0);
         util = new HBaseTestingUtility(conf);
         util.startMiniCluster();
         String clientPort = util.getConfiguration().get(QueryServices.ZOOKEEPER_PORT_ATTRIB);
@@ -98,21 +98,7 @@ public class MutableIndexFailureIT extends BaseTest {
         }
     }
 
-    private static void destroyIndexTable() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = driver.connect(url, props);
-        ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices();
-        HBaseAdmin admin = services.getAdmin();
-        try {
-            admin.disableTable(INDEX_TABLE_FULL_NAME);
-            admin.deleteTable(INDEX_TABLE_FULL_NAME);
-        } catch (TableNotFoundException e) {} finally {
-            conn.close();
-            admin.close();
-        }
-    }
-
-    @Test
+    @Test(timeout=300000)
     public void testWriteFailureDisablesIndex() throws Exception {
         String query;
         ResultSet rs;
@@ -139,17 +125,29 @@ public class MutableIndexFailureIT extends BaseTest {
         assertEquals(INDEX_TABLE_NAME, rs.getString(3));
         assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
         assertFalse(rs.next());
-
-        destroyIndexTable();
-
+        
         PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
         stmt.setString(1, "a");
         stmt.setString(2, "x");
         stmt.setString(3, "1");
         stmt.execute();
+        conn.commit();
+
+        byte[] indexTable = Bytes.toBytes(INDEX_TABLE_FULL_NAME);
+        HBaseAdmin admin = this.util.getHBaseAdmin();
+        HTableDescriptor indexTableDesc = admin.getTableDescriptor(indexTable);
+        try{
+          admin.disableTable(indexTable);
+          admin.deleteTable(indexTable);
+        } catch (TableNotFoundException ignore) {}
+
+        stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
+        stmt.setString(1, "a2");
+        stmt.setString(2, "x2");
+        stmt.setString(3, "2");
+        stmt.execute();
         try {
             conn.commit();
-            fail();
         } catch (SQLException e) {}
 
         // Verify the metadata for index is correct.
@@ -159,5 +157,32 @@ public class MutableIndexFailureIT extends BaseTest {
         assertEquals(INDEX_TABLE_NAME, rs.getString(3));
         assertEquals(PIndexState.DISABLE.toString(), rs.getString("INDEX_STATE"));
         assertFalse(rs.next());
+        
+        stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
+        stmt.setString(1, "a3");
+        stmt.setString(2, "x3");
+        stmt.setString(3, "3");
+        stmt.execute();
+        conn.commit();
+        
+        // recreate index table
+        admin.createTable(indexTableDesc);
+        do {
+          Thread.sleep(15 * 1000); // sleep 15 secs
+          rs = conn.getMetaData().getTables(null, "", INDEX_TABLE_NAME, new String[] {PTableType.INDEX.toString()});
+          assertTrue(rs.next());
+          if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
+              break;
+          }
+        } while(true);
+        
+        // verify index table has data
+        query = "SELECT count(1) FROM " + INDEX_TABLE_FULL_NAME;
+        rs = conn.createStatement().executeQuery(query);
+        assertTrue(rs.next());
+        
+        // using 2 here because we onluy partially build index from where we failed and the oldest 
+        // index row has been deleted when we dropped the index table during test.
+        assertEquals(2, rs.getInt(1));
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
index 06d5f89..387696b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -24,6 +24,7 @@ import java.util.List;
 
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixStatement;
@@ -68,6 +69,7 @@ public class StatementContext {
     private TableRef currentTable;
     private List<Pair<byte[], byte[]>> whereConditionColumns;
     private TupleProjector clientTupleProjector;
+    private TimeRange scanTimeRange = null;
     
     public StatementContext(PhoenixStatement statement) {
         this(statement, FromCompiler.EMPTY_TABLE_RESOLVER, new Scan(), new SequenceManager(statement));
@@ -265,4 +267,13 @@ public class StatementContext {
     public void setClientTupleProjector(TupleProjector projector) {
         this.clientTupleProjector = projector;
     }
+
+    public void setScanTimeRange(TimeRange value){
+    	this.scanTimeRange = value;
+    }
+    
+    public TimeRange getScanTimeRange() {
+    	return this.scanTimeRange;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index b28fd2d..13adff8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -49,6 +49,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT_BYTE
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_BYTES;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT_BYTES;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE_BYTES;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES;
 import static org.apache.phoenix.schema.PTableType.INDEX;
 import static org.apache.phoenix.util.SchemaUtil.getVarCharLength;
 import static org.apache.phoenix.util.SchemaUtil.getVarChars;
@@ -63,6 +64,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Delete;
@@ -995,14 +997,6 @@ public class MetaDataEndpointImpl extends BaseEndpointCoprocessor implements Met
         
     }
     
-    private static MetaDataMutationResult checkTableKeyInRegion(byte[] key, HRegion region) {
-        byte[] startKey = region.getStartKey();
-        byte[] endKey = region.getEndKey();
-        if (Bytes.compareTo(startKey, key) <= 0 && (Bytes.compareTo(HConstants.LAST_ROW, endKey) == 0 || Bytes.compareTo(key, endKey) < 0)) {
-            return null; // normal case;
-        }
-        return new MetaDataMutationResult(MutationCode.TABLE_NOT_IN_REGION, EnvironmentEdgeManager.currentTimeMillis(), null);
-    }
 
     @Override
     public MetaDataMutationResult getTable(byte[] tenantId, byte[] schemaName, byte[] tableName, long tableTimeStamp, long clientTimeStamp) throws IOException {
@@ -1126,7 +1120,17 @@ public class MetaDataEndpointImpl extends BaseEndpointCoprocessor implements Met
             long timeStamp = MetaDataUtil.getClientTimeStamp(tableMetadata);
             ImmutableBytesPtr cacheKey = new ImmutableBytesPtr(key);
             List<KeyValue> newKVs = tableMetadata.get(0).getFamilyMap().get(TABLE_FAMILY_BYTES);
-            KeyValue newKV = newKVs.get(0);
+            KeyValue newKV = null;
+            int disableTimeStampKVIndex = -1;
+            int index = 0;
+            for(KeyValue cell : newKVs){
+                if(Bytes.compareTo(cell.getQualifier(), INDEX_STATE_BYTES) == 0){
+                  newKV = cell;
+                } else if (Bytes.compareTo(cell.getQualifier(), INDEX_DISABLE_TIMESTAMP_BYTES) == 0){
+                  disableTimeStampKVIndex = index;
+                }
+                index++;
+            }
             PIndexState newState =  PIndexState.fromSerializedValue(newKV.getBuffer()[newKV.getValueOffset()]);
             Integer lid = region.getLock(null, key, true);
             if (lid == null) {
@@ -1136,19 +1140,39 @@ public class MetaDataEndpointImpl extends BaseEndpointCoprocessor implements Met
                 Get get = new Get(key);
                 get.setTimeRange(PTable.INITIAL_SEQ_NUM, timeStamp);
                 get.addColumn(TABLE_FAMILY_BYTES, INDEX_STATE_BYTES);
+                get.addColumn(TABLE_FAMILY_BYTES, INDEX_DISABLE_TIMESTAMP_BYTES);
                 Result currentResult = region.get(get);
                 if (currentResult.raw().length == 0) {
                     return new MetaDataMutationResult(MutationCode.TABLE_NOT_FOUND, EnvironmentEdgeManager.currentTimeMillis(), null);
                 }
-                KeyValue currentStateKV = currentResult.raw()[0];
+                KeyValue currentStateKV = currentResult.getColumnLatest(TABLE_FAMILY_BYTES, INDEX_STATE_BYTES);
+                KeyValue currentDisableTimeStamp = currentResult.getColumnLatest(TABLE_FAMILY_BYTES, INDEX_DISABLE_TIMESTAMP_BYTES);
+               
                 PIndexState currentState = PIndexState.fromSerializedValue(currentStateKV.getBuffer()[currentStateKV.getValueOffset()]);
+                
+                // check if we need reset disable time stamp
+                if( (newState == PIndexState.DISABLE) && 
+                    (currentState == PIndexState.DISABLE || currentState == PIndexState.INACTIVE) && 
+                    (currentDisableTimeStamp != null && currentDisableTimeStamp.getValueLength() > 0) &&
+                    (disableTimeStampKVIndex >= 0)) {
+                    Long curTimeStampVal = (Long)PDataType.LONG.toObject(currentDisableTimeStamp.getBuffer());
+                    // new DisableTimeStamp is passed in
+                    KeyValue newDisableTimeStampCell = newKVs.get(disableTimeStampKVIndex);
+                    Long newDisableTimeStamp = (Long)PDataType.LONG.toObject(newDisableTimeStampCell.getBuffer());
+                    if(curTimeStampVal > 0 && curTimeStampVal < newDisableTimeStamp){
+                        // not reset disable timestamp
+                        newKVs.remove(disableTimeStampKVIndex);
+                    }
+                }
+                
                 // Detect invalid transitions
                 if (currentState == PIndexState.BUILDING) {
                     if (newState == PIndexState.USABLE) {
                         return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION, EnvironmentEdgeManager.currentTimeMillis(), null);
                     }
                 } else if (currentState == PIndexState.DISABLE) {
-                    if (newState != PIndexState.BUILDING && newState != PIndexState.DISABLE) {
+                    if (newState != PIndexState.BUILDING && newState != PIndexState.DISABLE &&
+                        newState != PIndexState.INACTIVE) {
                         return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION, EnvironmentEdgeManager.currentTimeMillis(), null);
                     }
                     // Done building, but was disable before that, so that in disabled state
@@ -1167,6 +1191,7 @@ public class MetaDataEndpointImpl extends BaseEndpointCoprocessor implements Met
                     newState = PIndexState.ACTIVE;
                     newKVs.set(0, KeyValueUtil.newKeyValue(key, TABLE_FAMILY_BYTES, INDEX_STATE_BYTES, timeStamp, Bytes.toBytes(newState.getSerializedValue())));
                 }
+                
                 if (currentState != newState) {
                     region.mutateRowsWithLocks(tableMetadata, Collections.<byte[]>emptySet());
                     // Invalidate from cache
@@ -1185,6 +1210,18 @@ public class MetaDataEndpointImpl extends BaseEndpointCoprocessor implements Met
         }
     }
     
+    private static MetaDataMutationResult checkTableKeyInRegion(byte[] key, HRegion region) {
+        byte[] startKey = region.getStartKey();
+        byte[] endKey = region.getEndKey();
+        if (Bytes.compareTo(startKey, key) <= 0
+                && (Bytes.compareTo(HConstants.LAST_ROW, endKey) == 0 || Bytes.compareTo(key,
+                    endKey) < 0)) {
+            return null; // normal case;
+        }
+        return new MetaDataMutationResult(MutationCode.TABLE_NOT_IN_REGION,
+                EnvironmentEdgeManager.currentTimeMillis(), null);
+    }
+
     /**
      * 
      * Matches rows that end with a given byte array suffix

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
index 2ef8b3a..5fcdb8b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
@@ -19,13 +19,44 @@ package org.apache.phoenix.coprocessor;
 
 import java.io.IOException;
 
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.phoenix.jdbc.PhoenixDriver;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.cache.GlobalCache;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.MetaDataClient;
+import org.apache.phoenix.schema.PDataType;
+import org.apache.phoenix.schema.PIndexState;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.util.MetaDataUtil;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.SchemaUtil;
 
 
 /**
@@ -33,26 +64,195 @@ import org.apache.phoenix.query.QueryServicesOptions;
  * to SYSTEM.TABLE.
  */
 public class MetaDataRegionObserver extends BaseRegionObserver {
-
+    public static final Log LOG = LogFactory.getLog(MetaDataRegionObserver.class);
+    protected Timer scheduleTimer = new Timer(true);
+    private boolean enableRebuildIndex = QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD;
+    private long rebuildIndexTimeInterval = QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL;
+  
     @Override
     public void preClose(final ObserverContext<RegionCoprocessorEnvironment> c,
             boolean abortRequested) {
+        scheduleTimer.cancel();
         GlobalCache.getInstance(c.getEnvironment()).getMetaDataCache().invalidateAll();
     }
     
     @Override
     public void start(CoprocessorEnvironment env) throws IOException {
-      // sleep a little bit to compensate time clock skew when SYSTEM.CATALOG moves 
-      // among region servers because we relies on server time of RS which is hosting
-      // SYSTEM.CATALOG
-      long sleepTime = env.getConfiguration().getLong(QueryServices.CLOCK_SKEW_INTERVAL_ATTRIB, 
-          QueryServicesOptions.DEFAULT_CLOCK_SKEW_INTERVAL);
-      try {
-          if(sleepTime > 0) {
-              Thread.sleep(sleepTime);
-          }
-      } catch (InterruptedException ie) {
-          Thread.currentThread().interrupt();
+        // sleep a little bit to compensate time clock skew when SYSTEM.CATALOG moves 
+        // among region servers because we relies on server time of RS which is hosting
+        // SYSTEM.CATALOG
+        long sleepTime = env.getConfiguration().getLong(QueryServices.CLOCK_SKEW_INTERVAL_ATTRIB, 
+            QueryServicesOptions.DEFAULT_CLOCK_SKEW_INTERVAL);
+        try {
+            if(sleepTime > 0) {
+                Thread.sleep(sleepTime);
+            }
+        } catch (InterruptedException ie) {
+            Thread.currentThread().interrupt();
+        }
+        enableRebuildIndex = env.getConfiguration().getBoolean(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_ATTRIB, 
+            QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD);
+        rebuildIndexTimeInterval = env.getConfiguration().getLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, 
+            QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL);
+    }
+    
+
+    @Override
+    public void postOpen(ObserverContext<RegionCoprocessorEnvironment> e) {
+        if (!enableRebuildIndex) {
+            LOG.info("Failure Index Rebuild is skipped by configuration.");
+            return;
+        }
+        // turn off verbose deprecation logging
+        Logger deprecationLogger = Logger.getLogger("org.apache.hadoop.conf.Configuration.deprecation");
+        if (deprecationLogger != null) {
+            deprecationLogger.setLevel(Level.WARN);
+        }
+        try {
+            Class.forName(PhoenixDriver.class.getName());
+            // starts index rebuild schedule work
+            BuildIndexScheduleTask task = new BuildIndexScheduleTask(e.getEnvironment());
+            // run scheduled task every 10 secs
+            scheduleTimer.schedule(task, 10000, rebuildIndexTimeInterval);
+        } catch (ClassNotFoundException ex) {
+            LOG.error("BuildIndexScheduleTask cannot start!", ex);
+        }
+    }
+    
+    /**
+     * Task runs periodically to build indexes whose INDEX_NEED_PARTIALLY_REBUILD is set true
+     *
+     */
+    public static class BuildIndexScheduleTask extends TimerTask {
+      // inProgress is to prevent timer from invoking a new task while previous one is still running
+      private final static AtomicInteger inProgress = new AtomicInteger(0);
+      RegionCoprocessorEnvironment env;
+      public BuildIndexScheduleTask(RegionCoprocessorEnvironment env) {
+        this.env = env;
       }
+      
+        private String getJdbcUrl() {
+            String zkQuorum = this.env.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM);
+            String zkClientPort = this.env.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT,
+                Integer.toString(HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT));
+            String zkParentNode = this.env.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT,
+                HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
+            return PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum
+                + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkClientPort
+                + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkParentNode;
+        }
+      
+        public void run() {
+            RegionScanner scanner = null;
+            PhoenixConnection conn = null;
+            if (inProgress.get() > 0) {
+                LOG.debug("New ScheduledBuildIndexTask skipped as there is already one running");
+                return;
+            }
+            try {
+                inProgress.incrementAndGet();
+                Scan scan = new Scan();
+                SingleColumnValueFilter filter = new SingleColumnValueFilter(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                    PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES,
+                    CompareFilter.CompareOp.NOT_EQUAL, PDataType.LONG.toBytes(0L));
+                filter.setFilterIfMissing(true);
+                scan.setFilter(filter);
+                scan.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                    PhoenixDatabaseMetaData.TABLE_NAME_BYTES);
+                scan.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                    PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES);
+                scan.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                    PhoenixDatabaseMetaData.INDEX_STATE_BYTES);
+                scan.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                    PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES);
+
+                boolean hasMore = false;
+                List<KeyValue> results = new ArrayList<KeyValue>();
+                scanner = this.env.getRegion().getScanner(scan);
+
+                do {
+                    results.clear();
+                    hasMore = scanner.next(results);
+                    if (results.isEmpty()) break;
+
+                    Result r = new Result(results);
+                    byte[] disabledTimeStamp = r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                        PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES);
+
+                    Long disabledTimeStampVal = 0L;
+                    if (disabledTimeStamp == null || disabledTimeStamp.length == 0) {
+                        continue;
+                    }
+
+                    // disableTimeStamp has to be a positive value
+                    disabledTimeStampVal = (Long) PDataType.LONG.toObject(disabledTimeStamp);
+                    if (disabledTimeStampVal <= 0) {
+                        continue;
+                    }
+
+                    byte[] dataTable = r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                        PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES);
+                    byte[] indexStat = r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                        PhoenixDatabaseMetaData.INDEX_STATE_BYTES);
+                    if ((dataTable == null || dataTable.length == 0)
+                            || (indexStat == null || indexStat.length == 0)
+                            || ((Bytes.compareTo(PIndexState.DISABLE.getSerializedBytes(), indexStat) != 0) 
+                                    && (Bytes.compareTo(PIndexState.INACTIVE.getSerializedBytes(), indexStat) != 0))) {
+                        // index has to be either in disable or inactive state
+                        // data table name can't be empty
+                        continue;
+                    }
+
+                    byte[][] rowKeyMetaData = new byte[3][];
+                    SchemaUtil.getVarChars(r.getRow(), 3, rowKeyMetaData);
+                    byte[] indexTable = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
+
+                    // validity check
+                    if (indexTable == null || indexTable.length == 0) {
+                        LOG.debug("Index rebuild has been skipped for row=" + r);
+                        continue;
+                    }
+
+                    if (conn == null) {
+                        conn = DriverManager.getConnection(getJdbcUrl()).unwrap(PhoenixConnection.class);
+                    }
+                    PTable dataPTable = PhoenixRuntime.getTable(conn, Bytes.toString(dataTable));
+                    PTable indexPTable = PhoenixRuntime.getTable(conn, Bytes.toString(indexTable));
+                    if (!MetaDataUtil.tableRegionsOnline(this.env.getConfiguration(), indexPTable)) {
+                        LOG.debug("Index rebuild has been skipped because not all regions of index table="
+                                + indexPTable.getName() + " are online.");
+                        continue;
+                    }
+
+                    MetaDataClient client = new MetaDataClient(conn);
+                    long overlapTime = env.getConfiguration().getLong(
+                        QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB,
+                        QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME);
+                    long timeStamp = Math.max(0, disabledTimeStampVal - overlapTime);
+
+                    LOG.info("Starting to build index=" + indexPTable.getName() + " from timestamp=" + timeStamp);
+                    client.buildPartialIndexFromTimeStamp(indexPTable, new TableRef(dataPTable, Long.MAX_VALUE, timeStamp));
+
+                } while (hasMore);
+            } catch (Throwable t) {
+                LOG.warn("ScheduledBuildIndexTask failed!", t);
+            } finally {
+                inProgress.decrementAndGet();
+                if (scanner != null) {
+                    try {
+                        scanner.close();
+                    } catch (IOException ignored) {
+                        LOG.debug("ScheduledBuildIndexTask can't close scanner.", ignored);
+                    }
+                }
+                if (conn != null) {
+                    try {
+                        conn.close();
+                    } catch (SQLException ignored) {
+                        LOG.debug("ScheduledBuildIndexTask can't close connection", ignored);
+                    }
+                }
+            }
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/execute/BasicQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BasicQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BasicQueryPlan.java
index d0c97bd..42720a0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BasicQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BasicQueryPlan.java
@@ -17,6 +17,7 @@
  */
 package org.apache.phoenix.execute;
 
+import java.io.IOException;
 import java.sql.ParameterMetaData;
 import java.sql.SQLException;
 import java.util.Collections;
@@ -139,16 +140,24 @@ public abstract class BasicQueryPlan implements QueryPlan {
         // is resolved.
         // TODO: include time range in explain plan?
         PhoenixConnection connection = context.getConnection();
-        Long scn = connection.getSCN();
-        if(scn == null) {
+        if (context.getScanTimeRange() == null) {
+          Long scn = connection.getSCN();
+          if (scn == null) {
             scn = context.getCurrentTime();
             // Add one to server time since max of time range is exclusive
             // and we need to account of OSs with lower resolution clocks.
-            if(scn < HConstants.LATEST_TIMESTAMP) {
-                scn++;
+            if (scn < HConstants.LATEST_TIMESTAMP) {
+              scn++;
             }
+          }
+          ScanUtil.setTimeRange(scan, scn);
+        } else {
+          try {
+            scan.setTimeRange(context.getScanTimeRange().getMin(), context.getScanTimeRange().getMax());
+          } catch (IOException e) {
+            throw new SQLException(e);
+          }
         }
-        ScanUtil.setTimeRange(scan, scn);
         ScanUtil.setTenantId(scan, connection.getTenantId() == null ? null : connection.getTenantId().getBytes());
         ResultIterator iterator = newIterator();
         return dependencies.isEmpty() ? 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
index 694e931..3ab400a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
@@ -584,7 +584,7 @@ public class Indexer extends BaseRegionObserver {
      * hopes they come up before the primary table finishes.
      */
     Collection<Pair<Mutation, byte[]>> indexUpdates = extractIndexUpdate(logEdit);
-    recoveryWriter.writeAndKillYourselfOnFailure(indexUpdates);
+    recoveryWriter.write(indexUpdates);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
index 5943e04..cf8335e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
@@ -18,6 +18,7 @@
 package org.apache.phoenix.index;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
@@ -25,6 +26,7 @@ import java.util.Set;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Mutation;
@@ -38,6 +40,8 @@ import org.apache.phoenix.coprocessor.MetaDataProtocol.MutationCode;
 import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.write.KillServerOnFailurePolicy;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+
+import org.apache.phoenix.schema.PDataType;
 import org.apache.phoenix.schema.PIndexState;
 import org.apache.phoenix.util.SchemaUtil;
 
@@ -68,9 +72,22 @@ public class PhoenixIndexFailurePolicy extends  KillServerOnFailurePolicy {
     @Override
     public void handleFailure(Multimap<HTableInterfaceReference, Mutation> attempted, Exception cause) throws IOException {
         Set<HTableInterfaceReference> refs = attempted.asMap().keySet();
-        StringBuilder buf = new StringBuilder("Disabled index" + (refs.size() > 1 ? "es " : " "));
         try {
             for (HTableInterfaceReference ref : refs) {
+                long minTimeStamp = 0;
+                Collection<Mutation> mutations = attempted.get(ref);
+                if (mutations != null) {
+                  for (Mutation m : mutations) {
+                    for (List<KeyValue> kvs : m.getFamilyMap().values()) {
+                      for (KeyValue kv : kvs) {
+                        if (minTimeStamp == 0 || (kv.getTimestamp() >=0 && minTimeStamp < kv.getTimestamp())) {
+                          minTimeStamp = kv.getTimestamp();
+                        }
+                      }
+                    }
+                  }
+                }
+                
                 // Disable the index by using the updateIndexState method of MetaDataProtocol end point coprocessor.
                 String indexTableName = ref.getTableName();
                 byte[] indexTableKey = SchemaUtil.getTableKeyFromFullName(indexTableName);
@@ -78,23 +95,22 @@ public class PhoenixIndexFailurePolicy extends  KillServerOnFailurePolicy {
                 MetaDataProtocol mdProxy = systemTable.coprocessorProxy(MetaDataProtocol.class, indexTableKey);
                 // Mimic the Put that gets generated by the client on an update of the index state
                 Put put = new Put(indexTableKey);
-                put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_STATE_BYTES, PIndexState.DISABLE.getSerializedBytes());
+                put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_STATE_BYTES, 
+                    PIndexState.DISABLE.getSerializedBytes());
+                put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES, 
+                    PDataType.LONG.toBytes(minTimeStamp));
                 List<Mutation> tableMetadata = Collections.<Mutation>singletonList(put);
                 MetaDataMutationResult result = mdProxy.updateIndexState(tableMetadata);
                 if (result.getMutationCode() != MutationCode.TABLE_ALREADY_EXISTS) {
                     LOG.warn("Attempt to disable index " + indexTableName + " failed with code = " + result.getMutationCode() + ". Will use default failure policy instead.");
                     super.handleFailure(attempted, cause);
+                    throw new DoNotRetryIOException("Attemp to writes to " + indexTableName + " failed.", cause);
                 }
-                LOG.info("Successfully disabled index " + indexTableName);
-                buf.append(indexTableName);
-                buf.append(',');
+                LOG.info("Successfully disabled index " + indexTableName + " due to an exception while writing updates.", cause);
             }
-            buf.setLength(buf.length()-1);
-            buf.append(" due to an exception while writing updates");
         } catch (Throwable t) {
             super.handleFailure(attempted, cause);
         }
-        throw new DoNotRetryIOException(buf.toString(), cause);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 310ec03..987169a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -109,6 +109,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.pho
     public static final String SYSTEM_CATALOG_ALIAS = "\"SYSTEM.TABLE\"";
 
     public static final String TABLE_NAME = "TABLE_NAME";
+    public static final byte[] TABLE_NAME_BYTES = Bytes.toBytes(TABLE_NAME);
     public static final String TABLE_TYPE = "TABLE_TYPE";
     public static final byte[] TABLE_TYPE_BYTES = Bytes.toBytes(TABLE_TYPE);
     
@@ -218,6 +219,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.pho
     		
     public static final String TYPE_ID = "TYPE_ID";
     public static final String INDEX_TYPE = "INDEX_TYPE";
+    public static final String INDEX_DISABLE_TIMESTAMP = "INDEX_DISABLE_TIMESTAMP";
+    public static final byte[] INDEX_DISABLE_TIMESTAMP_BYTES = Bytes.toBytes(INDEX_DISABLE_TIMESTAMP);
     
     private final PhoenixConnection connection;
     private final ResultSet emptyResultSet;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
index 6e2b0c7..6d398f6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
@@ -182,7 +182,7 @@ public class CsvBulkLoadTool extends Configured implements Tool {
 	private int loadData(Configuration conf, CommandLine cmdLine,
 			Connection conn) throws SQLException, InterruptedException,
 			ExecutionException {
-		String tableName = cmdLine.getOptionValue(TABLE_NAME_OPT.getOpt());
+		    String tableName = cmdLine.getOptionValue(TABLE_NAME_OPT.getOpt());
         String schemaName = cmdLine.getOptionValue(SCHEMA_NAME_OPT.getOpt());
         String indexTableName = cmdLine.getOptionValue(INDEX_TABLE_NAME_OPT.getOpt());
         String qualifiedTableName = getQualifiedTableName(schemaName, tableName);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index e6c99fe..ec6193e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -26,6 +26,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAM
 import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA;
 
 import java.io.IOException;
+import java.sql.Connection;
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.List;
@@ -122,7 +123,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     private static final Logger logger = LoggerFactory.getLogger(ConnectionQueryServicesImpl.class);
     private static final int INITIAL_CHILD_SERVICES_CAPACITY = 100;
     private static final int DEFAULT_OUT_OF_ORDER_MUTATIONS_WAIT_TIME_MS = 1000;
-    
     protected final Configuration config;
     // Copy of config.getProps(), but read-only to prevent synchronization that we
     // don't need.
@@ -1218,7 +1218,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     }
 
     // Keeping this to use for further upgrades
-    protected PhoenixConnection addColumnsIfNotExists(PhoenixConnection oldMetaConnection, String tableName, long timestamp, String columns) throws SQLException {
+    protected PhoenixConnection addColumnsIfNotExists(PhoenixConnection oldMetaConnection, 
+        String tableName, long timestamp, String columns) throws SQLException {
+
         Properties props = new Properties(oldMetaConnection.getClientInfo());
         props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp));
         // Cannot go through DriverManager or you end up in an infinite loop because it'll call init again
@@ -1227,6 +1229,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         try {
             metaConnection.createStatement().executeUpdate("ALTER TABLE " + tableName + " ADD IF NOT EXISTS " + columns );
         } catch (SQLException e) {
+            logger.warn("addColumnsIfNotExists failed due to:" + e);
             sqlE = e;
         } finally {
             try {
@@ -1289,7 +1292,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                             } catch (TableAlreadyExistsException ignore) {
                                 // This will occur if we have an older SYSTEM.CATALOG and we need to update it to include
                                 // any new columns we've added.
-                                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP, PhoenixDatabaseMetaData.INDEX_TYPE + " " + PDataType.UNSIGNED_TINYINT.getSqlTypeName());
+                                metaConnection = addColumnsIfNotExists(metaConnection, 
+                                  PhoenixDatabaseMetaData.SYSTEM_CATALOG, 
+                                  MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP, 
+                                  PhoenixDatabaseMetaData.INDEX_TYPE + " " + PDataType.UNSIGNED_TINYINT.getSqlTypeName() + 
+                                  ", " + PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP + " " + PDataType.LONG.getSqlTypeName());
                             }
                             try {
                                 metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_SEQUENCE_METADATA);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 107b03f..1e128c2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -77,7 +77,8 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
-
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
 import java.math.BigDecimal;
 
 import org.apache.hadoop.hbase.HConstants;
@@ -211,6 +212,8 @@ public interface QueryConstants {
             SCOPE_TABLE + " VARCHAR," +
             SOURCE_DATA_TYPE + " SMALLINT," +
             IS_AUTOINCREMENT + " VARCHAR," +
+            INDEX_TYPE + " UNSIGNED_TINYINT," +
+            INDEX_DISABLE_TIMESTAMP + " BIGINT," +
             "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
             + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" +
             HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 639eaa5..3e0e461 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -113,6 +113,17 @@ public interface QueryServices extends SQLCloseable {
     // The following config settings is to deal with SYSTEM.CATALOG moves(PHOENIX-916) among region servers
     public static final String CLOCK_SKEW_INTERVAL_ATTRIB = "phoenix.clock.skew.interval";
     
+    // A master switch if to enable auto rebuild an index which failed to be updated previously 
+    public static final String INDEX_FAILURE_HANDLING_REBUILD_ATTRIB = "phoenix.index.failure.handling.rebuild";
+    
+    // Time interval to check if there is an index needs to be rebuild
+    public static final String INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB = 
+        "phoenix.index.failure.handling.rebuild.interval";
+    
+    // Index will be partially re-built from index disable time stamp - following overlap time 
+    public static final String INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB = 
+        "phoenix.index.failure.handling.rebuild.overlap.time";
+    
     /**
      * Get executor service used for parallel scans
      */

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 07106a6..99bd7ef 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -123,6 +123,9 @@ public class QueryServicesOptions {
     public static final long DEFAULT_MAX_CLIENT_METADATA_CACHE_SIZE =  1024L*1024L*10L; // 10 Mb
     public static final int DEFAULT_GROUPBY_ESTIMATED_DISTINCT_VALUES = 1000;
     public static final int DEFAULT_CLOCK_SKEW_INTERVAL = 2000;
+    public static final boolean DEFAULT_INDEX_FAILURE_HANDLING_REBUILD = true; // auto rebuild on
+    public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL = 10000; // 10 secs
+    public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME = 300000; // 5 mins
     
     private final Configuration config;
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 53aebf5..39bd21f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -53,10 +53,12 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
 import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
 import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA;
 import static org.apache.phoenix.schema.PDataType.VARCHAR;
 
+import java.io.IOException;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSetMetaData;
@@ -80,6 +82,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.phoenix.compile.ColumnResolver;
@@ -199,12 +202,20 @@ public class MetaDataClient {
             IMMUTABLE_ROWS + 
             ") VALUES (?, ?, ?, ?)";
     private static final String UPDATE_INDEX_STATE =
+        "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + 
+        TENANT_ID + "," +
+        TABLE_SCHEM + "," +
+        TABLE_NAME + "," +
+        INDEX_STATE + 
+        ") VALUES (?, ?, ?, ?)";
+    private static final String UPDATE_INDEX_STATE_TO_ACTIVE =
             "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + 
             TENANT_ID + "," +
             TABLE_SCHEM + "," +
             TABLE_NAME + "," +
-            INDEX_STATE +
-            ") VALUES (?, ?, ?, ?)";
+            INDEX_STATE + "," +
+            INDEX_DISABLE_TIMESTAMP +
+            ") VALUES (?, ?, ?, ?, ?)";
     private static final String INSERT_COLUMN =
         "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + 
         TENANT_ID + "," +
@@ -481,17 +492,24 @@ public class MetaDataClient {
     }
     
     private MutationState buildIndex(PTable index, TableRef dataTableRef) throws SQLException {
+        AlterIndexStatement indexStatement = null;
         boolean wasAutoCommit = connection.getAutoCommit();
         connection.rollback();
         try {
             connection.setAutoCommit(true);
             PostIndexDDLCompiler compiler = new PostIndexDDLCompiler(connection, dataTableRef);
             MutationPlan plan = compiler.compile(index);
+            try {
+                plan.getContext().setScanTimeRange(new TimeRange(dataTableRef.getLowerBoundTimeStamp(), Long.MAX_VALUE));
+            } catch (IOException e) {
+                throw new SQLException(e);
+            }
             MutationState state = connection.getQueryServices().updateData(plan);
-            AlterIndexStatement indexStatement = FACTORY.alterIndex(FACTORY.namedTable(null, 
+            indexStatement = FACTORY.alterIndex(FACTORY.namedTable(null, 
                     TableName.create(index.getSchemaName().getString(), index.getTableName().getString())),
                     dataTableRef.getTable().getTableName().getString(), false, PIndexState.ACTIVE);
             alterIndex(indexStatement);
+            
             return state;
         } finally {
             connection.setAutoCommit(wasAutoCommit);
@@ -499,6 +517,32 @@ public class MetaDataClient {
     }
 
     /**
+     * Rebuild indexes from a timestamp which is the value from hbase row key timestamp field
+     */
+    public void buildPartialIndexFromTimeStamp(PTable index, TableRef dataTableRef) throws SQLException {
+        boolean needRestoreIndexState = false;
+        // Need to change index state from Disable to InActive when build index partially so that
+        // new changes will be indexed during index rebuilding
+        AlterIndexStatement indexStatement = FACTORY.alterIndex(FACTORY.namedTable(null,
+            TableName.create(index.getSchemaName().getString(), index.getTableName().getString())),
+            dataTableRef.getTable().getTableName().getString(), false, PIndexState.INACTIVE);
+        alterIndex(indexStatement);
+        needRestoreIndexState = true;
+        try {
+            buildIndex(index, dataTableRef);
+            needRestoreIndexState = false;
+        } finally {
+            if(needRestoreIndexState) {
+                // reset index state to disable
+                indexStatement = FACTORY.alterIndex(FACTORY.namedTable(null,
+                    TableName.create(index.getSchemaName().getString(), index.getTableName().getString())),
+                    dataTableRef.getTable().getTableName().getString(), false, PIndexState.DISABLE);
+                alterIndex(indexStatement);
+            }
+        }
+    }
+
+    /**
      * Create an index table by morphing the CreateIndexStatement into a CreateTableStatement and calling
      * MetaDataClient.createTable. In doing so, we perform the following translations:
      * 1) Change the type of any columns being indexed to types that support null if the column is nullable.
@@ -1956,11 +2000,18 @@ public class MetaDataClient {
             TableRef indexRef = FromCompiler.getResolverForMutation(statement, connection).getTables().get(0);
             PreparedStatement tableUpsert = null;
             try {
-                tableUpsert = connection.prepareStatement(UPDATE_INDEX_STATE);
+                if(newIndexState == PIndexState.ACTIVE){
+                    tableUpsert = connection.prepareStatement(UPDATE_INDEX_STATE_TO_ACTIVE);
+                } else {
+                    tableUpsert = connection.prepareStatement(UPDATE_INDEX_STATE);
+                }
                 tableUpsert.setString(1, connection.getTenantId() == null ? null : connection.getTenantId().getString());
                 tableUpsert.setString(2, schemaName);
                 tableUpsert.setString(3, indexName);
                 tableUpsert.setString(4, newIndexState.getSerializedValue());
+                if(newIndexState == PIndexState.ACTIVE){
+                    tableUpsert.setLong(5, 0);
+                }
                 tableUpsert.execute();
             } finally {
                 if(tableUpsert != null) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
index 6019eb8..e820bb3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
@@ -24,7 +24,8 @@ import org.apache.hadoop.hbase.HConstants;
 public final class TableRef {
     private final PTable table;
     private final String alias;
-    private final long timeStamp;
+    private final long upperBoundTimeStamp;
+    private final long lowerBoundTimeStamp;
     private final boolean hasDynamicCols;
 
     public TableRef(TableRef tableRef, long timeStamp) {
@@ -34,11 +35,21 @@ public final class TableRef {
     public TableRef(PTable table) {
         this(null, table, HConstants.LATEST_TIMESTAMP, false);
     }
+    
+    public TableRef(PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp) {
+        this(null, table, upperBoundTimeStamp, lowerBoundTimeStamp, false);
+    }
 
-    public TableRef(String alias, PTable table, long timeStamp, boolean hasDynamicCols) {
+    public TableRef(String alias, PTable table, long upperBoundTimeStamp, boolean hasDynamicCols) {
+        this(alias, table, upperBoundTimeStamp, 0, hasDynamicCols);
+    }
+    
+    public TableRef(String alias, PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp, 
+        boolean hasDynamicCols) {
         this.alias = alias;
         this.table = table;
-        this.timeStamp = timeStamp;
+        this.upperBoundTimeStamp = upperBoundTimeStamp;
+        this.lowerBoundTimeStamp = lowerBoundTimeStamp;
         this.hasDynamicCols = hasDynamicCols;
     }
     
@@ -70,7 +81,11 @@ public final class TableRef {
     }
 
     public long getTimeStamp() {
-        return timeStamp;
+        return this.upperBoundTimeStamp;
+    }
+    
+    public long getLowerBoundTimeStamp() {
+        return this.lowerBoundTimeStamp;
     }
 
     public boolean hasDynamicCols() {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ddf970a3/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index 438477b..d909be3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -19,16 +19,27 @@ package org.apache.phoenix.util;
 
 import static org.apache.phoenix.util.SchemaUtil.getVarChars;
 
+import java.io.IOException;
 import java.sql.SQLException;
 import java.util.Collection;
 import java.util.List;
 
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.ipc.RemoteException;
 import org.apache.phoenix.coprocessor.MetaDataProtocol;
 import org.apache.phoenix.hbase.index.util.ClientKeyValue;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
@@ -39,13 +50,20 @@ import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.PDataType;
 import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.SequenceKey;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.TableNotFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.protobuf.ServiceException;
 
 
 public class MetaDataUtil {
+    private static final Logger logger = LoggerFactory.getLogger(MetaDataUtil.class);
+  
     public static final String VIEW_INDEX_TABLE_PREFIX = "_IDX_";
     public static final byte[] VIEW_INDEX_TABLE_PREFIX_BYTES = Bytes.toBytes(VIEW_INDEX_TABLE_PREFIX);
     public static final String VIEW_INDEX_SEQUENCE_PREFIX = "_SEQ_";
@@ -280,6 +298,40 @@ public class MetaDataUtil {
                 PhoenixDatabaseMetaData.SEQUENCE_SCHEMA + " = '" + key.getSchemaName() + "'");
         
     }
+    
+    /**
+     * This function checks if all regions of a table is online
+     * @param table
+     * @return true when all regions of a table are online
+     * @throws IOException
+     * @throws
+     */
+    public static boolean tableRegionsOnline(Configuration conf, PTable table) {
+        HConnection hcon = null;
+
+        try {
+            hcon = HConnectionManager.getConnection(conf);
+            List<HRegionLocation> locations = hcon.locateRegions(table.getTableName().getBytes());
+
+            for (HRegionLocation loc : locations) {
+                HRegionInterface server = hcon.getHRegionConnection(
+                    loc.getHostname(), loc.getPort());
+                server.getRegionInfo(loc.getRegionInfo().getRegionName());
+            }
+        } catch (IOException ex) {
+            logger.warn("tableRegionsOnline failed due to:" + ex);
+            return false;
+        } finally {
+            if (hcon != null) {
+                try {
+                    hcon.close();
+                } catch (IOException ignored) {
+                }
+            }
+        }
+
+        return true;
+    }
 
     public static final String IS_VIEW_INDEX_TABLE_PROP_NAME = "IS_VIEW_INDEX_TABLE";
     public static final byte[] IS_VIEW_INDEX_TABLE_PROP_BYTES = Bytes.toBytes(IS_VIEW_INDEX_TABLE_PROP_NAME);