You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2011/11/21 18:29:33 UTC

svn commit: r1204611 [1/2] - in /hbase/trunk: ./ src/main/java/org/apache/hadoop/hbase/ src/main/java/org/apache/hadoop/hbase/executor/ src/main/java/org/apache/hadoop/hbase/ipc/ src/main/java/org/apache/hadoop/hbase/master/ src/main/java/org/apache/ha...

Author: tedyu
Date: Mon Nov 21 17:29:32 2011
New Revision: 1204611

URL: http://svn.apache.org/viewvc?rev=1204611&view=rev
Log:
HBASE-4213 Support for fault tolerant, instant schema updates with out master's intervention through ZK

Added:
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/SchemaChangeTracker.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/InstantSchemaChangeTestBase.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChange.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeFailover.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeSplit.java
Modified:
    hbase/trunk/pom.xml
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
    hbase/trunk/src/main/resources/hbase-default.xml
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java

Modified: hbase/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/pom.xml?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/pom.xml (original)
+++ hbase/trunk/pom.xml Mon Nov 21 17:29:32 2011
@@ -338,7 +338,7 @@
 
           <configuration>
             <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
-            <argLine>-enableassertions -Xmx1400m</argLine>
+            <argLine>-enableassertions -Xmx1900m</argLine>
             <redirectTestOutputToFile>true</redirectTestOutputToFile>
           </configuration>
         </plugin>

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java Mon Nov 21 17:29:32 2011
@@ -305,7 +305,9 @@ public class LocalHBaseCluster {
    */
   public HMaster getActiveMaster() {
     for (JVMClusterUtil.MasterThread mt : masterThreads) {
-      if (mt.getMaster().isActiveMaster()) {
+      // Ensure that the current active master is not stopped.
+      // We don't want to return a stopping master as an active master.
+      if (mt.getMaster().isActiveMaster()  && !mt.getMaster().isStopped()) {
         return mt.getMaster();
       }
     }
@@ -449,4 +451,4 @@ public class LocalHBaseCluster {
     admin.createTable(htd);
     cluster.shutdown();
   }
-}
\ No newline at end of file
+}

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java Mon Nov 21 17:29:32 2011
@@ -141,13 +141,12 @@ public abstract class EventHandler imple
      * Constructor
      */
     EventType(int value) {}
-    public boolean isOnlineSchemaChangeSupported() {
+    public boolean isSchemaChangeEvent() {
       return (
-        this.equals(EventType.C_M_ADD_FAMILY) ||
-        this.equals(EventType.C_M_DELETE_FAMILY) ||
-        this.equals(EventType.C_M_MODIFY_FAMILY) ||
-        this.equals(EventType.C_M_MODIFY_TABLE)
-      );
+          this.equals(EventType.C_M_ADD_FAMILY) ||
+          this.equals(EventType.C_M_DELETE_FAMILY) ||
+          this.equals(EventType.C_M_MODIFY_FAMILY) ||
+          this.equals(EventType.C_M_MODIFY_TABLE));
     }
   }
 

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java Mon Nov 21 17:29:32 2011
@@ -245,17 +245,17 @@ public interface HMasterInterface extend
   public HTableDescriptor[] getHTableDescriptors();
 
   /**
-   * Get current HTD for a given tablename
-   * @param tableName
-   * @return HTableDescriptor for the table
-   */
-  //public HTableDescriptor getHTableDescriptor(final byte[] tableName);
-
-  /**
    * Get array of HTDs for requested tables.
    * @param tableNames
    * @return array of HTableDescriptor
    */
   public HTableDescriptor[] getHTableDescriptors(List<String> tableNames);
 
+  /**
+   * Returns the current running status of load balancer.
+   * @return True if LoadBalancer is running now else False.
+   */
+  public boolean isLoadBalancerRunning();
+
+
 }

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Mon Nov 21 17:29:32 2011
@@ -272,10 +272,9 @@ public class AssignmentManager extends Z
    * @param tableName
    * @return Pair indicating the status of the alter command
    * @throws IOException
-   * @throws InterruptedException 
    */
   public Pair<Integer, Integer> getReopenStatus(byte[] tableName)
-  throws IOException, InterruptedException {
+  throws IOException {
     List <HRegionInfo> hris =
       MetaReader.getTableRegions(this.master.getCatalogTracker(), tableName);
     Integer pending = 0;

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Mon Nov 21 17:29:32 2011
@@ -27,8 +27,8 @@ import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.client.Me
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -72,7 +73,6 @@ import org.apache.hadoop.hbase.master.ha
 import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
 import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
-import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
@@ -88,8 +88,9 @@ import org.apache.hadoop.hbase.util.Thre
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.hbase.zookeeper.ClusterId;
 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
-import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
+import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
+import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.Text;
@@ -158,7 +159,10 @@ implements HMasterInterface, HMasterRegi
   private CatalogTracker catalogTracker;
   // Cluster status zk tracker and local setter
   private ClusterStatusTracker clusterStatusTracker;
-  
+
+  // Schema change tracker
+  private MasterSchemaChangeTracker schemaChangeTracker;
+
   // buffer for "fatal error" notices from region servers
   // in the cluster. This is only used for assisting
   // operations/debugging.
@@ -184,11 +188,18 @@ implements HMasterInterface, HMasterRegi
 
   private CatalogJanitor catalogJanitorChore;
   private LogCleaner logCleaner;
+  private Thread schemaJanitorChore;
 
   private MasterCoprocessorHost cpHost;
   private final ServerName serverName;
 
   private TableDescriptors tableDescriptors;
+
+  // Whether or not schema alter changes go through ZK or not.
+  private boolean supportInstantSchemaChanges = false;
+
+  private volatile boolean loadBalancerRunning = false;
+
   
   /**
    * Initializes the HMaster. The steps are as follows:
@@ -254,6 +265,17 @@ implements HMasterInterface, HMasterRegi
     this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(), this, true);
     this.rpcServer.startThreads();
     this.metrics = new MasterMetrics(getServerName().toString());
+    // initialize instant schema change settings
+    this.supportInstantSchemaChanges = conf.getBoolean(
+        "hbase.instant.schema.alter.enabled", false);
+    if (supportInstantSchemaChanges) {
+      LOG.info("Instant schema change enabled. All schema alter operations will " +
+          "happen through ZK.");
+    }
+   else {
+      LOG.info("Instant schema change disabled. All schema alter operations will " +
+          "happen normally.");
+    }
   }
 
   /**
@@ -386,6 +408,12 @@ implements HMasterInterface, HMasterRegi
     boolean wasUp = this.clusterStatusTracker.isClusterUp();
     if (!wasUp) this.clusterStatusTracker.setClusterUp();
 
+    // initialize schema change tracker
+    this.schemaChangeTracker = new MasterSchemaChangeTracker(getZooKeeper(),
+        this, this,
+        conf.getInt("hbase.instant.schema.alter.timeout", 60000));
+    this.schemaChangeTracker.start();
+
     LOG.info("Server active/primary master; " + this.serverName +
         ", sessionid=0x" +
         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
@@ -497,6 +525,9 @@ implements HMasterInterface, HMasterRegi
     this.catalogJanitorChore = new CatalogJanitor(this, this);
     Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());
 
+    // Schema janitor chore.
+    this.schemaJanitorChore = getAndStartSchemaJanitorChore(this);
+
     status.markComplete("Initialization successful");
     LOG.info("Master has completed initialization");
     initialized = true;
@@ -622,6 +653,15 @@ implements HMasterInterface, HMasterRegi
     return this.tableDescriptors;
   }
 
+  @Override
+  public MasterSchemaChangeTracker getSchemaChangeTracker() {
+    return this.schemaChangeTracker;
+  }
+
+  public RegionServerTracker getRegionServerTracker() {
+    return this.regionServerTracker;
+  }
+
   /** @return InfoServer object. Maybe null.*/
   public InfoServer getInfoServer() {
     return this.infoServer;
@@ -724,7 +764,28 @@ implements HMasterInterface, HMasterRegi
     if (this.executorService != null) this.executorService.shutdown();
   }
 
-  private static Thread getAndStartBalancerChore(final HMaster master) {
+  /**
+   * Start the schema janitor. This Janitor will periodically sweep the failed/expired schema
+   * changes.
+   * @param master
+   * @return
+   */
+  private Thread getAndStartSchemaJanitorChore(final HMaster master) {
+    String name = master.getServerName() + "-SchemaJanitorChore";
+    int schemaJanitorPeriod =
+      master.getConfiguration().getInt("hbase.instant.schema.janitor.period", 120000);
+    // Start up the schema janitor chore
+    Chore chore = new Chore(name, schemaJanitorPeriod, master) {
+      @Override
+      protected void chore() {
+        master.getSchemaChangeTracker().handleFailedOrExpiredSchemaChanges();
+      }
+    };
+    return Threads.setDaemonThreadRunning(chore.getThread());
+  }
+
+
+  private Thread getAndStartBalancerChore(final HMaster master) {
     String name = master.getServerName() + "-BalancerChore";
     int balancerPeriod =
       master.getConfiguration().getInt("hbase.balancer.period", 300000);
@@ -745,6 +806,10 @@ implements HMasterInterface, HMasterRegi
     if (this.catalogJanitorChore != null) {
       this.catalogJanitorChore.interrupt();
     }
+    if (this.schemaJanitorChore != null) {
+      this.schemaJanitorChore.interrupt();
+    }
+
   }
 
   @Override
@@ -815,6 +880,15 @@ implements HMasterInterface, HMasterRegi
     return balancerCutoffTime;
   }
 
+
+  /**
+   * Check whether the Load Balancer is currently running.
+   * @return true if the Load balancer is currently running.
+   */
+  public boolean isLoadBalancerRunning() {
+    return loadBalancerRunning;
+  }
+
   @Override
   public boolean balance() {
     // If balance not true, don't run balancer.
@@ -822,23 +896,33 @@ implements HMasterInterface, HMasterRegi
     // Do this call outside of synchronized block.
     int maximumBalanceTime = getBalancerCutoffTime();
     long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
-    boolean balancerRan;
+    boolean balancerRan = false;
     synchronized (this.balancer) {
+      if (loadBalancerRunning) {
+        LOG.debug("Load balancer is currently running. Skipping the current execution.");
+        return false;
+      }
+
       // Only allow one balance run at at time.
       if (this.assignmentManager.isRegionsInTransition()) {
         LOG.debug("Not running balancer because " +
-          this.assignmentManager.getRegionsInTransition().size() +
-          " region(s) in transition: " +
-          org.apache.commons.lang.StringUtils.
+            this.assignmentManager.getRegionsInTransition().size() +
+            " region(s) in transition: " +
+            org.apache.commons.lang.StringUtils.
             abbreviate(this.assignmentManager.getRegionsInTransition().toString(), 256));
         return false;
       }
       if (this.serverManager.areDeadServersInProgress()) {
         LOG.debug("Not running balancer because processing dead regionserver(s): " +
-          this.serverManager.getDeadServers());
+            this.serverManager.getDeadServers());
         return false;
       }
-
+      if (schemaChangeTracker.isSchemaChangeInProgress()) {
+        LOG.debug("Schema change operation is in progress. Waiting for " +
+        "it to complete before running the load balancer.");
+        return false;
+      }
+      loadBalancerRunning = true;
       if (this.cpHost != null) {
         try {
           if (this.cpHost.preBalance()) {
@@ -855,13 +939,13 @@ implements HMasterInterface, HMasterRegi
         this.assignmentManager.getAssignments();
       // Returned Map from AM does not include mention of servers w/o assignments.
       for (Map.Entry<ServerName, HServerLoad> e:
-          this.serverManager.getOnlineServers().entrySet()) {
+        this.serverManager.getOnlineServers().entrySet()) {
         if (!assignments.containsKey(e.getKey())) {
           assignments.put(e.getKey(), new ArrayList<HRegionInfo>());
         }
       }
       List<RegionPlan> plans = this.balancer.balanceCluster(assignments);
-      int rpCount = 0;	// number of RegionPlans balanced so far
+      int rpCount = 0;  // number of RegionPlans balanced so far
       long totalRegPlanExecTime = 0;
       balancerRan = plans != null;
       if (plans != null && !plans.isEmpty()) {
@@ -875,7 +959,7 @@ implements HMasterInterface, HMasterRegi
               // if performing next balance exceeds cutoff time, exit the loop
               (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
             LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
-              maximumBalanceTime);
+                maximumBalanceTime);
             break;
           }
         }
@@ -888,6 +972,7 @@ implements HMasterInterface, HMasterRegi
           LOG.error("Error invoking master coprocessor postBalance()", ioe);
         }
       }
+      loadBalancerRunning = false;
     }
     return balancerRan;
   }
@@ -1036,22 +1121,63 @@ implements HMasterInterface, HMasterRegi
     if (cpHost != null) {
       cpHost.preDeleteTable(tableName);
     }
-    this.executorService.submit(new DeleteTableHandler(tableName, this, this));
+    this.executorService.submit(new DeleteTableHandler(tableName, this, this, this,
+        supportInstantSchemaChanges));
 
     if (cpHost != null) {
       cpHost.postDeleteTable(tableName);
     }
   }
 
+  /**
+   * Get the number of regions of the table that have been updated by the alter.
+   *
+   * @return Pair indicating the number of regions updated Pair.getFirst is the
+   *         regions that are yet to be updated Pair.getSecond is the total number
+   *         of regions of the table
+   */
   public Pair<Integer, Integer> getAlterStatus(byte[] tableName)
   throws IOException {
+    if (supportInstantSchemaChanges) {
+      return getAlterStatusFromSchemaChangeTracker(tableName);
+    }
+    return this.assignmentManager.getReopenStatus(tableName);
+  }
+
+  /**
+   * Used by the client to identify if all regions have the schema updates
+   *
+   * @param tableName
+   * @return Pair indicating the status of the alter command
+   * @throws IOException
+   */
+  private Pair<Integer, Integer> getAlterStatusFromSchemaChangeTracker(byte[] tableName)
+      throws IOException {
+    MasterSchemaChangeTracker.MasterAlterStatus alterStatus = null;
     try {
-      return this.assignmentManager.getReopenStatus(tableName);
-    } catch (InterruptedException e) {
-      throw new IOException("Interrupted", e);
+      alterStatus =
+          this.schemaChangeTracker.getMasterAlterStatus(Bytes.toString(tableName));
+    } catch (KeeperException ke) {
+      LOG.error("KeeperException while getting schema alter status for table = "
+      + Bytes.toString(tableName), ke);
+    }
+    if (alterStatus != null) {
+      LOG.debug("Getting AlterStatus from SchemaChangeTracker for table = "
+          + Bytes.toString(tableName) + " Alter Status = "
+          + alterStatus.toString());
+      int numberPending = alterStatus.getNumberOfRegionsToProcess() -
+          alterStatus.getNumberOfRegionsProcessed();
+      return new Pair<Integer, Integer>(alterStatus.getNumberOfRegionsProcessed(),
+          alterStatus.getNumberOfRegionsToProcess());
+    } else {
+      LOG.debug("MasterAlterStatus is NULL for table = "
+          + Bytes.toString(tableName));
+      // should we throw IOException here as it makes more sense?
+      return new Pair<Integer, Integer>(0,0);
     }
   }
 
+
   public void addColumn(byte [] tableName, HColumnDescriptor column)
   throws IOException {
     if (cpHost != null) {
@@ -1059,7 +1185,8 @@ implements HMasterInterface, HMasterRegi
         return;
       }
     }
-    new TableAddFamilyHandler(tableName, column, this, this).process();
+    new TableAddFamilyHandler(tableName, column, this, this,
+        this, supportInstantSchemaChanges).process();
     if (cpHost != null) {
       cpHost.postAddColumn(tableName, column);
     }
@@ -1072,7 +1199,8 @@ implements HMasterInterface, HMasterRegi
         return;
       }
     }
-    new TableModifyFamilyHandler(tableName, descriptor, this, this).process();
+    new TableModifyFamilyHandler(tableName, descriptor, this, this,
+        this, supportInstantSchemaChanges).process();
     if (cpHost != null) {
       cpHost.postModifyColumn(tableName, descriptor);
     }
@@ -1085,7 +1213,8 @@ implements HMasterInterface, HMasterRegi
         return;
       }
     }
-    new TableDeleteFamilyHandler(tableName, c, this, this).process();
+    new TableDeleteFamilyHandler(tableName, c, this, this,
+        this, supportInstantSchemaChanges).process();
     if (cpHost != null) {
       cpHost.postDeleteColumn(tableName, c);
     }
@@ -1096,7 +1225,7 @@ implements HMasterInterface, HMasterRegi
       cpHost.preEnableTable(tableName);
     }
     this.executorService.submit(new EnableTableHandler(this, tableName,
-      catalogTracker, assignmentManager, false));
+        catalogTracker, assignmentManager, false));
 
     if (cpHost != null) {
       cpHost.postEnableTable(tableName);
@@ -1152,21 +1281,37 @@ implements HMasterInterface, HMasterRegi
 
   @Override
   public void modifyTable(final byte[] tableName, HTableDescriptor htd)
-  throws IOException {
+      throws IOException {
     if (cpHost != null) {
       cpHost.preModifyTable(tableName, htd);
     }
-
     this.executorService.submit(new ModifyTableHandler(tableName, htd, this,
-      this));
-
+      this, this, supportInstantSchemaChanges));
     if (cpHost != null) {
       cpHost.postModifyTable(tableName, htd);
     }
   }
 
+  private boolean isOnlineSchemaChangeAllowed() {
+    return conf.getBoolean(
+        "hbase.online.schema.update.enable", false);
+  }
+  
   @Override
-  public void checkTableModifiable(final byte [] tableName)
+  public void checkTableModifiable(final byte [] tableName,
+                                   EventHandler.EventType eventType)
+  throws IOException {
+    preCheckTableModifiable(tableName);
+    if (!eventType.isSchemaChangeEvent() ||
+        !isOnlineSchemaChangeAllowed()) {
+      if (!getAssignmentManager().getZKTable().
+          isDisabledTable(Bytes.toString(tableName))) {
+        throw new TableNotDisabledException(tableName);
+      }
+    }
+  }
+
+  private void preCheckTableModifiable(final byte[] tableName)
   throws IOException {
     String tableNameStr = Bytes.toString(tableName);
     if (isCatalogTable(tableName)) {
@@ -1175,10 +1320,6 @@ implements HMasterInterface, HMasterRegi
     if (!MetaReader.tableExists(getCatalogTracker(), tableNameStr)) {
       throw new TableNotFoundException(tableNameStr);
     }
-    if (!getAssignmentManager().getZKTable().
-        isDisabledTable(Bytes.toString(tableName))) {
-      throw new TableNotDisabledException(tableName);
-    }
   }
 
   public void clearFromTransition(HRegionInfo hri) {

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Mon Nov 21 17:29:32 2011
@@ -504,7 +504,7 @@ public class MasterFileSystem {
    */
   public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
       throws IOException {
-    LOG.info("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
+    LOG.debug("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
       hcd.toString());
     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
     if (htd == null) {

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java Mon Nov 21 17:29:32 2011
@@ -26,7 +26,10 @@ import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
+import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
 
 /**
  * Services Master supplies
@@ -53,12 +56,15 @@ public interface MasterServices extends 
   public ExecutorService getExecutorService();
 
   /**
-   * Check table is modifiable; i.e. exists and is offline.
-   * @param tableName Name of table to check.
-   * @throws TableNotDisabledException
-   * @throws TableNotFoundException 
+   * Check table modifiable. i.e not ROOT or META and offlined for all commands except
+   * alter commands
+   * @param tableName
+   * @param eventType
+   * @throws IOException
    */
-  public void checkTableModifiable(final byte [] tableName) throws IOException;
+  public void checkTableModifiable(final byte [] tableName,
+                                   EventHandler.EventType eventType)
+      throws IOException;
 
   /**
    * Create a table using the given table definition.
@@ -73,4 +79,17 @@ public interface MasterServices extends 
    * @return Return table descriptors implementation.
    */
   public TableDescriptors getTableDescriptors();
+
+  /**
+   * Get Master Schema change tracker
+   * @return
+   */
+  public MasterSchemaChangeTracker getSchemaChangeTracker();
+
+  /**
+   * Return the Region server tracker.
+   * @return RegionServerTracker
+   */
+  public RegionServerTracker getRegionServerTracker();
+
 }

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Mon Nov 21 17:29:32 2011
@@ -333,11 +333,21 @@ public class ServerManager {
     }
   }
 
+  /**
+   * Exclude a RS from any pending schema change process.
+   * @param serverName
+   */
+  private void excludeRegionServerFromSchemaChanges(final ServerName serverName) {
+    this.services.getSchemaChangeTracker()
+        .excludeRegionServerForSchemaChanges(serverName.getServerName());
+  }
+
   /*
    * Expire the passed server.  Add it to list of deadservers and queue a
    * shutdown processing.
    */
   public synchronized void expireServer(final ServerName serverName) {
+    excludeRegionServerFromSchemaChanges(serverName);
     if (!this.onlineServers.containsKey(serverName)) {
       LOG.warn("Received expiration of " + serverName +
         " but server is not currently online");

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java Mon Nov 21 17:29:32 2011
@@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -37,9 +38,11 @@ public class DeleteTableHandler extends 
   private static final Log LOG = LogFactory.getLog(DeleteTableHandler.class);
 
   public DeleteTableHandler(byte [] tableName, Server server,
-      final MasterServices masterServices)
+      final MasterServices masterServices, HMasterInterface masterInterface,
+      boolean instantChange)
   throws IOException {
-    super(EventType.C_M_DELETE_TABLE, tableName, server, masterServices);
+    super(EventType.C_M_DELETE_TABLE, tableName, server, masterServices,
+        masterInterface, instantChange);
     // The next call fails if no such table.
     getTableDescriptor();
   }

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java Mon Nov 21 17:29:32 2011
@@ -25,6 +25,7 @@ import java.util.List;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.MasterServices;
 
 public class ModifyTableHandler extends TableEventHandler {
@@ -32,9 +33,11 @@ public class ModifyTableHandler extends 
 
   public ModifyTableHandler(final byte [] tableName,
       final HTableDescriptor htd, final Server server,
-      final MasterServices masterServices)
+      final MasterServices masterServices, final HMasterInterface masterInterface,
+      boolean instantModify)
   throws IOException {
-    super(EventType.C_M_MODIFY_TABLE, tableName, server, masterServices);
+    super(EventType.C_M_MODIFY_TABLE, tableName, server, masterServices,
+        masterInterface, instantModify);
     // Check table exists.
     getTableDescriptor();
     // This is the new schema we are going to write out as this modification.
@@ -57,4 +60,4 @@ public class ModifyTableHandler extends 
     return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" +
       tableNameStr;
   }
-}
\ No newline at end of file
+}

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java Mon Nov 21 17:29:32 2011
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HRegionIn
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.MasterServices;
 
 /**
@@ -37,8 +38,10 @@ public class TableAddFamilyHandler exten
   private final HColumnDescriptor familyDesc;
 
   public TableAddFamilyHandler(byte[] tableName, HColumnDescriptor familyDesc,
-      Server server, final MasterServices masterServices) throws IOException {
-    super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices);
+      Server server, final MasterServices masterServices,
+      HMasterInterface masterInterface, boolean instantChange) throws IOException {
+    super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices,
+        masterInterface, instantChange);
     HTableDescriptor htd = getTableDescriptor();
     if (htd.hasFamily(familyDesc.getName())) {
       throw new InvalidFamilyOperationException("Family '" +

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java Mon Nov 21 17:29:32 2011
@@ -25,6 +25,7 @@ import java.util.List;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -36,8 +37,10 @@ public class TableDeleteFamilyHandler ex
   private final byte [] familyName;
 
   public TableDeleteFamilyHandler(byte[] tableName, byte [] familyName,
-      Server server, final MasterServices masterServices) throws IOException {
-    super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices);
+      Server server, final MasterServices masterServices,
+      HMasterInterface masterInterface, boolean instantChange) throws IOException {
+    super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices,
+        masterInterface, instantChange);
     HTableDescriptor htd = getTableDescriptor();
     this.familyName = hasColumnFamily(htd, familyName);
   }

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java Mon Nov 21 17:29:32 2011
@@ -35,13 +35,19 @@ import org.apache.hadoop.hbase.InvalidFa
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.BulkReOpen;
 import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 
 import com.google.common.collect.Lists;
@@ -57,32 +63,22 @@ import com.google.common.collect.Maps;
 public abstract class TableEventHandler extends EventHandler {
   private static final Log LOG = LogFactory.getLog(TableEventHandler.class);
   protected final MasterServices masterServices;
+  protected HMasterInterface master = null;
   protected final byte [] tableName;
   protected final String tableNameStr;
+  protected boolean instantAction = false;
 
   public TableEventHandler(EventType eventType, byte [] tableName, Server server,
-      MasterServices masterServices)
+      MasterServices masterServices, HMasterInterface masterInterface,
+      boolean instantSchemaChange)
   throws IOException {
     super(server, eventType);
     this.masterServices = masterServices;
     this.tableName = tableName;
-    try {
-      this.masterServices.checkTableModifiable(tableName);
-    } catch (TableNotDisabledException ex)  {
-      if (isOnlineSchemaChangeAllowed()
-          && eventType.isOnlineSchemaChangeSupported()) {
-        LOG.debug("Ignoring table not disabled exception " +
-            "for supporting online schema changes.");
-      }	else {
-        throw ex;
-      }
-    }
+    this.masterServices.checkTableModifiable(tableName, eventType);
     this.tableNameStr = Bytes.toString(this.tableName);
-  }
-
-  private boolean isOnlineSchemaChangeAllowed() {
-    return this.server.getConfiguration().getBoolean(
-        "hbase.online.schema.update.enable", false);
+    this.instantAction = instantSchemaChange;
+    this.master = masterInterface;
   }
 
   @Override
@@ -94,16 +90,7 @@ public abstract class TableEventHandler 
         MetaReader.getTableRegions(this.server.getCatalogTracker(),
           tableName);
       handleTableOperation(hris);
-      if (eventType.isOnlineSchemaChangeSupported() && this.masterServices.
-          getAssignmentManager().getZKTable().
-          isEnabledTable(Bytes.toString(tableName))) {
-        if (reOpenAllRegions(hris)) {
-          LOG.info("Completed table operation " + eventType + " on table " +
-              Bytes.toString(tableName));
-        } else {
-          LOG.warn("Error on reopening the regions");
-        }
-      }
+      handleSchemaChanges(hris);
     } catch (IOException e) {
       LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
     } catch (KeeperException e) {
@@ -111,13 +98,47 @@ public abstract class TableEventHandler 
     }
   }
 
+  private void handleSchemaChanges(List<HRegionInfo> regions)
+      throws IOException {
+    if (instantAction && regions != null && !regions.isEmpty()) {
+      handleInstantSchemaChanges(regions);
+    } else {
+      handleRegularSchemaChanges(regions);
+    }
+  }
+
+
+  /**
+   * Perform schema changes only if the table is in enabled state.
+   * @return
+   */
+  private boolean canPerformSchemaChange() {
+    return (eventType.isSchemaChangeEvent() && this.masterServices.
+        getAssignmentManager().getZKTable().
+        isEnabledTable(Bytes.toString(tableName)));
+  }
+
+  private void handleRegularSchemaChanges(List<HRegionInfo> regions)
+      throws IOException {
+    if (canPerformSchemaChange()) {
+      this.masterServices.getAssignmentManager().setRegionsToReopen(regions);
+      if (reOpenAllRegions(regions)) {
+        LOG.info("Completed table operation " + eventType + " on table " +
+            Bytes.toString(tableName));
+      } else {
+        LOG.warn("Error on reopening the regions");
+      }
+    }
+  }
+
   public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
     boolean done = false;
     LOG.info("Bucketing regions by region server...");
     HTable table = new HTable(masterServices.getConfiguration(), tableName);
     TreeMap<ServerName, List<HRegionInfo>> serverToRegions = Maps
         .newTreeMap();
-    NavigableMap<HRegionInfo, ServerName> hriHserverMapping = table.getRegionLocations();
+    NavigableMap<HRegionInfo, ServerName> hriHserverMapping
+        = table.getRegionLocations();
     List<HRegionInfo> reRegions = new ArrayList<HRegionInfo>();
     for (HRegionInfo hri : regions) {
       ServerName rsLocation = hriHserverMapping.get(hri);
@@ -160,6 +181,91 @@ public abstract class TableEventHandler 
   }
 
   /**
+   * Check whether any of the regions from the list of regions is undergoing a split.
+   * We simply check whether there is a unassigned node for any of the region and if so
+   * we return as true.
+   * @param regionInfos
+   * @return
+   */
+  private boolean isSplitInProgress(List<HRegionInfo> regionInfos) {
+    for (HRegionInfo hri : regionInfos) {
+      ZooKeeperWatcher zkw = this.masterServices.getZooKeeper();
+      String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
+      try {
+        if (ZKUtil.checkExists(zkw, node) != -1) {
+          LOG.debug("Region " + hri.getRegionNameAsString() + " is unassigned. Assuming" +
+          " that it is undergoing a split");
+          return true;
+        }
+      } catch (KeeperException ke) {
+        LOG.debug("KeeperException while determining splits in progress.", ke);
+        // Assume no splits happening?
+        return false;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Wait for region split transaction in progress (if any)
+   * @param regions
+   * @param status
+   */
+  private void waitForInflightSplit(List<HRegionInfo> regions, MonitoredTask status) {
+    while (isSplitInProgress(regions)) {
+      try {
+        status.setStatus("Alter Schema is waiting for split region to complete.");
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+      }
+    }
+  }
+
+  protected void handleInstantSchemaChanges(List<HRegionInfo> regions) {
+    if (regions == null || regions.isEmpty()) {
+      LOG.debug("Region size is null or empty. Ignoring alter request.");
+      return;
+    }
+    MonitoredTask status = TaskMonitor.get().createStatus(
+        "Handling alter table request for table = " + tableNameStr);
+    if (canPerformSchemaChange()) {
+      boolean prevBalanceSwitch = false;
+      try {
+        // turn off load balancer synchronously
+        prevBalanceSwitch = master.synchronousBalanceSwitch(false);
+        waitForInflightSplit(regions, status);
+        MasterSchemaChangeTracker masterSchemaChangeTracker =
+          this.masterServices.getSchemaChangeTracker();
+        masterSchemaChangeTracker
+        .createSchemaChangeNode(Bytes.toString(tableName),
+            regions.size());
+        while(!masterSchemaChangeTracker.doesSchemaChangeNodeExists(
+            Bytes.toString(tableName))) {
+          try {
+            Thread.sleep(50);
+          } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+          }
+        }
+        status.markComplete("Created ZK node for handling the alter table request for table = "
+            + tableNameStr);
+      } catch (KeeperException e) {
+        LOG.warn("Instant schema change failed for table " + tableNameStr, e);
+        status.setStatus("Instant schema change failed for table " + tableNameStr
+            + " Cause = " + e.getCause());
+
+      } catch (IOException ioe) {
+        LOG.warn("Instant schema change failed for table " + tableNameStr, ioe);
+        status.setStatus("Instant schema change failed for table " + tableNameStr
+            + " Cause = " + ioe.getCause());
+      } finally {
+        master.synchronousBalanceSwitch(prevBalanceSwitch);
+      }
+    }
+  }
+
+  /**
    * @return Table descriptor for this table
    * @throws TableExistsException
    * @throws FileNotFoundException

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java Mon Nov 21 17:29:32 2011
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HRegionIn
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -38,8 +39,10 @@ public class TableModifyFamilyHandler ex
 
   public TableModifyFamilyHandler(byte[] tableName,
       HColumnDescriptor familyDesc, Server server,
-      final MasterServices masterServices) throws IOException {
-    super(EventType.C_M_MODIFY_FAMILY, tableName, server, masterServices);
+      final MasterServices masterServices,
+      HMasterInterface masterInterface, boolean instantChange) throws IOException {
+    super(EventType.C_M_MODIFY_FAMILY, tableName, server, masterServices,
+        masterInterface, instantChange);
     HTableDescriptor htd = getTableDescriptor();
     hasColumnFamily(htd, familyDesc.getName());
     this.familyDesc = familyDesc;

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Mon Nov 21 17:29:32 2011
@@ -155,12 +155,29 @@ public class CompactSplitThread implemen
     return false;
   }
 
+  /**
+   * Wait for mid-flight schema alter requests. (if any). We don't want to execute a split
+   * when a schema alter is in progress as we end up in an inconsistent state.
+   * @param tableName
+   */
+  private void waitForInflightSchemaChange(String tableName) {
+    while (this.server.getSchemaChangeTracker()
+        .isSchemaChangeInProgress(tableName)) {
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+      }
+    }
+  }
+
   public synchronized void requestSplit(final HRegion r, byte[] midKey) {
     if (midKey == null) {
       LOG.debug("Region " + r.getRegionNameAsString() +
         " not splittable because midkey=null");
       return;
     }
+    waitForInflightSchemaChange(r.getRegionInfo().getTableNameAsString());
     try {
       this.splits.execute(new SplitRequest(r, midKey, this.server));
       if (LOG.isDebugEnabled()) {

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Mon Nov 21 17:29:32 2011
@@ -145,6 +145,7 @@ import org.apache.hadoop.hbase.zookeeper
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.zookeeper.SchemaChangeTracker;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.ipc.RemoteException;
@@ -283,6 +284,9 @@ public class HRegionServer implements HR
   // Cluster Status Tracker
   private ClusterStatusTracker clusterStatusTracker;
 
+  // Schema change Tracker
+  private SchemaChangeTracker schemaChangeTracker;
+
   // Log Splitting Worker
   private SplitLogWorker splitLogWorker;
 
@@ -575,6 +579,11 @@ public class HRegionServer implements HR
     this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf,
       this, this.conf.getInt("hbase.regionserver.catalog.timeout", Integer.MAX_VALUE));
     catalogTracker.start();
+
+    // Schema change tracker
+    this.schemaChangeTracker = new SchemaChangeTracker(this.zooKeeper,
+        this, this);
+    this.schemaChangeTracker.start();
   }
 
   /**
@@ -1670,7 +1679,9 @@ public class HRegionServer implements HR
   protected void join() {
     Threads.shutdown(this.compactionChecker.getThread());
     Threads.shutdown(this.cacheFlusher.getThread());
-    Threads.shutdown(this.hlogRoller.getThread());
+    if (this.hlogRoller != null) {
+      Threads.shutdown(this.hlogRoller.getThread());
+    }
     if (this.compactSplitThread != null) {
       this.compactSplitThread.join();
     }
@@ -2658,9 +2669,26 @@ public class HRegionServer implements HR
     splitRegion(regionInfo, null);
   }
 
+  /**
+   * Wait for mid-flight schema change requests. (if any)
+   * @param tableName
+   */
+  private void waitForSchemaChange(String tableName) {
+    while (schemaChangeTracker.isSchemaChangeInProgress(tableName)) {
+      try {
+        LOG.debug("Schema alter is inprogress for table = " + tableName
+            + " Waiting for alter to complete before a split");
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+      }
+    }
+  }
+
   @Override
   public void splitRegion(HRegionInfo regionInfo, byte[] splitPoint)
       throws NotServingRegionException, IOException {
+    waitForSchemaChange(Bytes.toString(regionInfo.getTableName()));
     checkOpen();
     HRegion region = getRegion(regionInfo.getRegionName());
     region.flushcache();
@@ -3373,6 +3401,60 @@ public class HRegionServer implements HR
     return wal.rollWriter(true);
   }
 
+  /**
+  * Refresh schema changes for given region.
+  * @param hRegion HRegion to refresh
+  * @throws IOException
+  */
+ public void refreshRegion(HRegion hRegion) throws IOException {
+
+   if (hRegion != null) {
+     synchronized (this.onlineRegions) {
+       HRegionInfo regionInfo = hRegion.getRegionInfo();
+       // Close the region
+       hRegion.close();
+       // Remove from online regions
+       removeFromOnlineRegions(regionInfo.getEncodedName());
+       // Get new HTD
+       HTableDescriptor htd = this.tableDescriptors.get(regionInfo.getTableName());
+       LOG.debug("HTD for region = " + regionInfo.getRegionNameAsString()
+           + " Is = " + htd );
+       HRegion region =
+         HRegion.openHRegion(hRegion.getRegionInfo(), htd, hlog, conf,
+             this, null);
+       // Add new region to the onlineRegions
+       addToOnlineRegions(region);
+     }
+   }
+ }
+
+ /**
+  * Gets the online regions of the specified table.
+  * This method looks at the in-memory onlineRegions.  It does not go to <code>.META.</code>.
+  * Only returns <em>online</em> regions.  If a region on this table has been
+  * closed during a disable, etc., it will not be included in the returned list.
+  * So, the returned list may not necessarily be ALL regions in this table, its
+  * all the ONLINE regions in the table.
+  * @param tableName
+  * @return Online regions from <code>tableName</code>
+  */
+  public List<HRegion> getOnlineRegions(byte[] tableName) {
+    List<HRegion> tableRegions = new ArrayList<HRegion>();
+    synchronized (this.onlineRegions) {
+      for (HRegion region: this.onlineRegions.values()) {
+        HRegionInfo regionInfo = region.getRegionInfo();
+        if(Bytes.equals(regionInfo.getTableName(), tableName)) {
+          tableRegions.add(region);
+        }
+      }
+    }
+    return tableRegions;
+  }
+
+  public SchemaChangeTracker getSchemaChangeTracker() {
+    return this.schemaChangeTracker;
+  }
+
   // used by org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon (HBASE-4070).
   public String[] getCoprocessors() {
     HServerLoad hsl = buildServerLoad();

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java Mon Nov 21 17:29:32 2011
@@ -21,6 +21,9 @@ package org.apache.hadoop.hbase.regionse
 
 import org.apache.hadoop.hbase.Server;
 
+import java.io.IOException;
+import java.util.List;
+
 /**
  * Interface to Map of online regions.  In the  Map, the key is the region's
  * encoded name and the value is an {@link HRegion} instance.
@@ -49,4 +52,18 @@ interface OnlineRegions extends Server {
    * null if named region is not member of the online regions.
    */
   public HRegion getFromOnlineRegions(String encodedRegionName);
-}
\ No newline at end of file
+  /**
+   * Get all online regions of a table in this RS.
+   * @param tableName
+   * @return List of HRegion
+   * @throws java.io.IOException
+   */
+  public List<HRegion> getOnlineRegions(byte[] tableName) throws IOException;
+
+  /**
+   * Refresh a given region updating it with latest HTD info.
+   * @param hRegion
+   */
+  public void refreshRegion(HRegion hRegion) throws IOException;
+
+}

Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java?rev=1204611&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java Mon Nov 21 17:29:32 2011
@@ -0,0 +1,826 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.Writable;
+import org.apache.zookeeper.KeeperException;
+
+public class MasterSchemaChangeTracker extends ZooKeeperNodeTracker {
+  public static final Log LOG = LogFactory.getLog(MasterSchemaChangeTracker.class);
+  private final MasterServices masterServices;
+  // Used by tests only. Do not change this.
+  private volatile int sleepTimeMillis = 0;
+  // schema changes pending more than this time will be timed out.
+  private long schemaChangeTimeoutMillis = 30000;
+
+  /**
+   * Constructs a new ZK node tracker.
+   * <p/>
+   * <p>After construction, use {@link #start} to kick off tracking.
+   *
+   * @param watcher
+   * @param abortable
+   */
+  public MasterSchemaChangeTracker(ZooKeeperWatcher watcher,
+                                   Abortable abortable, MasterServices masterServices,
+                                   long schemaChangeTimeoutMillis) {
+    super(watcher, watcher.schemaZNode, abortable);
+    this.masterServices = masterServices;
+    this.schemaChangeTimeoutMillis = schemaChangeTimeoutMillis;
+  }
+
+  @Override
+  public void start() {
+    try {
+      watcher.registerListener(this);
+      List<String> tables =
+          ZKUtil.listChildrenNoWatch(watcher, watcher.schemaZNode);
+      processCompletedSchemaChanges(tables);
+    } catch (KeeperException e) {
+      LOG.error("MasterSchemaChangeTracker startup failed.", e);
+      abortable.abort("MasterSchemaChangeTracker startup failed", e);
+    }
+  }
+
+  private List<String> getCurrentTables() throws KeeperException {
+    return
+        ZKUtil.listChildrenNoWatch(watcher, watcher.schemaZNode);
+  }
+
+  /**
+   * When a primary master crashes and the secondary master takes over
+   * mid-flight during an alter process, the secondary should cleanup any completed
+   * schema changes not handled by the previous master.
+   * @param tables
+   * @throws KeeperException
+   */
+  private void processCompletedSchemaChanges(List<String> tables)
+      throws KeeperException {
+    if (tables == null || tables.isEmpty()) {
+      String msg = "No current schema change in progress. Skipping cleanup";
+      LOG.debug(msg);
+      return;
+    }
+    String msg = "Master seeing following tables undergoing schema change " +
+        "process. Tables = " + tables;
+    MonitoredTask status = TaskMonitor.get().createStatus(msg);
+    LOG.debug(msg);
+    for (String table : tables) {
+      LOG.debug("Processing table = "+ table);
+      status.setStatus("Processing table = "+ table);
+      try {
+        processTableNode(table);
+      } catch (IOException e) {
+        String errmsg = "IOException while processing completed schema changes."
+            + " Cause = " + e.getCause();
+        LOG.error(errmsg, e);
+        status.setStatus(errmsg);
+      }
+    }
+  }
+
+  /**
+   * Get current alter status for a table.
+   * @param tableName
+   * @return MasterAlterStatus
+   * @throws KeeperException
+   * @throws IOException
+   */
+  public MasterAlterStatus getMasterAlterStatus(String tableName)
+      throws KeeperException, IOException {
+    String path = getSchemaChangeNodePathForTable(tableName);
+    byte[] state = ZKUtil.getData(watcher, path);
+    if (state == null || state.length <= 0) {
+      return null;
+    }
+    MasterAlterStatus mas = new MasterAlterStatus();
+    Writables.getWritable(state, mas);
+    return mas;
+  }
+
+  /**
+   * Get RS specific alter status for a table & server
+   * @param tableName
+   * @param serverName
+   * @return Region Server's Schema alter status
+   * @throws KeeperException
+   * @throws IOException
+   */
+  private SchemaChangeTracker.SchemaAlterStatus getRSSchemaAlterStatus(
+      String tableName, String serverName)
+      throws KeeperException, IOException {
+    String childPath =
+        getSchemaChangeNodePathForTableAndServer(tableName, serverName);
+    byte[] childData = ZKUtil.getData(this.watcher, childPath);
+    if (childData == null || childData.length <= 0) {
+      return null;
+    }
+    SchemaChangeTracker.SchemaAlterStatus sas =
+        new SchemaChangeTracker.SchemaAlterStatus();
+    Writables.getWritable(childData, sas);
+    LOG.debug("Schema Status data for server = " + serverName + " table = "
+        + tableName + " == " + sas);
+    return sas;
+  }
+
+  /**
+   * Update the master's alter status based on all region server's response.
+   * @param servers
+   * @param tableName
+   * @throws IOException
+   */
+  private void updateMasterAlterStatus(MasterAlterStatus mas,
+                                       List<String> servers, String tableName)
+      throws IOException, KeeperException {
+    for (String serverName : servers) {
+      SchemaChangeTracker.SchemaAlterStatus sas =
+          getRSSchemaAlterStatus(tableName, serverName);
+      if (sas != null) {
+        mas.update(sas);
+        LOG.debug("processTableNodeWithState:Updated Master Alter Status = "
+            + mas + " for server = " + serverName);
+      } else {
+        LOG.debug("SchemaAlterStatus is NULL for table = " + tableName);
+      }
+    }
+  }
+
+  /**
+   * If schema alter is handled for this table, then delete all the ZK nodes
+   * created for this table.
+   * @param tableName
+   * @throws KeeperException
+   */
+  private void processTableNode(String tableName) throws KeeperException,
+      IOException {
+    LOG.debug("processTableNodeWithState. TableName = " + tableName);
+    List<String> servers =
+      ZKUtil.listChildrenAndWatchThem(watcher,
+          getSchemaChangeNodePathForTable(tableName));
+    MasterAlterStatus mas = getMasterAlterStatus(tableName);
+    if (mas == null) {
+      LOG.debug("MasterAlterStatus is NULL. Table = " + tableName);
+      return;
+    }
+    updateMasterAlterStatus(mas, servers, tableName);
+    LOG.debug("Current Alter status = " + mas);
+    String nodePath = getSchemaChangeNodePathForTable(tableName);
+    ZKUtil.updateExistingNodeData(this.watcher, nodePath,
+        Writables.getBytes(mas), getZKNodeVersion(nodePath));
+    processAlterStatus(mas, tableName, servers);
+  }
+
+  /**
+   * Evaluate the master alter status and determine the current status.
+   * @param alterStatus
+   * @param tableName
+   * @param servers
+   * @param status
+   */
+  private void processAlterStatus(MasterAlterStatus alterStatus,
+                                  String tableName, List<String> servers)
+  throws KeeperException {
+    if (alterStatus.getNumberOfRegionsToProcess()
+        == alterStatus.getNumberOfRegionsProcessed()) {
+      // schema change completed.
+      String msg = "All region servers have successfully processed the " +
+          "schema changes for table = " + tableName
+          + " . Deleting the schema change node for table = "
+          + tableName + " Region servers processed the schema change" +
+          " request = " + alterStatus.getProcessedHosts()
+          + " Total number of regions = " + alterStatus.getNumberOfRegionsToProcess()
+          + " Processed regions = " + alterStatus.getNumberOfRegionsProcessed();
+      MonitoredTask status = TaskMonitor.get().createStatus(
+          "Checking alter schema request status for table = " + tableName);
+      status.markComplete(msg);
+      LOG.debug(msg);
+      cleanProcessedTableNode(getSchemaChangeNodePathForTable(tableName));
+    } else {
+      if (alterStatus.getErrorCause() != null
+          && alterStatus.getErrorCause().trim().length() > 0) {
+        String msg = "Alter schema change failed "
+            + "for table = " + tableName + " Number of online regions = "
+            + alterStatus.getNumberOfRegionsToProcess() + " processed regions count = "
+            + alterStatus.getNumberOfRegionsProcessed()
+            + " Original list = " + alterStatus.hostsToProcess + " Processed servers = "
+            + servers
+            + " Error Cause = " + alterStatus.getErrorCause();
+        MonitoredTask status = TaskMonitor.get().createStatus(
+            "Checking alter schema request status for table = " + tableName);
+        // we have errors.
+        LOG.debug(msg);
+        status.abort(msg);
+      } else {
+        String msg = "Not all region servers have processed the schema changes"
+            + "for table = " + tableName + " Number of online regions = "
+            + alterStatus.getNumberOfRegionsToProcess() + " processed regions count = "
+            + alterStatus.getNumberOfRegionsProcessed()
+            + " Original list = " + alterStatus.hostsToProcess + " Processed servers = "
+            + servers + " Alter STate = "
+            + alterStatus.getCurrentAlterStatus();
+        LOG.debug(msg);
+        // status.setStatus(msg);
+      }
+    }
+  }
+
+  /**
+   * Check whether a in-flight schema change request has expired.
+   * @param tableName
+   * @return true is the schema change request expired.
+   * @throws IOException
+   */
+  private boolean hasSchemaChangeExpiredFor(String tableName)
+      throws IOException, KeeperException {
+    MasterAlterStatus mas = getMasterAlterStatus(tableName);
+    long createdTimeStamp = mas.getStamp();
+    long duration = System.currentTimeMillis() - createdTimeStamp;
+    LOG.debug("Created TimeStamp = " + createdTimeStamp
+        + " duration = " + duration + " Table = " + tableName
+        + " Master Alter Status = " + mas);
+    return (duration > schemaChangeTimeoutMillis);
+  }
+
+  /**
+   * Handle failed and expired schema changes. We simply delete all the
+   * expired/failed schema change attempts. Why we should do this ?
+   * 1) Keeping the failed/expired schema change nodes longer prohibits any
+   *    future schema changes for the table.
+   * 2) Any lingering expired/failed schema change requests will prohibit the
+   *    load balancer from running.
+   */
+  public void handleFailedOrExpiredSchemaChanges() {
+    try {
+      List<String> tables = getCurrentTables();
+      for (String table : tables) {
+        String statmsg = "Cleaning failed or expired schema change requests. " +
+            "current tables undergoing " +
+            "schema change process = " + tables;
+         MonitoredTask status = TaskMonitor.get().createStatus(statmsg);
+        LOG.debug(statmsg);
+        if (hasSchemaChangeExpiredFor(table)) {
+          // time out.. currently, we abandon the in-flight schema change due to
+          // time out.
+          // Here, there are couple of options to consider. One could be to
+          // attempt a retry of the schema change and see if it succeeds, and
+          // another could be to simply rollback the schema change effort and
+          // see if it succeeds.
+          String msg = "Schema change for table = " + table + " has expired."
+              + " Schema change for this table has been in progress for " +
+              + schemaChangeTimeoutMillis +
+              "Deleting the node now.";
+          LOG.debug(msg);
+          ZKUtil.deleteNodeRecursively(this.watcher,
+              getSchemaChangeNodePathForTable(table));
+        } else {
+          String msg = "Schema change request is in progress for " +
+              " table = " + table;
+          LOG.debug(msg);
+          status.setStatus(msg);
+        }
+      }
+    } catch (IOException e) {
+      String msg = "IOException during handleFailedExpiredSchemaChanges."
+            + e.getCause();
+      LOG.error(msg, e);
+      TaskMonitor.get().createStatus(msg);
+    } catch (KeeperException ke) {
+      String msg = "KeeperException during handleFailedExpiredSchemaChanges."
+          + ke.getCause();
+      LOG.error(msg, ke);
+      TaskMonitor.get().createStatus(msg);
+    }
+  }
+
+  /**
+   * Clean the nodes of completed schema change table.
+   * @param path
+   * @throws KeeperException
+   */
+  private void cleanProcessedTableNode(String path) throws KeeperException {
+    if (sleepTimeMillis > 0) {
+      try {
+        LOG.debug("Master schema change tracker sleeping for "
+            + sleepTimeMillis);
+        Thread.sleep(sleepTimeMillis);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+      }
+    }
+    ZKUtil.deleteNodeRecursively(this.watcher, path);
+    LOG.debug("Deleted all nodes for path " + path);
+
+  }
+
+  /**
+   * Exclude a RS from schema change request (if applicable)
+   * We will exclude a RS from schema change request processing if 1) RS
+   * has online regions for the table AND 2) RS went down mid-flight
+   * during schema change process. We don't have to deal with RS going
+   * down mid-flight during a schema change as the online regions from
+   * the dead RS will get reassigned to some other RS and the
+   * process of reassign inherently takes care of the schema change as well.
+   * @param serverName
+   */
+  public void excludeRegionServerForSchemaChanges(String serverName) {
+    try {
+      MonitoredTask status = TaskMonitor.get().createStatus(
+        "Processing schema change exclusion for region server = " + serverName);
+      List<String> tables =
+            ZKUtil.listChildrenNoWatch(watcher, watcher.schemaZNode);
+      if (tables == null || tables.isEmpty()) {
+        String msg = "No schema change in progress. Skipping exclusion for " +
+            "server = "+ serverName;
+        LOG.debug(msg);
+        status.setStatus(msg);
+        return ;
+      }
+      for (String tableName : tables) {
+        excludeRegionServer(tableName, serverName, status);
+      }
+    } catch(KeeperException ke) {
+      LOG.error("KeeperException during excludeRegionServerForSchemaChanges", ke);
+    } catch(IOException ioe) {
+      LOG.error("IOException during excludeRegionServerForSchemaChanges", ioe);
+
+    }
+  }
+
+  /**
+   * Check whether a schema change is in progress for a given table on a
+   * given RS.
+   * @param tableName
+   * @param serverName
+   * @return TRUE is this RS is currently processing a schema change request
+   * for the table.
+   * @throws KeeperException
+   */
+  private boolean isSchemaChangeApplicableFor(String tableName,
+                                              String serverName)
+      throws KeeperException {
+    List<String> servers = ZKUtil.listChildrenAndWatchThem(watcher,
+        getSchemaChangeNodePathForTable(tableName));
+    return (servers.contains(serverName));
+  }
+
+  /**
+   * Exclude a region server for a table (if applicable) from schema change processing.
+   * @param tableName
+   * @param serverName
+   * @param status
+   * @throws KeeperException
+   * @throws IOException
+   */
+  private void excludeRegionServer(String tableName, String serverName,
+                                   MonitoredTask status)
+      throws KeeperException, IOException {
+    if (isSchemaChangeApplicableFor(tableName, serverName)) {
+      String msg = "Excluding RS " + serverName + " from schema change process" +
+          " for table = " + tableName;
+      LOG.debug(msg);
+      status.setStatus(msg);
+      SchemaChangeTracker.SchemaAlterStatus sas =
+          getRSSchemaAlterStatus(tableName, serverName);
+      if (sas == null) {
+        LOG.debug("SchemaAlterStatus is NULL for table = " + tableName
+            + " server = " + serverName);
+        return;
+      }
+      // Set the status to IGNORED so we can process it accordingly.
+      sas.setCurrentAlterStatus(
+          SchemaChangeTracker.SchemaAlterStatus.AlterState.IGNORED);
+      LOG.debug("Updating the current schema status to " + sas);
+      String nodePath = getSchemaChangeNodePathForTableAndServer(tableName,
+              serverName);
+      ZKUtil.updateExistingNodeData(this.watcher,
+          nodePath, Writables.getBytes(sas), getZKNodeVersion(nodePath));
+    } else {
+      LOG.debug("Skipping exclusion of RS " + serverName
+          + " from schema change process"
+          + " for table = " + tableName
+          + " as it did not possess any online regions for the table");
+    }
+    processTableNode(tableName);
+  }
+
+  private int getZKNodeVersion(String nodePath) throws KeeperException {
+    return ZKUtil.checkExists(this.watcher, nodePath);
+  }
+
+  /**
+   * Create a new schema change ZK node.
+   * @param tableName Table name that is getting altered
+   * @throws KeeperException
+   */
+  public void createSchemaChangeNode(String tableName,
+                                     int numberOfRegions)
+      throws KeeperException, IOException {
+    MonitoredTask status = TaskMonitor.get().createStatus(
+        "Creating schema change node for table = " + tableName);
+    LOG.debug("Creating schema change node for table = "
+        + tableName + " Path = "
+        + getSchemaChangeNodePathForTable(tableName));
+    if (doesSchemaChangeNodeExists(tableName)) {
+      LOG.debug("Schema change node already exists for table = " + tableName
+          + " Deleting the schema change node.");
+      // If we already see a schema change node for this table we wait till the previous
+      // alter process is complete. Ideally, we need not wait and we could simply delete
+      // existing schema change node for this table and create new one. But then the
+      // RS cloud will not be able to process concurrent schema updates for the same table
+      // as they will be working with same set of online regions for this table. Meaning the
+      // second alter change will not see any online regions (as they were being closed and
+      // re opened by the first change) and will miss the second one.
+      // We either handle this at the RS level using explicit locks while processing a table
+      // or do it here. I prefer doing it here as it seems much simpler and cleaner.
+      while(doesSchemaChangeNodeExists(tableName)) {
+        try {
+          Thread.sleep(50);
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+        }
+      }
+    }
+
+    int rsCount = ZKUtil.getNumberOfChildren(this.watcher, watcher.rsZNode);
+    // if number of online RS = 0, we should not do anything!
+    if (rsCount <= 0) {
+      String msg = "Master is not seeing any online region servers. Aborting the " +
+          "schema change processing by region servers.";
+      LOG.debug(msg);
+      status.abort(msg);
+    } else {
+      LOG.debug("Master is seeing " + rsCount + " region servers online before " +
+          "the schema change process.");
+      MasterAlterStatus mas = new MasterAlterStatus(numberOfRegions,
+          getActiveRegionServersAsString());
+      LOG.debug("Master creating the master alter status = " + mas);
+      ZKUtil.createSetData(this.watcher,
+          getSchemaChangeNodePathForTable(tableName), Writables.getBytes(mas));
+      status.markComplete("Created the ZK node for schema change. Current Alter Status = "
+          + mas.toString());
+      ZKUtil.listChildrenAndWatchThem(this.watcher,
+          getSchemaChangeNodePathForTable(tableName));
+    }
+  }
+
+  private String getActiveRegionServersAsString() {
+    StringBuffer sbuf = new StringBuffer();
+    List<ServerName> currentRS =
+        masterServices.getRegionServerTracker().getOnlineServers();
+    for (ServerName serverName : currentRS) {
+      sbuf.append(serverName.getServerName());
+      sbuf.append(" ");
+    }
+    LOG.debug("Current list of RS to process the schema change = "
+        + sbuf.toString());
+    return sbuf.toString();
+  }
+
+  /**
+   * Create a new schema change ZK node.
+   * @param tableName
+   * @throws KeeperException
+   */
+  public boolean doesSchemaChangeNodeExists(String tableName)
+      throws KeeperException {
+    return ZKUtil.checkExists(watcher,
+            getSchemaChangeNodePathForTable(tableName)) != -1;
+  }
+
+  /**
+   * Check whether there are any schema change requests that are in progress now.
+   * We simply assume that a schema change is in progress if we see a ZK schema node for
+   * any table. We may revisit for fine grained checks such as check the current alter status
+   * et al, but it is not required now.
+   * @return
+   */
+  public boolean isSchemaChangeInProgress() {
+    try {
+      int schemaChangeCount = ZKUtil.getNumberOfChildren(this.watcher, watcher.schemaZNode);
+      return schemaChangeCount > 0;
+    } catch (KeeperException ke) {
+      LOG.debug("KeeperException while getting current schema change progress.");
+      // What do we do now??? currently reporting as false.
+    }
+    return false;
+  }
+
+  /**
+   * We get notified when a RS processes/or completed the schema change request.
+   * The path will be of the format /hbase/schema/<table name>
+   * @param path full path of the node whose children have changed
+   */
+  @Override
+  public void nodeChildrenChanged(String path) {
+    String tableName = null;
+    if (path.startsWith(watcher.schemaZNode) &&
+        !path.equals(watcher.schemaZNode)) {
+      try {
+        LOG.debug("NodeChildrenChanged Path = " + path);
+        tableName = path.substring(path.lastIndexOf("/")+1, path.length());
+        processTableNode(tableName);
+      } catch (KeeperException e) {
+        TaskMonitor.get().createStatus(
+        "MasterSchemaChangeTracker: ZK exception while processing " +
+            " nodeChildrenChanged() event for table = " + tableName
+            + " Cause = " + e.getCause());
+        LOG.error("MasterSchemaChangeTracker: Unexpected zk exception getting"
+            + " schema change nodes", e);
+      } catch(IOException ioe) {
+        TaskMonitor.get().createStatus(
+        "MasterSchemaChangeTracker: ZK exception while processing " +
+            " nodeChildrenChanged() event for table = " + tableName
+            + " Cause = " + ioe.getCause());
+        LOG.error("MasterSchemaChangeTracker: Unexpected IO exception getting"
+            + " schema change nodes", ioe);
+      }
+    }
+  }
+
+  /**
+   * We get notified as and when the RS cloud updates their ZK nodes with
+   * progress information. The path will be of the format
+   * /hbase/schema/<table name>/<RS host name>
+   * @param path
+   */
+  @Override
+  public void nodeDataChanged(String path) {
+    String tableName = null;
+    if (path.startsWith(watcher.schemaZNode) &&
+        !path.equals(watcher.schemaZNode)) {
+      try {
+        LOG.debug("NodeDataChanged Path = " + path);
+        String[] paths = path.split("/");
+        tableName = paths[3];
+        processTableNode(tableName);
+      } catch (KeeperException e) {
+        TaskMonitor.get().createStatus(
+        "MasterSchemaChangeTracker: ZK exception while processing " +
+            " nodeDataChanged() event for table = " + tableName
+            + " Cause = " + e.getCause());
+        LOG.error("MasterSchemaChangeTracker: Unexpected zk exception getting"
+            + " schema change nodes", e);
+      } catch(IOException ioe) {
+        TaskMonitor.get().createStatus(
+        "MasterSchemaChangeTracker: IO exception while processing " +
+            " nodeDataChanged() event for table = " + tableName
+            + " Cause = " + ioe.getCause());
+        LOG.error("MasterSchemaChangeTracker: Unexpected IO exception getting"
+            + " schema change nodes", ioe);
+
+      }
+    }
+  }
+
+  public String getSchemaChangeNodePathForTable(String tableName) {
+    return ZKUtil.joinZNode(watcher.schemaZNode, tableName);
+  }
+
+  /**
+   * Used only for tests. Do not use this. See TestInstantSchemaChange for more details
+   * on how this is getting used. This is primarily used to delay the schema complete
+   * processing by master so that we can test some complex scenarios such as
+   * master failover.
+   * @param sleepTimeMillis
+   */
+  public void setSleepTimeMillis(int sleepTimeMillis) {
+    this.sleepTimeMillis = sleepTimeMillis;
+  }
+
+  private String getSchemaChangeNodePathForTableAndServer(
+      String tableName, String regionServerName) {
+    return ZKUtil.joinZNode(getSchemaChangeNodePathForTable(tableName),
+        regionServerName);
+  }
+
+
+  /**
+   * Holds the current alter state for a table. Alter state includes the
+   * current alter status (INPROCESS, FAILURE or SUCCESS (success is not getting
+   * used now.), timestamp of alter request, number of hosts online at the time
+   * of alter request, number of online regions to process for the schema change
+   * request, number of processed regions and a list of region servers that
+   * actually processed the schema change request.
+   *
+   * Master keeps track of schema change requests using the alter status and
+   * periodically updates the alter status based on RS cloud processings.
+   */
+  public static class MasterAlterStatus implements Writable {
+
+    public enum AlterState {
+      INPROCESS,        // Inprocess alter
+      SUCCESS,          // completed alter
+      FAILURE           // failure alter
+    }
+
+    private AlterState currentAlterStatus;
+    // TimeStamp
+    private long stamp;
+    private int numberOfRegionsToProcess;
+    private StringBuffer errorCause = new StringBuffer(" ");
+    private StringBuffer processedHosts = new StringBuffer(" ");
+    private String hostsToProcess;
+    private int numberOfRegionsProcessed = 0;
+
+    public MasterAlterStatus() {
+
+    }
+
+    public MasterAlterStatus(int numberOfRegions, String activeHosts) {
+      this.numberOfRegionsToProcess = numberOfRegions;
+      this.stamp = System.currentTimeMillis();
+      this.currentAlterStatus = AlterState.INPROCESS;
+      //this.rsToProcess = activeHosts;
+      this.hostsToProcess = activeHosts;
+    }
+
+    public AlterState getCurrentAlterStatus() {
+      return currentAlterStatus;
+    }
+
+    public void setCurrentAlterStatus(AlterState currentAlterStatus) {
+      this.currentAlterStatus = currentAlterStatus;
+    }
+
+    public long getStamp() {
+      return stamp;
+    }
+
+    public void setStamp(long stamp) {
+      this.stamp = stamp;
+    }
+
+    public int getNumberOfRegionsToProcess() {
+      return numberOfRegionsToProcess;
+    }
+
+    public void setNumberOfRegionsToProcess(int numberOfRegionsToProcess) {
+      this.numberOfRegionsToProcess = numberOfRegionsToProcess;
+    }
+
+    public int getNumberOfRegionsProcessed() {
+      return numberOfRegionsProcessed;
+    }
+
+    public void setNumberOfRegionsProcessed(int numberOfRegionsProcessed) {
+      this.numberOfRegionsProcessed += numberOfRegionsProcessed;
+    }
+
+    public String getHostsToProcess() {
+      return hostsToProcess;
+    }
+
+    public void setHostsToProcess(String hostsToProcess) {
+      this.hostsToProcess = hostsToProcess;
+    }
+
+    public String getErrorCause() {
+      return errorCause == null ? null : errorCause.toString();
+    }
+
+    public void setErrorCause(String errorCause) {
+      if (errorCause == null || errorCause.trim().length() <= 0) {
+        return;
+      }
+      if (this.errorCause == null) {
+        this.errorCause = new StringBuffer(errorCause);
+      } else {
+        this.errorCause.append(errorCause);
+      }
+    }
+
+    public String getProcessedHosts() {
+      return processedHosts.toString();
+    }
+
+    public void setProcessedHosts(String processedHosts) {
+      if (this.processedHosts == null) {
+        this.processedHosts = new StringBuffer(processedHosts);
+      } else {
+        this.processedHosts.append(" ").append(processedHosts);
+      }
+    }
+
+    /**
+     * Ignore or exempt a RS from schema change processing.
+     * Master will tweak the number of regions to process based on the
+     * number of online regions on the target RS and also remove the
+     * RS from list of hosts to process.
+     * @param schemaAlterStatus
+     */
+    private void ignoreRSForSchemaChange(
+        SchemaChangeTracker.SchemaAlterStatus schemaAlterStatus) {
+      LOG.debug("Removing RS " + schemaAlterStatus.getHostName()
+          + " from schema change process.");
+      hostsToProcess =
+          new String(hostsToProcess).replaceAll(schemaAlterStatus.getHostName(), "");
+      int ignoreRegionsCount = schemaAlterStatus.getNumberOfOnlineRegions();
+      LOG.debug("Current number of regions processed = "
+          + this.numberOfRegionsProcessed + " deducting ignored = "
+          + ignoreRegionsCount
+          + " final = " + (this.numberOfRegionsToProcess-ignoreRegionsCount));
+      if (this.numberOfRegionsToProcess > 0) {
+        this.numberOfRegionsToProcess -= ignoreRegionsCount;
+      } else {
+        LOG.debug("Number of regions to process is less than zero. This is odd");
+      }
+    }
+
+    /**
+     * Update the master alter status for this table based on RS alter status.
+     * @param schemaAlterStatus
+     */
+    public void update(SchemaChangeTracker.SchemaAlterStatus schemaAlterStatus) {
+      this.setProcessedHosts(schemaAlterStatus.getHostName());
+      SchemaChangeTracker.SchemaAlterStatus.AlterState rsState =
+          schemaAlterStatus.getCurrentAlterStatus();
+      switch(rsState) {
+        case FAILURE:
+          LOG.debug("Schema update failure Status = "
+            + schemaAlterStatus);
+          this.setCurrentAlterStatus(
+              MasterAlterStatus.AlterState.FAILURE);
+          this.setNumberOfRegionsProcessed(
+              schemaAlterStatus.getNumberOfRegionsProcessed());
+          this.setErrorCause(schemaAlterStatus.getErrorCause());
+          break;
+        case SUCCESS:
+          LOG.debug("Schema update SUCCESS Status = "
+            + schemaAlterStatus);
+          this.setNumberOfRegionsProcessed(
+              schemaAlterStatus.getNumberOfRegionsProcessed());
+          this.setCurrentAlterStatus(MasterAlterStatus.AlterState.SUCCESS);
+          break;
+        case IGNORED:
+          LOG.debug("Schema update IGNORED Updating regions to " +
+              "process count. Status = "+ schemaAlterStatus);
+          ignoreRSForSchemaChange(schemaAlterStatus);
+          break;
+        default:
+            break;
+      }
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+      currentAlterStatus = AlterState.valueOf(in.readUTF());
+      stamp = in.readLong();
+      numberOfRegionsToProcess = in.readInt();
+      hostsToProcess = Bytes.toString(Bytes.readByteArray(in));
+      processedHosts = new StringBuffer(Bytes.toString(Bytes.readByteArray(in)));
+      errorCause = new StringBuffer(Bytes.toString(Bytes.readByteArray(in)));
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+      out.writeUTF(currentAlterStatus.name());
+      out.writeLong(stamp);
+      out.writeInt(numberOfRegionsToProcess);
+      Bytes.writeByteArray(out, Bytes.toBytes(hostsToProcess));
+      Bytes.writeByteArray(out, Bytes.toBytes(processedHosts.toString()));
+      Bytes.writeByteArray(out, Bytes.toBytes(errorCause.toString()));
+    }
+
+    @Override
+    public String toString() {
+      return
+         " state= " + currentAlterStatus
+        + ", ts= " + stamp
+        + ", number of regions to process = " + numberOfRegionsToProcess
+        + ", number of regions processed = " + numberOfRegionsProcessed
+        + ", hosts = " + hostsToProcess
+        + " , processed hosts = " + processedHosts
+        + " , errorCause = " + errorCause;
+    }
+  }
+}