You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2011/11/21 18:29:33 UTC
svn commit: r1204611 [1/2] - in /hbase/trunk: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/executor/
src/main/java/org/apache/hadoop/hbase/ipc/
src/main/java/org/apache/hadoop/hbase/master/ src/main/java/org/apache/ha...
Author: tedyu
Date: Mon Nov 21 17:29:32 2011
New Revision: 1204611
URL: http://svn.apache.org/viewvc?rev=1204611&view=rev
Log:
HBASE-4213 Support for fault tolerant, instant schema updates with out master's intervention through ZK
Added:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/SchemaChangeTracker.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/InstantSchemaChangeTestBase.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChange.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeFailover.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestInstantSchemaChangeSplit.java
Modified:
hbase/trunk/pom.xml
hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
hbase/trunk/src/main/resources/hbase-default.xml
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
Modified: hbase/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/pom.xml?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/pom.xml (original)
+++ hbase/trunk/pom.xml Mon Nov 21 17:29:32 2011
@@ -338,7 +338,7 @@
<configuration>
<forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
- <argLine>-enableassertions -Xmx1400m</argLine>
+ <argLine>-enableassertions -Xmx1900m</argLine>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
</configuration>
</plugin>
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java Mon Nov 21 17:29:32 2011
@@ -305,7 +305,9 @@ public class LocalHBaseCluster {
*/
public HMaster getActiveMaster() {
for (JVMClusterUtil.MasterThread mt : masterThreads) {
- if (mt.getMaster().isActiveMaster()) {
+ // Ensure that the current active master is not stopped.
+ // We don't want to return a stopping master as an active master.
+ if (mt.getMaster().isActiveMaster() && !mt.getMaster().isStopped()) {
return mt.getMaster();
}
}
@@ -449,4 +451,4 @@ public class LocalHBaseCluster {
admin.createTable(htd);
cluster.shutdown();
}
-}
\ No newline at end of file
+}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java Mon Nov 21 17:29:32 2011
@@ -141,13 +141,12 @@ public abstract class EventHandler imple
* Constructor
*/
EventType(int value) {}
- public boolean isOnlineSchemaChangeSupported() {
+ public boolean isSchemaChangeEvent() {
return (
- this.equals(EventType.C_M_ADD_FAMILY) ||
- this.equals(EventType.C_M_DELETE_FAMILY) ||
- this.equals(EventType.C_M_MODIFY_FAMILY) ||
- this.equals(EventType.C_M_MODIFY_TABLE)
- );
+ this.equals(EventType.C_M_ADD_FAMILY) ||
+ this.equals(EventType.C_M_DELETE_FAMILY) ||
+ this.equals(EventType.C_M_MODIFY_FAMILY) ||
+ this.equals(EventType.C_M_MODIFY_TABLE));
}
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java Mon Nov 21 17:29:32 2011
@@ -245,17 +245,17 @@ public interface HMasterInterface extend
public HTableDescriptor[] getHTableDescriptors();
/**
- * Get current HTD for a given tablename
- * @param tableName
- * @return HTableDescriptor for the table
- */
- //public HTableDescriptor getHTableDescriptor(final byte[] tableName);
-
- /**
* Get array of HTDs for requested tables.
* @param tableNames
* @return array of HTableDescriptor
*/
public HTableDescriptor[] getHTableDescriptors(List<String> tableNames);
+ /**
+ * Returns the current running status of load balancer.
+ * @return True if LoadBalancer is running now else False.
+ */
+ public boolean isLoadBalancerRunning();
+
+
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Mon Nov 21 17:29:32 2011
@@ -272,10 +272,9 @@ public class AssignmentManager extends Z
* @param tableName
* @return Pair indicating the status of the alter command
* @throws IOException
- * @throws InterruptedException
*/
public Pair<Integer, Integer> getReopenStatus(byte[] tableName)
- throws IOException, InterruptedException {
+ throws IOException {
List <HRegionInfo> hris =
MetaReader.getTableRegions(this.master.getCatalogTracker(), tableName);
Integer pending = 0;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Mon Nov 21 17:29:32 2011
@@ -27,8 +27,8 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.client.Me
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -72,7 +73,6 @@ import org.apache.hadoop.hbase.master.ha
import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
-import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
@@ -88,8 +88,9 @@ import org.apache.hadoop.hbase.util.Thre
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.hbase.zookeeper.ClusterId;
import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
-import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
+import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
+import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Text;
@@ -158,7 +159,10 @@ implements HMasterInterface, HMasterRegi
private CatalogTracker catalogTracker;
// Cluster status zk tracker and local setter
private ClusterStatusTracker clusterStatusTracker;
-
+
+ // Schema change tracker
+ private MasterSchemaChangeTracker schemaChangeTracker;
+
// buffer for "fatal error" notices from region servers
// in the cluster. This is only used for assisting
// operations/debugging.
@@ -184,11 +188,18 @@ implements HMasterInterface, HMasterRegi
private CatalogJanitor catalogJanitorChore;
private LogCleaner logCleaner;
+ private Thread schemaJanitorChore;
private MasterCoprocessorHost cpHost;
private final ServerName serverName;
private TableDescriptors tableDescriptors;
+
+ // Whether or not schema alter changes go through ZK or not.
+ private boolean supportInstantSchemaChanges = false;
+
+ private volatile boolean loadBalancerRunning = false;
+
/**
* Initializes the HMaster. The steps are as follows:
@@ -254,6 +265,17 @@ implements HMasterInterface, HMasterRegi
this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(), this, true);
this.rpcServer.startThreads();
this.metrics = new MasterMetrics(getServerName().toString());
+ // initialize instant schema change settings
+ this.supportInstantSchemaChanges = conf.getBoolean(
+ "hbase.instant.schema.alter.enabled", false);
+ if (supportInstantSchemaChanges) {
+ LOG.info("Instant schema change enabled. All schema alter operations will " +
+ "happen through ZK.");
+ }
+ else {
+ LOG.info("Instant schema change disabled. All schema alter operations will " +
+ "happen normally.");
+ }
}
/**
@@ -386,6 +408,12 @@ implements HMasterInterface, HMasterRegi
boolean wasUp = this.clusterStatusTracker.isClusterUp();
if (!wasUp) this.clusterStatusTracker.setClusterUp();
+ // initialize schema change tracker
+ this.schemaChangeTracker = new MasterSchemaChangeTracker(getZooKeeper(),
+ this, this,
+ conf.getInt("hbase.instant.schema.alter.timeout", 60000));
+ this.schemaChangeTracker.start();
+
LOG.info("Server active/primary master; " + this.serverName +
", sessionid=0x" +
Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
@@ -497,6 +525,9 @@ implements HMasterInterface, HMasterRegi
this.catalogJanitorChore = new CatalogJanitor(this, this);
Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());
+ // Schema janitor chore.
+ this.schemaJanitorChore = getAndStartSchemaJanitorChore(this);
+
status.markComplete("Initialization successful");
LOG.info("Master has completed initialization");
initialized = true;
@@ -622,6 +653,15 @@ implements HMasterInterface, HMasterRegi
return this.tableDescriptors;
}
+ @Override
+ public MasterSchemaChangeTracker getSchemaChangeTracker() {
+ return this.schemaChangeTracker;
+ }
+
+ public RegionServerTracker getRegionServerTracker() {
+ return this.regionServerTracker;
+ }
+
/** @return InfoServer object. Maybe null.*/
public InfoServer getInfoServer() {
return this.infoServer;
@@ -724,7 +764,28 @@ implements HMasterInterface, HMasterRegi
if (this.executorService != null) this.executorService.shutdown();
}
- private static Thread getAndStartBalancerChore(final HMaster master) {
+ /**
+ * Start the schema janitor. This Janitor will periodically sweep the failed/expired schema
+ * changes.
+ * @param master
+ * @return
+ */
+ private Thread getAndStartSchemaJanitorChore(final HMaster master) {
+ String name = master.getServerName() + "-SchemaJanitorChore";
+ int schemaJanitorPeriod =
+ master.getConfiguration().getInt("hbase.instant.schema.janitor.period", 120000);
+ // Start up the schema janitor chore
+ Chore chore = new Chore(name, schemaJanitorPeriod, master) {
+ @Override
+ protected void chore() {
+ master.getSchemaChangeTracker().handleFailedOrExpiredSchemaChanges();
+ }
+ };
+ return Threads.setDaemonThreadRunning(chore.getThread());
+ }
+
+
+ private Thread getAndStartBalancerChore(final HMaster master) {
String name = master.getServerName() + "-BalancerChore";
int balancerPeriod =
master.getConfiguration().getInt("hbase.balancer.period", 300000);
@@ -745,6 +806,10 @@ implements HMasterInterface, HMasterRegi
if (this.catalogJanitorChore != null) {
this.catalogJanitorChore.interrupt();
}
+ if (this.schemaJanitorChore != null) {
+ this.schemaJanitorChore.interrupt();
+ }
+
}
@Override
@@ -815,6 +880,15 @@ implements HMasterInterface, HMasterRegi
return balancerCutoffTime;
}
+
+ /**
+ * Check whether the Load Balancer is currently running.
+ * @return true if the Load balancer is currently running.
+ */
+ public boolean isLoadBalancerRunning() {
+ return loadBalancerRunning;
+ }
+
@Override
public boolean balance() {
// If balance not true, don't run balancer.
@@ -822,23 +896,33 @@ implements HMasterInterface, HMasterRegi
// Do this call outside of synchronized block.
int maximumBalanceTime = getBalancerCutoffTime();
long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
- boolean balancerRan;
+ boolean balancerRan = false;
synchronized (this.balancer) {
+ if (loadBalancerRunning) {
+ LOG.debug("Load balancer is currently running. Skipping the current execution.");
+ return false;
+ }
+
// Only allow one balance run at at time.
if (this.assignmentManager.isRegionsInTransition()) {
LOG.debug("Not running balancer because " +
- this.assignmentManager.getRegionsInTransition().size() +
- " region(s) in transition: " +
- org.apache.commons.lang.StringUtils.
+ this.assignmentManager.getRegionsInTransition().size() +
+ " region(s) in transition: " +
+ org.apache.commons.lang.StringUtils.
abbreviate(this.assignmentManager.getRegionsInTransition().toString(), 256));
return false;
}
if (this.serverManager.areDeadServersInProgress()) {
LOG.debug("Not running balancer because processing dead regionserver(s): " +
- this.serverManager.getDeadServers());
+ this.serverManager.getDeadServers());
return false;
}
-
+ if (schemaChangeTracker.isSchemaChangeInProgress()) {
+ LOG.debug("Schema change operation is in progress. Waiting for " +
+ "it to complete before running the load balancer.");
+ return false;
+ }
+ loadBalancerRunning = true;
if (this.cpHost != null) {
try {
if (this.cpHost.preBalance()) {
@@ -855,13 +939,13 @@ implements HMasterInterface, HMasterRegi
this.assignmentManager.getAssignments();
// Returned Map from AM does not include mention of servers w/o assignments.
for (Map.Entry<ServerName, HServerLoad> e:
- this.serverManager.getOnlineServers().entrySet()) {
+ this.serverManager.getOnlineServers().entrySet()) {
if (!assignments.containsKey(e.getKey())) {
assignments.put(e.getKey(), new ArrayList<HRegionInfo>());
}
}
List<RegionPlan> plans = this.balancer.balanceCluster(assignments);
- int rpCount = 0; // number of RegionPlans balanced so far
+ int rpCount = 0; // number of RegionPlans balanced so far
long totalRegPlanExecTime = 0;
balancerRan = plans != null;
if (plans != null && !plans.isEmpty()) {
@@ -875,7 +959,7 @@ implements HMasterInterface, HMasterRegi
// if performing next balance exceeds cutoff time, exit the loop
(System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
- maximumBalanceTime);
+ maximumBalanceTime);
break;
}
}
@@ -888,6 +972,7 @@ implements HMasterInterface, HMasterRegi
LOG.error("Error invoking master coprocessor postBalance()", ioe);
}
}
+ loadBalancerRunning = false;
}
return balancerRan;
}
@@ -1036,22 +1121,63 @@ implements HMasterInterface, HMasterRegi
if (cpHost != null) {
cpHost.preDeleteTable(tableName);
}
- this.executorService.submit(new DeleteTableHandler(tableName, this, this));
+ this.executorService.submit(new DeleteTableHandler(tableName, this, this, this,
+ supportInstantSchemaChanges));
if (cpHost != null) {
cpHost.postDeleteTable(tableName);
}
}
+ /**
+ * Get the number of regions of the table that have been updated by the alter.
+ *
+ * @return Pair indicating the number of regions updated Pair.getFirst is the
+ * regions that are yet to be updated Pair.getSecond is the total number
+ * of regions of the table
+ */
public Pair<Integer, Integer> getAlterStatus(byte[] tableName)
throws IOException {
+ if (supportInstantSchemaChanges) {
+ return getAlterStatusFromSchemaChangeTracker(tableName);
+ }
+ return this.assignmentManager.getReopenStatus(tableName);
+ }
+
+ /**
+ * Used by the client to identify if all regions have the schema updates
+ *
+ * @param tableName
+ * @return Pair indicating the status of the alter command
+ * @throws IOException
+ */
+ private Pair<Integer, Integer> getAlterStatusFromSchemaChangeTracker(byte[] tableName)
+ throws IOException {
+ MasterSchemaChangeTracker.MasterAlterStatus alterStatus = null;
try {
- return this.assignmentManager.getReopenStatus(tableName);
- } catch (InterruptedException e) {
- throw new IOException("Interrupted", e);
+ alterStatus =
+ this.schemaChangeTracker.getMasterAlterStatus(Bytes.toString(tableName));
+ } catch (KeeperException ke) {
+ LOG.error("KeeperException while getting schema alter status for table = "
+ + Bytes.toString(tableName), ke);
+ }
+ if (alterStatus != null) {
+ LOG.debug("Getting AlterStatus from SchemaChangeTracker for table = "
+ + Bytes.toString(tableName) + " Alter Status = "
+ + alterStatus.toString());
+ int numberPending = alterStatus.getNumberOfRegionsToProcess() -
+ alterStatus.getNumberOfRegionsProcessed();
+ return new Pair<Integer, Integer>(alterStatus.getNumberOfRegionsProcessed(),
+ alterStatus.getNumberOfRegionsToProcess());
+ } else {
+ LOG.debug("MasterAlterStatus is NULL for table = "
+ + Bytes.toString(tableName));
+ // should we throw IOException here as it makes more sense?
+ return new Pair<Integer, Integer>(0,0);
}
}
+
public void addColumn(byte [] tableName, HColumnDescriptor column)
throws IOException {
if (cpHost != null) {
@@ -1059,7 +1185,8 @@ implements HMasterInterface, HMasterRegi
return;
}
}
- new TableAddFamilyHandler(tableName, column, this, this).process();
+ new TableAddFamilyHandler(tableName, column, this, this,
+ this, supportInstantSchemaChanges).process();
if (cpHost != null) {
cpHost.postAddColumn(tableName, column);
}
@@ -1072,7 +1199,8 @@ implements HMasterInterface, HMasterRegi
return;
}
}
- new TableModifyFamilyHandler(tableName, descriptor, this, this).process();
+ new TableModifyFamilyHandler(tableName, descriptor, this, this,
+ this, supportInstantSchemaChanges).process();
if (cpHost != null) {
cpHost.postModifyColumn(tableName, descriptor);
}
@@ -1085,7 +1213,8 @@ implements HMasterInterface, HMasterRegi
return;
}
}
- new TableDeleteFamilyHandler(tableName, c, this, this).process();
+ new TableDeleteFamilyHandler(tableName, c, this, this,
+ this, supportInstantSchemaChanges).process();
if (cpHost != null) {
cpHost.postDeleteColumn(tableName, c);
}
@@ -1096,7 +1225,7 @@ implements HMasterInterface, HMasterRegi
cpHost.preEnableTable(tableName);
}
this.executorService.submit(new EnableTableHandler(this, tableName,
- catalogTracker, assignmentManager, false));
+ catalogTracker, assignmentManager, false));
if (cpHost != null) {
cpHost.postEnableTable(tableName);
@@ -1152,21 +1281,37 @@ implements HMasterInterface, HMasterRegi
@Override
public void modifyTable(final byte[] tableName, HTableDescriptor htd)
- throws IOException {
+ throws IOException {
if (cpHost != null) {
cpHost.preModifyTable(tableName, htd);
}
-
this.executorService.submit(new ModifyTableHandler(tableName, htd, this,
- this));
-
+ this, this, supportInstantSchemaChanges));
if (cpHost != null) {
cpHost.postModifyTable(tableName, htd);
}
}
+ private boolean isOnlineSchemaChangeAllowed() {
+ return conf.getBoolean(
+ "hbase.online.schema.update.enable", false);
+ }
+
@Override
- public void checkTableModifiable(final byte [] tableName)
+ public void checkTableModifiable(final byte [] tableName,
+ EventHandler.EventType eventType)
+ throws IOException {
+ preCheckTableModifiable(tableName);
+ if (!eventType.isSchemaChangeEvent() ||
+ !isOnlineSchemaChangeAllowed()) {
+ if (!getAssignmentManager().getZKTable().
+ isDisabledTable(Bytes.toString(tableName))) {
+ throw new TableNotDisabledException(tableName);
+ }
+ }
+ }
+
+ private void preCheckTableModifiable(final byte[] tableName)
throws IOException {
String tableNameStr = Bytes.toString(tableName);
if (isCatalogTable(tableName)) {
@@ -1175,10 +1320,6 @@ implements HMasterInterface, HMasterRegi
if (!MetaReader.tableExists(getCatalogTracker(), tableNameStr)) {
throw new TableNotFoundException(tableNameStr);
}
- if (!getAssignmentManager().getZKTable().
- isDisabledTable(Bytes.toString(tableName))) {
- throw new TableNotDisabledException(tableName);
- }
}
public void clearFromTransition(HRegionInfo hri) {
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Mon Nov 21 17:29:32 2011
@@ -504,7 +504,7 @@ public class MasterFileSystem {
*/
public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
throws IOException {
- LOG.info("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
+ LOG.debug("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
hcd.toString());
HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
if (htd == null) {
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java Mon Nov 21 17:29:32 2011
@@ -26,7 +26,10 @@ import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
+import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
/**
* Services Master supplies
@@ -53,12 +56,15 @@ public interface MasterServices extends
public ExecutorService getExecutorService();
/**
- * Check table is modifiable; i.e. exists and is offline.
- * @param tableName Name of table to check.
- * @throws TableNotDisabledException
- * @throws TableNotFoundException
+ * Check table modifiable. i.e not ROOT or META and offlined for all commands except
+ * alter commands
+ * @param tableName
+ * @param eventType
+ * @throws IOException
*/
- public void checkTableModifiable(final byte [] tableName) throws IOException;
+ public void checkTableModifiable(final byte [] tableName,
+ EventHandler.EventType eventType)
+ throws IOException;
/**
* Create a table using the given table definition.
@@ -73,4 +79,17 @@ public interface MasterServices extends
* @return Return table descriptors implementation.
*/
public TableDescriptors getTableDescriptors();
+
+ /**
+ * Get Master Schema change tracker
+ * @return
+ */
+ public MasterSchemaChangeTracker getSchemaChangeTracker();
+
+ /**
+ * Return the Region server tracker.
+ * @return RegionServerTracker
+ */
+ public RegionServerTracker getRegionServerTracker();
+
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Mon Nov 21 17:29:32 2011
@@ -333,11 +333,21 @@ public class ServerManager {
}
}
+ /**
+ * Exclude a RS from any pending schema change process.
+ * @param serverName
+ */
+ private void excludeRegionServerFromSchemaChanges(final ServerName serverName) {
+ this.services.getSchemaChangeTracker()
+ .excludeRegionServerForSchemaChanges(serverName.getServerName());
+ }
+
/*
* Expire the passed server. Add it to list of deadservers and queue a
* shutdown processing.
*/
public synchronized void expireServer(final ServerName serverName) {
+ excludeRegionServerFromSchemaChanges(serverName);
if (!this.onlineServers.containsKey(serverName)) {
LOG.warn("Received expiration of " + serverName +
" but server is not currently online");
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java Mon Nov 21 17:29:32 2011
@@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.util.Bytes;
@@ -37,9 +38,11 @@ public class DeleteTableHandler extends
private static final Log LOG = LogFactory.getLog(DeleteTableHandler.class);
public DeleteTableHandler(byte [] tableName, Server server,
- final MasterServices masterServices)
+ final MasterServices masterServices, HMasterInterface masterInterface,
+ boolean instantChange)
throws IOException {
- super(EventType.C_M_DELETE_TABLE, tableName, server, masterServices);
+ super(EventType.C_M_DELETE_TABLE, tableName, server, masterServices,
+ masterInterface, instantChange);
// The next call fails if no such table.
getTableDescriptor();
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java Mon Nov 21 17:29:32 2011
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.master.MasterServices;
public class ModifyTableHandler extends TableEventHandler {
@@ -32,9 +33,11 @@ public class ModifyTableHandler extends
public ModifyTableHandler(final byte [] tableName,
final HTableDescriptor htd, final Server server,
- final MasterServices masterServices)
+ final MasterServices masterServices, final HMasterInterface masterInterface,
+ boolean instantModify)
throws IOException {
- super(EventType.C_M_MODIFY_TABLE, tableName, server, masterServices);
+ super(EventType.C_M_MODIFY_TABLE, tableName, server, masterServices,
+ masterInterface, instantModify);
// Check table exists.
getTableDescriptor();
// This is the new schema we are going to write out as this modification.
@@ -57,4 +60,4 @@ public class ModifyTableHandler extends
return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" +
tableNameStr;
}
-}
\ No newline at end of file
+}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java Mon Nov 21 17:29:32 2011
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HRegionIn
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.master.MasterServices;
/**
@@ -37,8 +38,10 @@ public class TableAddFamilyHandler exten
private final HColumnDescriptor familyDesc;
public TableAddFamilyHandler(byte[] tableName, HColumnDescriptor familyDesc,
- Server server, final MasterServices masterServices) throws IOException {
- super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices);
+ Server server, final MasterServices masterServices,
+ HMasterInterface masterInterface, boolean instantChange) throws IOException {
+ super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices,
+ masterInterface, instantChange);
HTableDescriptor htd = getTableDescriptor();
if (htd.hasFamily(familyDesc.getName())) {
throw new InvalidFamilyOperationException("Family '" +
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java Mon Nov 21 17:29:32 2011
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.util.Bytes;
@@ -36,8 +37,10 @@ public class TableDeleteFamilyHandler ex
private final byte [] familyName;
public TableDeleteFamilyHandler(byte[] tableName, byte [] familyName,
- Server server, final MasterServices masterServices) throws IOException {
- super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices);
+ Server server, final MasterServices masterServices,
+ HMasterInterface masterInterface, boolean instantChange) throws IOException {
+ super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices,
+ masterInterface, instantChange);
HTableDescriptor htd = getTableDescriptor();
this.familyName = hasColumnFamily(htd, familyName);
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java Mon Nov 21 17:29:32 2011
@@ -35,13 +35,19 @@ import org.apache.hadoop.hbase.InvalidFa
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.master.BulkReOpen;
import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
import com.google.common.collect.Lists;
@@ -57,32 +63,22 @@ import com.google.common.collect.Maps;
public abstract class TableEventHandler extends EventHandler {
private static final Log LOG = LogFactory.getLog(TableEventHandler.class);
protected final MasterServices masterServices;
+ protected HMasterInterface master = null;
protected final byte [] tableName;
protected final String tableNameStr;
+ protected boolean instantAction = false;
public TableEventHandler(EventType eventType, byte [] tableName, Server server,
- MasterServices masterServices)
+ MasterServices masterServices, HMasterInterface masterInterface,
+ boolean instantSchemaChange)
throws IOException {
super(server, eventType);
this.masterServices = masterServices;
this.tableName = tableName;
- try {
- this.masterServices.checkTableModifiable(tableName);
- } catch (TableNotDisabledException ex) {
- if (isOnlineSchemaChangeAllowed()
- && eventType.isOnlineSchemaChangeSupported()) {
- LOG.debug("Ignoring table not disabled exception " +
- "for supporting online schema changes.");
- } else {
- throw ex;
- }
- }
+ this.masterServices.checkTableModifiable(tableName, eventType);
this.tableNameStr = Bytes.toString(this.tableName);
- }
-
- private boolean isOnlineSchemaChangeAllowed() {
- return this.server.getConfiguration().getBoolean(
- "hbase.online.schema.update.enable", false);
+ this.instantAction = instantSchemaChange;
+ this.master = masterInterface;
}
@Override
@@ -94,16 +90,7 @@ public abstract class TableEventHandler
MetaReader.getTableRegions(this.server.getCatalogTracker(),
tableName);
handleTableOperation(hris);
- if (eventType.isOnlineSchemaChangeSupported() && this.masterServices.
- getAssignmentManager().getZKTable().
- isEnabledTable(Bytes.toString(tableName))) {
- if (reOpenAllRegions(hris)) {
- LOG.info("Completed table operation " + eventType + " on table " +
- Bytes.toString(tableName));
- } else {
- LOG.warn("Error on reopening the regions");
- }
- }
+ handleSchemaChanges(hris);
} catch (IOException e) {
LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
} catch (KeeperException e) {
@@ -111,13 +98,47 @@ public abstract class TableEventHandler
}
}
+ private void handleSchemaChanges(List<HRegionInfo> regions)
+ throws IOException {
+ if (instantAction && regions != null && !regions.isEmpty()) {
+ handleInstantSchemaChanges(regions);
+ } else {
+ handleRegularSchemaChanges(regions);
+ }
+ }
+
+
+ /**
+ * Perform schema changes only if the table is in enabled state.
+ * @return
+ */
+ private boolean canPerformSchemaChange() {
+ return (eventType.isSchemaChangeEvent() && this.masterServices.
+ getAssignmentManager().getZKTable().
+ isEnabledTable(Bytes.toString(tableName)));
+ }
+
+ private void handleRegularSchemaChanges(List<HRegionInfo> regions)
+ throws IOException {
+ if (canPerformSchemaChange()) {
+ this.masterServices.getAssignmentManager().setRegionsToReopen(regions);
+ if (reOpenAllRegions(regions)) {
+ LOG.info("Completed table operation " + eventType + " on table " +
+ Bytes.toString(tableName));
+ } else {
+ LOG.warn("Error on reopening the regions");
+ }
+ }
+ }
+
public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
boolean done = false;
LOG.info("Bucketing regions by region server...");
HTable table = new HTable(masterServices.getConfiguration(), tableName);
TreeMap<ServerName, List<HRegionInfo>> serverToRegions = Maps
.newTreeMap();
- NavigableMap<HRegionInfo, ServerName> hriHserverMapping = table.getRegionLocations();
+ NavigableMap<HRegionInfo, ServerName> hriHserverMapping
+ = table.getRegionLocations();
List<HRegionInfo> reRegions = new ArrayList<HRegionInfo>();
for (HRegionInfo hri : regions) {
ServerName rsLocation = hriHserverMapping.get(hri);
@@ -160,6 +181,91 @@ public abstract class TableEventHandler
}
/**
+ * Check whether any of the regions from the list of regions is undergoing a split.
+ * We simply check whether there is a unassigned node for any of the region and if so
+ * we return as true.
+ * @param regionInfos
+ * @return
+ */
+ private boolean isSplitInProgress(List<HRegionInfo> regionInfos) {
+ for (HRegionInfo hri : regionInfos) {
+ ZooKeeperWatcher zkw = this.masterServices.getZooKeeper();
+ String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
+ try {
+ if (ZKUtil.checkExists(zkw, node) != -1) {
+ LOG.debug("Region " + hri.getRegionNameAsString() + " is unassigned. Assuming" +
+ " that it is undergoing a split");
+ return true;
+ }
+ } catch (KeeperException ke) {
+ LOG.debug("KeeperException while determining splits in progress.", ke);
+ // Assume no splits happening?
+ return false;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Wait for region split transaction in progress (if any)
+ * @param regions
+ * @param status
+ */
+ private void waitForInflightSplit(List<HRegionInfo> regions, MonitoredTask status) {
+ while (isSplitInProgress(regions)) {
+ try {
+ status.setStatus("Alter Schema is waiting for split region to complete.");
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ protected void handleInstantSchemaChanges(List<HRegionInfo> regions) {
+ if (regions == null || regions.isEmpty()) {
+ LOG.debug("Region size is null or empty. Ignoring alter request.");
+ return;
+ }
+ MonitoredTask status = TaskMonitor.get().createStatus(
+ "Handling alter table request for table = " + tableNameStr);
+ if (canPerformSchemaChange()) {
+ boolean prevBalanceSwitch = false;
+ try {
+ // turn off load balancer synchronously
+ prevBalanceSwitch = master.synchronousBalanceSwitch(false);
+ waitForInflightSplit(regions, status);
+ MasterSchemaChangeTracker masterSchemaChangeTracker =
+ this.masterServices.getSchemaChangeTracker();
+ masterSchemaChangeTracker
+ .createSchemaChangeNode(Bytes.toString(tableName),
+ regions.size());
+ while(!masterSchemaChangeTracker.doesSchemaChangeNodeExists(
+ Bytes.toString(tableName))) {
+ try {
+ Thread.sleep(50);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ status.markComplete("Created ZK node for handling the alter table request for table = "
+ + tableNameStr);
+ } catch (KeeperException e) {
+ LOG.warn("Instant schema change failed for table " + tableNameStr, e);
+ status.setStatus("Instant schema change failed for table " + tableNameStr
+ + " Cause = " + e.getCause());
+
+ } catch (IOException ioe) {
+ LOG.warn("Instant schema change failed for table " + tableNameStr, ioe);
+ status.setStatus("Instant schema change failed for table " + tableNameStr
+ + " Cause = " + ioe.getCause());
+ } finally {
+ master.synchronousBalanceSwitch(prevBalanceSwitch);
+ }
+ }
+ }
+
+ /**
* @return Table descriptor for this table
* @throws TableExistsException
* @throws FileNotFoundException
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java Mon Nov 21 17:29:32 2011
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HRegionIn
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.util.Bytes;
@@ -38,8 +39,10 @@ public class TableModifyFamilyHandler ex
public TableModifyFamilyHandler(byte[] tableName,
HColumnDescriptor familyDesc, Server server,
- final MasterServices masterServices) throws IOException {
- super(EventType.C_M_MODIFY_FAMILY, tableName, server, masterServices);
+ final MasterServices masterServices,
+ HMasterInterface masterInterface, boolean instantChange) throws IOException {
+ super(EventType.C_M_MODIFY_FAMILY, tableName, server, masterServices,
+ masterInterface, instantChange);
HTableDescriptor htd = getTableDescriptor();
hasColumnFamily(htd, familyDesc.getName());
this.familyDesc = familyDesc;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Mon Nov 21 17:29:32 2011
@@ -155,12 +155,29 @@ public class CompactSplitThread implemen
return false;
}
+ /**
+ * Wait for mid-flight schema alter requests. (if any). We don't want to execute a split
+ * when a schema alter is in progress as we end up in an inconsistent state.
+ * @param tableName
+ */
+ private void waitForInflightSchemaChange(String tableName) {
+ while (this.server.getSchemaChangeTracker()
+ .isSchemaChangeInProgress(tableName)) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
public synchronized void requestSplit(final HRegion r, byte[] midKey) {
if (midKey == null) {
LOG.debug("Region " + r.getRegionNameAsString() +
" not splittable because midkey=null");
return;
}
+ waitForInflightSchemaChange(r.getRegionInfo().getTableNameAsString());
try {
this.splits.execute(new SplitRequest(r, midKey, this.server));
if (LOG.isDebugEnabled()) {
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Mon Nov 21 17:29:32 2011
@@ -145,6 +145,7 @@ import org.apache.hadoop.hbase.zookeeper
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.zookeeper.SchemaChangeTracker;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.ipc.RemoteException;
@@ -283,6 +284,9 @@ public class HRegionServer implements HR
// Cluster Status Tracker
private ClusterStatusTracker clusterStatusTracker;
+ // Schema change Tracker
+ private SchemaChangeTracker schemaChangeTracker;
+
// Log Splitting Worker
private SplitLogWorker splitLogWorker;
@@ -575,6 +579,11 @@ public class HRegionServer implements HR
this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf,
this, this.conf.getInt("hbase.regionserver.catalog.timeout", Integer.MAX_VALUE));
catalogTracker.start();
+
+ // Schema change tracker
+ this.schemaChangeTracker = new SchemaChangeTracker(this.zooKeeper,
+ this, this);
+ this.schemaChangeTracker.start();
}
/**
@@ -1670,7 +1679,9 @@ public class HRegionServer implements HR
protected void join() {
Threads.shutdown(this.compactionChecker.getThread());
Threads.shutdown(this.cacheFlusher.getThread());
- Threads.shutdown(this.hlogRoller.getThread());
+ if (this.hlogRoller != null) {
+ Threads.shutdown(this.hlogRoller.getThread());
+ }
if (this.compactSplitThread != null) {
this.compactSplitThread.join();
}
@@ -2658,9 +2669,26 @@ public class HRegionServer implements HR
splitRegion(regionInfo, null);
}
+ /**
+ * Wait for mid-flight schema change requests. (if any)
+ * @param tableName
+ */
+ private void waitForSchemaChange(String tableName) {
+ while (schemaChangeTracker.isSchemaChangeInProgress(tableName)) {
+ try {
+ LOG.debug("Schema alter is inprogress for table = " + tableName
+ + " Waiting for alter to complete before a split");
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
@Override
public void splitRegion(HRegionInfo regionInfo, byte[] splitPoint)
throws NotServingRegionException, IOException {
+ waitForSchemaChange(Bytes.toString(regionInfo.getTableName()));
checkOpen();
HRegion region = getRegion(regionInfo.getRegionName());
region.flushcache();
@@ -3373,6 +3401,60 @@ public class HRegionServer implements HR
return wal.rollWriter(true);
}
+ /**
+ * Refresh schema changes for given region.
+ * @param hRegion HRegion to refresh
+ * @throws IOException
+ */
+ public void refreshRegion(HRegion hRegion) throws IOException {
+
+ if (hRegion != null) {
+ synchronized (this.onlineRegions) {
+ HRegionInfo regionInfo = hRegion.getRegionInfo();
+ // Close the region
+ hRegion.close();
+ // Remove from online regions
+ removeFromOnlineRegions(regionInfo.getEncodedName());
+ // Get new HTD
+ HTableDescriptor htd = this.tableDescriptors.get(regionInfo.getTableName());
+ LOG.debug("HTD for region = " + regionInfo.getRegionNameAsString()
+ + " Is = " + htd );
+ HRegion region =
+ HRegion.openHRegion(hRegion.getRegionInfo(), htd, hlog, conf,
+ this, null);
+ // Add new region to the onlineRegions
+ addToOnlineRegions(region);
+ }
+ }
+ }
+
+ /**
+ * Gets the online regions of the specified table.
+ * This method looks at the in-memory onlineRegions. It does not go to <code>.META.</code>.
+ * Only returns <em>online</em> regions. If a region on this table has been
+ * closed during a disable, etc., it will not be included in the returned list.
+ * So, the returned list may not necessarily be ALL regions in this table, its
+ * all the ONLINE regions in the table.
+ * @param tableName
+ * @return Online regions from <code>tableName</code>
+ */
+ public List<HRegion> getOnlineRegions(byte[] tableName) {
+ List<HRegion> tableRegions = new ArrayList<HRegion>();
+ synchronized (this.onlineRegions) {
+ for (HRegion region: this.onlineRegions.values()) {
+ HRegionInfo regionInfo = region.getRegionInfo();
+ if(Bytes.equals(regionInfo.getTableName(), tableName)) {
+ tableRegions.add(region);
+ }
+ }
+ }
+ return tableRegions;
+ }
+
+ public SchemaChangeTracker getSchemaChangeTracker() {
+ return this.schemaChangeTracker;
+ }
+
// used by org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon (HBASE-4070).
public String[] getCoprocessors() {
HServerLoad hsl = buildServerLoad();
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java?rev=1204611&r1=1204610&r2=1204611&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java Mon Nov 21 17:29:32 2011
@@ -21,6 +21,9 @@ package org.apache.hadoop.hbase.regionse
import org.apache.hadoop.hbase.Server;
+import java.io.IOException;
+import java.util.List;
+
/**
* Interface to Map of online regions. In the Map, the key is the region's
* encoded name and the value is an {@link HRegion} instance.
@@ -49,4 +52,18 @@ interface OnlineRegions extends Server {
* null if named region is not member of the online regions.
*/
public HRegion getFromOnlineRegions(String encodedRegionName);
-}
\ No newline at end of file
+ /**
+ * Get all online regions of a table in this RS.
+ * @param tableName
+ * @return List of HRegion
+ * @throws java.io.IOException
+ */
+ public List<HRegion> getOnlineRegions(byte[] tableName) throws IOException;
+
+ /**
+ * Refresh a given region updating it with latest HTD info.
+ * @param hRegion
+ */
+ public void refreshRegion(HRegion hRegion) throws IOException;
+
+}
Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java?rev=1204611&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterSchemaChangeTracker.java Mon Nov 21 17:29:32 2011
@@ -0,0 +1,826 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.Writable;
+import org.apache.zookeeper.KeeperException;
+
+public class MasterSchemaChangeTracker extends ZooKeeperNodeTracker {
+ public static final Log LOG = LogFactory.getLog(MasterSchemaChangeTracker.class);
+ private final MasterServices masterServices;
+ // Used by tests only. Do not change this.
+ private volatile int sleepTimeMillis = 0;
+ // schema changes pending more than this time will be timed out.
+ private long schemaChangeTimeoutMillis = 30000;
+
+ /**
+ * Constructs a new ZK node tracker.
+ * <p/>
+ * <p>After construction, use {@link #start} to kick off tracking.
+ *
+ * @param watcher
+ * @param abortable
+ */
+ public MasterSchemaChangeTracker(ZooKeeperWatcher watcher,
+ Abortable abortable, MasterServices masterServices,
+ long schemaChangeTimeoutMillis) {
+ super(watcher, watcher.schemaZNode, abortable);
+ this.masterServices = masterServices;
+ this.schemaChangeTimeoutMillis = schemaChangeTimeoutMillis;
+ }
+
+ @Override
+ public void start() {
+ try {
+ watcher.registerListener(this);
+ List<String> tables =
+ ZKUtil.listChildrenNoWatch(watcher, watcher.schemaZNode);
+ processCompletedSchemaChanges(tables);
+ } catch (KeeperException e) {
+ LOG.error("MasterSchemaChangeTracker startup failed.", e);
+ abortable.abort("MasterSchemaChangeTracker startup failed", e);
+ }
+ }
+
+ private List<String> getCurrentTables() throws KeeperException {
+ return
+ ZKUtil.listChildrenNoWatch(watcher, watcher.schemaZNode);
+ }
+
+ /**
+ * When a primary master crashes and the secondary master takes over
+ * mid-flight during an alter process, the secondary should cleanup any completed
+ * schema changes not handled by the previous master.
+ * @param tables
+ * @throws KeeperException
+ */
+ private void processCompletedSchemaChanges(List<String> tables)
+ throws KeeperException {
+ if (tables == null || tables.isEmpty()) {
+ String msg = "No current schema change in progress. Skipping cleanup";
+ LOG.debug(msg);
+ return;
+ }
+ String msg = "Master seeing following tables undergoing schema change " +
+ "process. Tables = " + tables;
+ MonitoredTask status = TaskMonitor.get().createStatus(msg);
+ LOG.debug(msg);
+ for (String table : tables) {
+ LOG.debug("Processing table = "+ table);
+ status.setStatus("Processing table = "+ table);
+ try {
+ processTableNode(table);
+ } catch (IOException e) {
+ String errmsg = "IOException while processing completed schema changes."
+ + " Cause = " + e.getCause();
+ LOG.error(errmsg, e);
+ status.setStatus(errmsg);
+ }
+ }
+ }
+
+ /**
+ * Get current alter status for a table.
+ * @param tableName
+ * @return MasterAlterStatus
+ * @throws KeeperException
+ * @throws IOException
+ */
+ public MasterAlterStatus getMasterAlterStatus(String tableName)
+ throws KeeperException, IOException {
+ String path = getSchemaChangeNodePathForTable(tableName);
+ byte[] state = ZKUtil.getData(watcher, path);
+ if (state == null || state.length <= 0) {
+ return null;
+ }
+ MasterAlterStatus mas = new MasterAlterStatus();
+ Writables.getWritable(state, mas);
+ return mas;
+ }
+
+ /**
+ * Get RS specific alter status for a table & server
+ * @param tableName
+ * @param serverName
+ * @return Region Server's Schema alter status
+ * @throws KeeperException
+ * @throws IOException
+ */
+ private SchemaChangeTracker.SchemaAlterStatus getRSSchemaAlterStatus(
+ String tableName, String serverName)
+ throws KeeperException, IOException {
+ String childPath =
+ getSchemaChangeNodePathForTableAndServer(tableName, serverName);
+ byte[] childData = ZKUtil.getData(this.watcher, childPath);
+ if (childData == null || childData.length <= 0) {
+ return null;
+ }
+ SchemaChangeTracker.SchemaAlterStatus sas =
+ new SchemaChangeTracker.SchemaAlterStatus();
+ Writables.getWritable(childData, sas);
+ LOG.debug("Schema Status data for server = " + serverName + " table = "
+ + tableName + " == " + sas);
+ return sas;
+ }
+
+ /**
+ * Update the master's alter status based on all region server's response.
+ * @param servers
+ * @param tableName
+ * @throws IOException
+ */
+ private void updateMasterAlterStatus(MasterAlterStatus mas,
+ List<String> servers, String tableName)
+ throws IOException, KeeperException {
+ for (String serverName : servers) {
+ SchemaChangeTracker.SchemaAlterStatus sas =
+ getRSSchemaAlterStatus(tableName, serverName);
+ if (sas != null) {
+ mas.update(sas);
+ LOG.debug("processTableNodeWithState:Updated Master Alter Status = "
+ + mas + " for server = " + serverName);
+ } else {
+ LOG.debug("SchemaAlterStatus is NULL for table = " + tableName);
+ }
+ }
+ }
+
+ /**
+ * If schema alter is handled for this table, then delete all the ZK nodes
+ * created for this table.
+ * @param tableName
+ * @throws KeeperException
+ */
+ private void processTableNode(String tableName) throws KeeperException,
+ IOException {
+ LOG.debug("processTableNodeWithState. TableName = " + tableName);
+ List<String> servers =
+ ZKUtil.listChildrenAndWatchThem(watcher,
+ getSchemaChangeNodePathForTable(tableName));
+ MasterAlterStatus mas = getMasterAlterStatus(tableName);
+ if (mas == null) {
+ LOG.debug("MasterAlterStatus is NULL. Table = " + tableName);
+ return;
+ }
+ updateMasterAlterStatus(mas, servers, tableName);
+ LOG.debug("Current Alter status = " + mas);
+ String nodePath = getSchemaChangeNodePathForTable(tableName);
+ ZKUtil.updateExistingNodeData(this.watcher, nodePath,
+ Writables.getBytes(mas), getZKNodeVersion(nodePath));
+ processAlterStatus(mas, tableName, servers);
+ }
+
+ /**
+ * Evaluate the master alter status and determine the current status.
+ * @param alterStatus
+ * @param tableName
+ * @param servers
+ * @param status
+ */
+ private void processAlterStatus(MasterAlterStatus alterStatus,
+ String tableName, List<String> servers)
+ throws KeeperException {
+ if (alterStatus.getNumberOfRegionsToProcess()
+ == alterStatus.getNumberOfRegionsProcessed()) {
+ // schema change completed.
+ String msg = "All region servers have successfully processed the " +
+ "schema changes for table = " + tableName
+ + " . Deleting the schema change node for table = "
+ + tableName + " Region servers processed the schema change" +
+ " request = " + alterStatus.getProcessedHosts()
+ + " Total number of regions = " + alterStatus.getNumberOfRegionsToProcess()
+ + " Processed regions = " + alterStatus.getNumberOfRegionsProcessed();
+ MonitoredTask status = TaskMonitor.get().createStatus(
+ "Checking alter schema request status for table = " + tableName);
+ status.markComplete(msg);
+ LOG.debug(msg);
+ cleanProcessedTableNode(getSchemaChangeNodePathForTable(tableName));
+ } else {
+ if (alterStatus.getErrorCause() != null
+ && alterStatus.getErrorCause().trim().length() > 0) {
+ String msg = "Alter schema change failed "
+ + "for table = " + tableName + " Number of online regions = "
+ + alterStatus.getNumberOfRegionsToProcess() + " processed regions count = "
+ + alterStatus.getNumberOfRegionsProcessed()
+ + " Original list = " + alterStatus.hostsToProcess + " Processed servers = "
+ + servers
+ + " Error Cause = " + alterStatus.getErrorCause();
+ MonitoredTask status = TaskMonitor.get().createStatus(
+ "Checking alter schema request status for table = " + tableName);
+ // we have errors.
+ LOG.debug(msg);
+ status.abort(msg);
+ } else {
+ String msg = "Not all region servers have processed the schema changes"
+ + "for table = " + tableName + " Number of online regions = "
+ + alterStatus.getNumberOfRegionsToProcess() + " processed regions count = "
+ + alterStatus.getNumberOfRegionsProcessed()
+ + " Original list = " + alterStatus.hostsToProcess + " Processed servers = "
+ + servers + " Alter STate = "
+ + alterStatus.getCurrentAlterStatus();
+ LOG.debug(msg);
+ // status.setStatus(msg);
+ }
+ }
+ }
+
+ /**
+ * Check whether a in-flight schema change request has expired.
+ * @param tableName
+ * @return true is the schema change request expired.
+ * @throws IOException
+ */
+ private boolean hasSchemaChangeExpiredFor(String tableName)
+ throws IOException, KeeperException {
+ MasterAlterStatus mas = getMasterAlterStatus(tableName);
+ long createdTimeStamp = mas.getStamp();
+ long duration = System.currentTimeMillis() - createdTimeStamp;
+ LOG.debug("Created TimeStamp = " + createdTimeStamp
+ + " duration = " + duration + " Table = " + tableName
+ + " Master Alter Status = " + mas);
+ return (duration > schemaChangeTimeoutMillis);
+ }
+
+ /**
+ * Handle failed and expired schema changes. We simply delete all the
+ * expired/failed schema change attempts. Why we should do this ?
+ * 1) Keeping the failed/expired schema change nodes longer prohibits any
+ * future schema changes for the table.
+ * 2) Any lingering expired/failed schema change requests will prohibit the
+ * load balancer from running.
+ */
+ public void handleFailedOrExpiredSchemaChanges() {
+ try {
+ List<String> tables = getCurrentTables();
+ for (String table : tables) {
+ String statmsg = "Cleaning failed or expired schema change requests. " +
+ "current tables undergoing " +
+ "schema change process = " + tables;
+ MonitoredTask status = TaskMonitor.get().createStatus(statmsg);
+ LOG.debug(statmsg);
+ if (hasSchemaChangeExpiredFor(table)) {
+ // time out.. currently, we abandon the in-flight schema change due to
+ // time out.
+ // Here, there are couple of options to consider. One could be to
+ // attempt a retry of the schema change and see if it succeeds, and
+ // another could be to simply rollback the schema change effort and
+ // see if it succeeds.
+ String msg = "Schema change for table = " + table + " has expired."
+ + " Schema change for this table has been in progress for " +
+ + schemaChangeTimeoutMillis +
+ "Deleting the node now.";
+ LOG.debug(msg);
+ ZKUtil.deleteNodeRecursively(this.watcher,
+ getSchemaChangeNodePathForTable(table));
+ } else {
+ String msg = "Schema change request is in progress for " +
+ " table = " + table;
+ LOG.debug(msg);
+ status.setStatus(msg);
+ }
+ }
+ } catch (IOException e) {
+ String msg = "IOException during handleFailedExpiredSchemaChanges."
+ + e.getCause();
+ LOG.error(msg, e);
+ TaskMonitor.get().createStatus(msg);
+ } catch (KeeperException ke) {
+ String msg = "KeeperException during handleFailedExpiredSchemaChanges."
+ + ke.getCause();
+ LOG.error(msg, ke);
+ TaskMonitor.get().createStatus(msg);
+ }
+ }
+
+ /**
+ * Clean the nodes of completed schema change table.
+ * @param path
+ * @throws KeeperException
+ */
+ private void cleanProcessedTableNode(String path) throws KeeperException {
+ if (sleepTimeMillis > 0) {
+ try {
+ LOG.debug("Master schema change tracker sleeping for "
+ + sleepTimeMillis);
+ Thread.sleep(sleepTimeMillis);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ ZKUtil.deleteNodeRecursively(this.watcher, path);
+ LOG.debug("Deleted all nodes for path " + path);
+
+ }
+
+ /**
+ * Exclude a RS from schema change request (if applicable)
+ * We will exclude a RS from schema change request processing if 1) RS
+ * has online regions for the table AND 2) RS went down mid-flight
+ * during schema change process. We don't have to deal with RS going
+ * down mid-flight during a schema change as the online regions from
+ * the dead RS will get reassigned to some other RS and the
+ * process of reassign inherently takes care of the schema change as well.
+ * @param serverName
+ */
+ public void excludeRegionServerForSchemaChanges(String serverName) {
+ try {
+ MonitoredTask status = TaskMonitor.get().createStatus(
+ "Processing schema change exclusion for region server = " + serverName);
+ List<String> tables =
+ ZKUtil.listChildrenNoWatch(watcher, watcher.schemaZNode);
+ if (tables == null || tables.isEmpty()) {
+ String msg = "No schema change in progress. Skipping exclusion for " +
+ "server = "+ serverName;
+ LOG.debug(msg);
+ status.setStatus(msg);
+ return ;
+ }
+ for (String tableName : tables) {
+ excludeRegionServer(tableName, serverName, status);
+ }
+ } catch(KeeperException ke) {
+ LOG.error("KeeperException during excludeRegionServerForSchemaChanges", ke);
+ } catch(IOException ioe) {
+ LOG.error("IOException during excludeRegionServerForSchemaChanges", ioe);
+
+ }
+ }
+
+ /**
+ * Check whether a schema change is in progress for a given table on a
+ * given RS.
+ * @param tableName
+ * @param serverName
+ * @return TRUE is this RS is currently processing a schema change request
+ * for the table.
+ * @throws KeeperException
+ */
+ private boolean isSchemaChangeApplicableFor(String tableName,
+ String serverName)
+ throws KeeperException {
+ List<String> servers = ZKUtil.listChildrenAndWatchThem(watcher,
+ getSchemaChangeNodePathForTable(tableName));
+ return (servers.contains(serverName));
+ }
+
+ /**
+ * Exclude a region server for a table (if applicable) from schema change processing.
+ * @param tableName
+ * @param serverName
+ * @param status
+ * @throws KeeperException
+ * @throws IOException
+ */
+ private void excludeRegionServer(String tableName, String serverName,
+ MonitoredTask status)
+ throws KeeperException, IOException {
+ if (isSchemaChangeApplicableFor(tableName, serverName)) {
+ String msg = "Excluding RS " + serverName + " from schema change process" +
+ " for table = " + tableName;
+ LOG.debug(msg);
+ status.setStatus(msg);
+ SchemaChangeTracker.SchemaAlterStatus sas =
+ getRSSchemaAlterStatus(tableName, serverName);
+ if (sas == null) {
+ LOG.debug("SchemaAlterStatus is NULL for table = " + tableName
+ + " server = " + serverName);
+ return;
+ }
+ // Set the status to IGNORED so we can process it accordingly.
+ sas.setCurrentAlterStatus(
+ SchemaChangeTracker.SchemaAlterStatus.AlterState.IGNORED);
+ LOG.debug("Updating the current schema status to " + sas);
+ String nodePath = getSchemaChangeNodePathForTableAndServer(tableName,
+ serverName);
+ ZKUtil.updateExistingNodeData(this.watcher,
+ nodePath, Writables.getBytes(sas), getZKNodeVersion(nodePath));
+ } else {
+ LOG.debug("Skipping exclusion of RS " + serverName
+ + " from schema change process"
+ + " for table = " + tableName
+ + " as it did not possess any online regions for the table");
+ }
+ processTableNode(tableName);
+ }
+
+ private int getZKNodeVersion(String nodePath) throws KeeperException {
+ return ZKUtil.checkExists(this.watcher, nodePath);
+ }
+
+ /**
+ * Create a new schema change ZK node.
+ * @param tableName Table name that is getting altered
+ * @throws KeeperException
+ */
+ public void createSchemaChangeNode(String tableName,
+ int numberOfRegions)
+ throws KeeperException, IOException {
+ MonitoredTask status = TaskMonitor.get().createStatus(
+ "Creating schema change node for table = " + tableName);
+ LOG.debug("Creating schema change node for table = "
+ + tableName + " Path = "
+ + getSchemaChangeNodePathForTable(tableName));
+ if (doesSchemaChangeNodeExists(tableName)) {
+ LOG.debug("Schema change node already exists for table = " + tableName
+ + " Deleting the schema change node.");
+ // If we already see a schema change node for this table we wait till the previous
+ // alter process is complete. Ideally, we need not wait and we could simply delete
+ // existing schema change node for this table and create new one. But then the
+ // RS cloud will not be able to process concurrent schema updates for the same table
+ // as they will be working with same set of online regions for this table. Meaning the
+ // second alter change will not see any online regions (as they were being closed and
+ // re opened by the first change) and will miss the second one.
+ // We either handle this at the RS level using explicit locks while processing a table
+ // or do it here. I prefer doing it here as it seems much simpler and cleaner.
+ while(doesSchemaChangeNodeExists(tableName)) {
+ try {
+ Thread.sleep(50);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ int rsCount = ZKUtil.getNumberOfChildren(this.watcher, watcher.rsZNode);
+ // if number of online RS = 0, we should not do anything!
+ if (rsCount <= 0) {
+ String msg = "Master is not seeing any online region servers. Aborting the " +
+ "schema change processing by region servers.";
+ LOG.debug(msg);
+ status.abort(msg);
+ } else {
+ LOG.debug("Master is seeing " + rsCount + " region servers online before " +
+ "the schema change process.");
+ MasterAlterStatus mas = new MasterAlterStatus(numberOfRegions,
+ getActiveRegionServersAsString());
+ LOG.debug("Master creating the master alter status = " + mas);
+ ZKUtil.createSetData(this.watcher,
+ getSchemaChangeNodePathForTable(tableName), Writables.getBytes(mas));
+ status.markComplete("Created the ZK node for schema change. Current Alter Status = "
+ + mas.toString());
+ ZKUtil.listChildrenAndWatchThem(this.watcher,
+ getSchemaChangeNodePathForTable(tableName));
+ }
+ }
+
+ private String getActiveRegionServersAsString() {
+ StringBuffer sbuf = new StringBuffer();
+ List<ServerName> currentRS =
+ masterServices.getRegionServerTracker().getOnlineServers();
+ for (ServerName serverName : currentRS) {
+ sbuf.append(serverName.getServerName());
+ sbuf.append(" ");
+ }
+ LOG.debug("Current list of RS to process the schema change = "
+ + sbuf.toString());
+ return sbuf.toString();
+ }
+
+ /**
+ * Create a new schema change ZK node.
+ * @param tableName
+ * @throws KeeperException
+ */
+ public boolean doesSchemaChangeNodeExists(String tableName)
+ throws KeeperException {
+ return ZKUtil.checkExists(watcher,
+ getSchemaChangeNodePathForTable(tableName)) != -1;
+ }
+
+ /**
+ * Check whether there are any schema change requests that are in progress now.
+ * We simply assume that a schema change is in progress if we see a ZK schema node for
+ * any table. We may revisit for fine grained checks such as check the current alter status
+ * et al, but it is not required now.
+ * @return
+ */
+ public boolean isSchemaChangeInProgress() {
+ try {
+ int schemaChangeCount = ZKUtil.getNumberOfChildren(this.watcher, watcher.schemaZNode);
+ return schemaChangeCount > 0;
+ } catch (KeeperException ke) {
+ LOG.debug("KeeperException while getting current schema change progress.");
+ // What do we do now??? currently reporting as false.
+ }
+ return false;
+ }
+
+ /**
+ * We get notified when a RS processes/or completed the schema change request.
+ * The path will be of the format /hbase/schema/<table name>
+ * @param path full path of the node whose children have changed
+ */
+ @Override
+ public void nodeChildrenChanged(String path) {
+ String tableName = null;
+ if (path.startsWith(watcher.schemaZNode) &&
+ !path.equals(watcher.schemaZNode)) {
+ try {
+ LOG.debug("NodeChildrenChanged Path = " + path);
+ tableName = path.substring(path.lastIndexOf("/")+1, path.length());
+ processTableNode(tableName);
+ } catch (KeeperException e) {
+ TaskMonitor.get().createStatus(
+ "MasterSchemaChangeTracker: ZK exception while processing " +
+ " nodeChildrenChanged() event for table = " + tableName
+ + " Cause = " + e.getCause());
+ LOG.error("MasterSchemaChangeTracker: Unexpected zk exception getting"
+ + " schema change nodes", e);
+ } catch(IOException ioe) {
+ TaskMonitor.get().createStatus(
+ "MasterSchemaChangeTracker: ZK exception while processing " +
+ " nodeChildrenChanged() event for table = " + tableName
+ + " Cause = " + ioe.getCause());
+ LOG.error("MasterSchemaChangeTracker: Unexpected IO exception getting"
+ + " schema change nodes", ioe);
+ }
+ }
+ }
+
+ /**
+ * We get notified as and when the RS cloud updates their ZK nodes with
+ * progress information. The path will be of the format
+ * /hbase/schema/<table name>/<RS host name>
+ * @param path
+ */
+ @Override
+ public void nodeDataChanged(String path) {
+ String tableName = null;
+ if (path.startsWith(watcher.schemaZNode) &&
+ !path.equals(watcher.schemaZNode)) {
+ try {
+ LOG.debug("NodeDataChanged Path = " + path);
+ String[] paths = path.split("/");
+ tableName = paths[3];
+ processTableNode(tableName);
+ } catch (KeeperException e) {
+ TaskMonitor.get().createStatus(
+ "MasterSchemaChangeTracker: ZK exception while processing " +
+ " nodeDataChanged() event for table = " + tableName
+ + " Cause = " + e.getCause());
+ LOG.error("MasterSchemaChangeTracker: Unexpected zk exception getting"
+ + " schema change nodes", e);
+ } catch(IOException ioe) {
+ TaskMonitor.get().createStatus(
+ "MasterSchemaChangeTracker: IO exception while processing " +
+ " nodeDataChanged() event for table = " + tableName
+ + " Cause = " + ioe.getCause());
+ LOG.error("MasterSchemaChangeTracker: Unexpected IO exception getting"
+ + " schema change nodes", ioe);
+
+ }
+ }
+ }
+
+ public String getSchemaChangeNodePathForTable(String tableName) {
+ return ZKUtil.joinZNode(watcher.schemaZNode, tableName);
+ }
+
+ /**
+ * Used only for tests. Do not use this. See TestInstantSchemaChange for more details
+ * on how this is getting used. This is primarily used to delay the schema complete
+ * processing by master so that we can test some complex scenarios such as
+ * master failover.
+ * @param sleepTimeMillis
+ */
+ public void setSleepTimeMillis(int sleepTimeMillis) {
+ this.sleepTimeMillis = sleepTimeMillis;
+ }
+
+ private String getSchemaChangeNodePathForTableAndServer(
+ String tableName, String regionServerName) {
+ return ZKUtil.joinZNode(getSchemaChangeNodePathForTable(tableName),
+ regionServerName);
+ }
+
+
+ /**
+ * Holds the current alter state for a table. Alter state includes the
+ * current alter status (INPROCESS, FAILURE or SUCCESS (success is not getting
+ * used now.), timestamp of alter request, number of hosts online at the time
+ * of alter request, number of online regions to process for the schema change
+ * request, number of processed regions and a list of region servers that
+ * actually processed the schema change request.
+ *
+ * Master keeps track of schema change requests using the alter status and
+ * periodically updates the alter status based on RS cloud processings.
+ */
+ public static class MasterAlterStatus implements Writable {
+
+ public enum AlterState {
+ INPROCESS, // Inprocess alter
+ SUCCESS, // completed alter
+ FAILURE // failure alter
+ }
+
+ private AlterState currentAlterStatus;
+ // TimeStamp
+ private long stamp;
+ private int numberOfRegionsToProcess;
+ private StringBuffer errorCause = new StringBuffer(" ");
+ private StringBuffer processedHosts = new StringBuffer(" ");
+ private String hostsToProcess;
+ private int numberOfRegionsProcessed = 0;
+
+ public MasterAlterStatus() {
+
+ }
+
+ public MasterAlterStatus(int numberOfRegions, String activeHosts) {
+ this.numberOfRegionsToProcess = numberOfRegions;
+ this.stamp = System.currentTimeMillis();
+ this.currentAlterStatus = AlterState.INPROCESS;
+ //this.rsToProcess = activeHosts;
+ this.hostsToProcess = activeHosts;
+ }
+
+ public AlterState getCurrentAlterStatus() {
+ return currentAlterStatus;
+ }
+
+ public void setCurrentAlterStatus(AlterState currentAlterStatus) {
+ this.currentAlterStatus = currentAlterStatus;
+ }
+
+ public long getStamp() {
+ return stamp;
+ }
+
+ public void setStamp(long stamp) {
+ this.stamp = stamp;
+ }
+
+ public int getNumberOfRegionsToProcess() {
+ return numberOfRegionsToProcess;
+ }
+
+ public void setNumberOfRegionsToProcess(int numberOfRegionsToProcess) {
+ this.numberOfRegionsToProcess = numberOfRegionsToProcess;
+ }
+
+ public int getNumberOfRegionsProcessed() {
+ return numberOfRegionsProcessed;
+ }
+
+ public void setNumberOfRegionsProcessed(int numberOfRegionsProcessed) {
+ this.numberOfRegionsProcessed += numberOfRegionsProcessed;
+ }
+
+ public String getHostsToProcess() {
+ return hostsToProcess;
+ }
+
+ public void setHostsToProcess(String hostsToProcess) {
+ this.hostsToProcess = hostsToProcess;
+ }
+
+ public String getErrorCause() {
+ return errorCause == null ? null : errorCause.toString();
+ }
+
+ public void setErrorCause(String errorCause) {
+ if (errorCause == null || errorCause.trim().length() <= 0) {
+ return;
+ }
+ if (this.errorCause == null) {
+ this.errorCause = new StringBuffer(errorCause);
+ } else {
+ this.errorCause.append(errorCause);
+ }
+ }
+
+ public String getProcessedHosts() {
+ return processedHosts.toString();
+ }
+
+ public void setProcessedHosts(String processedHosts) {
+ if (this.processedHosts == null) {
+ this.processedHosts = new StringBuffer(processedHosts);
+ } else {
+ this.processedHosts.append(" ").append(processedHosts);
+ }
+ }
+
+ /**
+ * Ignore or exempt a RS from schema change processing.
+ * Master will tweak the number of regions to process based on the
+ * number of online regions on the target RS and also remove the
+ * RS from list of hosts to process.
+ * @param schemaAlterStatus
+ */
+ private void ignoreRSForSchemaChange(
+ SchemaChangeTracker.SchemaAlterStatus schemaAlterStatus) {
+ LOG.debug("Removing RS " + schemaAlterStatus.getHostName()
+ + " from schema change process.");
+ hostsToProcess =
+ new String(hostsToProcess).replaceAll(schemaAlterStatus.getHostName(), "");
+ int ignoreRegionsCount = schemaAlterStatus.getNumberOfOnlineRegions();
+ LOG.debug("Current number of regions processed = "
+ + this.numberOfRegionsProcessed + " deducting ignored = "
+ + ignoreRegionsCount
+ + " final = " + (this.numberOfRegionsToProcess-ignoreRegionsCount));
+ if (this.numberOfRegionsToProcess > 0) {
+ this.numberOfRegionsToProcess -= ignoreRegionsCount;
+ } else {
+ LOG.debug("Number of regions to process is less than zero. This is odd");
+ }
+ }
+
+ /**
+ * Update the master alter status for this table based on RS alter status.
+ * @param schemaAlterStatus
+ */
+ public void update(SchemaChangeTracker.SchemaAlterStatus schemaAlterStatus) {
+ this.setProcessedHosts(schemaAlterStatus.getHostName());
+ SchemaChangeTracker.SchemaAlterStatus.AlterState rsState =
+ schemaAlterStatus.getCurrentAlterStatus();
+ switch(rsState) {
+ case FAILURE:
+ LOG.debug("Schema update failure Status = "
+ + schemaAlterStatus);
+ this.setCurrentAlterStatus(
+ MasterAlterStatus.AlterState.FAILURE);
+ this.setNumberOfRegionsProcessed(
+ schemaAlterStatus.getNumberOfRegionsProcessed());
+ this.setErrorCause(schemaAlterStatus.getErrorCause());
+ break;
+ case SUCCESS:
+ LOG.debug("Schema update SUCCESS Status = "
+ + schemaAlterStatus);
+ this.setNumberOfRegionsProcessed(
+ schemaAlterStatus.getNumberOfRegionsProcessed());
+ this.setCurrentAlterStatus(MasterAlterStatus.AlterState.SUCCESS);
+ break;
+ case IGNORED:
+ LOG.debug("Schema update IGNORED Updating regions to " +
+ "process count. Status = "+ schemaAlterStatus);
+ ignoreRSForSchemaChange(schemaAlterStatus);
+ break;
+ default:
+ break;
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ currentAlterStatus = AlterState.valueOf(in.readUTF());
+ stamp = in.readLong();
+ numberOfRegionsToProcess = in.readInt();
+ hostsToProcess = Bytes.toString(Bytes.readByteArray(in));
+ processedHosts = new StringBuffer(Bytes.toString(Bytes.readByteArray(in)));
+ errorCause = new StringBuffer(Bytes.toString(Bytes.readByteArray(in)));
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(currentAlterStatus.name());
+ out.writeLong(stamp);
+ out.writeInt(numberOfRegionsToProcess);
+ Bytes.writeByteArray(out, Bytes.toBytes(hostsToProcess));
+ Bytes.writeByteArray(out, Bytes.toBytes(processedHosts.toString()));
+ Bytes.writeByteArray(out, Bytes.toBytes(errorCause.toString()));
+ }
+
+ @Override
+ public String toString() {
+ return
+ " state= " + currentAlterStatus
+ + ", ts= " + stamp
+ + ", number of regions to process = " + numberOfRegionsToProcess
+ + ", number of regions processed = " + numberOfRegionsProcessed
+ + ", hosts = " + hostsToProcess
+ + " , processed hosts = " + processedHosts
+ + " , errorCause = " + errorCause;
+ }
+ }
+}