You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2013/05/09 12:46:29 UTC
svn commit: r1480587 - in /hbase/branches/0.95/hbase-server/src:
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/master/snapshot/
test/java/org/apache/hadoop/hbase/master/cleaner/
test/java/org/apache/hadoop/hbase/master/sna...
Author: mbertozzi
Date: Thu May 9 10:46:28 2013
New Revision: 1480587
URL: http://svn.apache.org/r1480587
Log:
HBASE-8446 Allow parallel snapshot of different tables
Modified:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu May 9 10:46:28 2013
@@ -2665,8 +2665,8 @@ MasterServices, Server {
try {
SnapshotDescription snapshot = request.getSnapshot();
IsRestoreSnapshotDoneResponse.Builder builder = IsRestoreSnapshotDoneResponse.newBuilder();
- boolean isRestoring = snapshotManager.isRestoringTable(snapshot);
- builder.setDone(!isRestoring);
+ boolean done = snapshotManager.isRestoreDone(request.getSnapshot());
+ builder.setDone(done);
return builder.build();
} catch (IOException e) {
throw new ServiceException(e);
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java Thu May 9 10:46:28 2013
@@ -37,6 +37,11 @@ public interface SnapshotSentinel {
public boolean isFinished();
/**
+ * @return -1 if the snapshot is in progress, otherwise the completion timestamp.
+ */
+ public long getCompletionTimestamp();
+
+ /**
* Actively cancel a running snapshot.
* @param why Reason for cancellation.
*/
@@ -54,4 +59,11 @@ public interface SnapshotSentinel {
*/
public ForeignException getExceptionIfFailed();
+ /**
+ * Rethrow the exception returned by {@link SnapshotSentinel#getExceptionIfFailed}.
+ * If there is no exception this is a no-op.
+ *
+ * @throws ForeignException all exceptions from remote sources are procedure exceptions
+ */
+ public void rethrowExceptionIfFailed() throws ForeignException;
}
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java Thu May 9 10:46:28 2013
@@ -70,8 +70,7 @@ public class CloneSnapshotHandler extend
public CloneSnapshotHandler(final MasterServices masterServices,
final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor,
- final MetricsMaster metricsMaster)
- throws NotAllMetaRegionsOnlineException, TableExistsException, IOException {
+ final MetricsMaster metricsMaster) {
super(masterServices, masterServices.getMasterFileSystem(), hTableDescriptor,
masterServices.getConfiguration(), null, masterServices);
this.metricsMaster = metricsMaster;
@@ -155,6 +154,11 @@ public class CloneSnapshotHandler extend
}
@Override
+ public long getCompletionTimestamp() {
+ return this.status.getCompletionTimestamp();
+ }
+
+ @Override
public SnapshotDescription getSnapshot() {
return snapshot;
}
@@ -173,4 +177,9 @@ public class CloneSnapshotHandler extend
public ForeignException getExceptionIfFailed() {
return this.monitor.getException();
}
+
+ @Override
+ public void rethrowExceptionIfFailed() throws ForeignException {
+ monitor.rethrowException();
+ }
}
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java Thu May 9 10:46:28 2013
@@ -62,10 +62,9 @@ public class DisabledTableSnapshotHandle
/**
* @param snapshot descriptor of the snapshot to take
* @param masterServices master services provider
- * @throws IOException on unexpected error
*/
public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
- final MasterServices masterServices, final MetricsMaster metricsMaster) throws IOException {
+ final MasterServices masterServices, final MetricsMaster metricsMaster) {
super(snapshot, masterServices, metricsMaster);
// setup the timer
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java Thu May 9 10:46:28 2013
@@ -50,7 +50,7 @@ public class EnabledTableSnapshotHandler
private final ProcedureCoordinator coordinator;
public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master,
- final SnapshotManager manager, final MetricsMaster metricsMaster) throws IOException {
+ final SnapshotManager manager, final MetricsMaster metricsMaster) {
super(snapshot, master, metricsMaster);
this.coordinator = manager.getCoordinator();
}
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java Thu May 9 10:46:28 2013
@@ -163,6 +163,11 @@ public class RestoreSnapshotHandler exte
}
@Override
+ public long getCompletionTimestamp() {
+ return this.status.getCompletionTimestamp();
+ }
+
+ @Override
public SnapshotDescription getSnapshot() {
return snapshot;
}
@@ -178,7 +183,13 @@ public class RestoreSnapshotHandler exte
this.monitor.receive(new ForeignException(masterServices.getServerName().toString(), ce));
}
+ @Override
public ForeignException getExceptionIfFailed() {
return this.monitor.getException();
}
+
+ @Override
+ public void rethrowExceptionIfFailed() throws ForeignException {
+ monitor.rethrowException();
+ }
}
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java Thu May 9 10:46:28 2013
@@ -69,6 +69,7 @@ import org.apache.hadoop.hbase.snapshot.
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.zookeeper.KeeperException;
@@ -90,6 +91,19 @@ public class SnapshotManager implements
/** By default, check to see if the snapshot is complete every WAKE MILLIS (ms) */
private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
+ /**
+ * Wait time before removing a finished sentinel from the in-progress map
+ *
+ * NOTE: This is used as a safety auto cleanup.
+ * The snapshot and restore handlers map entries are removed when a user asks if a snapshot or
+ * restore is completed. This operation is part of the HBaseAdmin snapshot/restore API flow.
+ * In case something fails on the client side and the snapshot/restore state is not reclaimed
+ * after a default timeout, the entry is removed from the in-progress map.
+ * At this point, if the user asks for the snapshot/restore status, the result will be
+ * snapshot done if exists or failed if it doesn't exists.
+ */
+ private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
+
/** Enable or disable snapshot support */
public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
@@ -111,10 +125,11 @@ public class SnapshotManager implements
/** Name of the operation to use in the controller */
public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
- // TODO - enable having multiple snapshots with multiple monitors/threads
- // this needs to be configuration based when running multiple snapshots is implemented
+ /** Conf key for # of threads used by the SnapshotManager thread pool */
+ private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
+
/** number of current operations running on the master */
- private static final int opThreads = 1;
+ private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
private boolean stopped;
private final long wakeFrequency;
@@ -125,18 +140,21 @@ public class SnapshotManager implements
// Is snapshot feature enabled?
private boolean isSnapshotSupported = false;
- // A reference to a handler. If the handler is non-null, then it is assumed that a snapshot is
- // in progress currently
- // TODO: this is a bad smell; likely replace with a collection in the future. Also this gets
- // reset by every operation.
- private TakeSnapshotHandler handler;
+ // Snapshot handlers map, with table name as key.
+ // The map is always accessed and modified under the object lock using synchronized.
+ // snapshotTable() will insert an Handler in the table.
+ // isSnapshotDone() will remove the handler requested if the operation is finished.
+ private Map<String, SnapshotSentinel> snapshotHandlers = new HashMap<String, SnapshotSentinel>();
+
+ // Restore Sentinels map, with table name as key.
+ // The map is always accessed and modified under the object lock using synchronized.
+ // restoreSnapshot()/cloneSnapshot() will insert an Handler in the table.
+ // isRestoreDone() will remove the handler requested if the operation is finished.
+ private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
private final Path rootDir;
private final ExecutorService executorService;
- // Restore Sentinels map, with table name as key
- private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
-
/**
* Construct a snapshot manager.
* @param master
@@ -153,6 +171,7 @@ public class SnapshotManager implements
Configuration conf = master.getConfiguration();
this.wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
long keepAliveTime = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
+ int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
// setup the default procedure coordinator
String name = master.getServerName().toString();
@@ -194,7 +213,7 @@ public class SnapshotManager implements
public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
}
-
+
/**
* Gets the list of all completed snapshots.
* @param snapshotDir snapshot directory
@@ -291,26 +310,8 @@ public class SnapshotManager implements
}
/**
- * Return the handler if it is currently running and has the same snapshot target name.
- * @param snapshot
- * @return null if doesn't match, else a live handler.
- */
- private synchronized TakeSnapshotHandler getTakeSnapshotHandler(SnapshotDescription snapshot) {
- TakeSnapshotHandler h = this.handler;
- if (h == null) {
- return null;
- }
-
- if (!h.getSnapshot().getName().equals(snapshot.getName())) {
- // specified snapshot is to the one currently running
- return null;
- }
-
- return h;
- }
-
- /**
* Check if the specified snapshot is done
+ *
* @param expected
* @return true if snapshot is ready to be restored, false if it is still being taken.
* @throws IOException IOException if error from HDFS or RPC
@@ -325,10 +326,20 @@ public class SnapshotManager implements
String ssString = ClientSnapshotDescriptionUtils.toString(expected);
- // check to see if the sentinel exists
- TakeSnapshotHandler handler = getTakeSnapshotHandler(expected);
+ // check to see if the sentinel exists,
+ // and if the task is complete removes it from the in-progress snapshots map.
+ SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
+
+ // stop tracking "abandoned" handlers
+ cleanupSentinels();
+
if (handler == null) {
- // doesn't exist, check if it is already completely done.
+ // If there's no handler in the in-progress map, it means one of the following:
+ // - someone has already requested the snapshot state
+ // - the requested snapshot was completed long time ago (cleanupSentinels() timeout)
+ // - the snapshot was never requested
+ // In those cases returns to the user the "done state" if the snapshots exists on disk,
+ // otherwise raise an exception saying that the snapshot is not running and doesn't exist.
if (!isSnapshotCompleted(expected)) {
throw new UnknownSnapshotException("Snapshot " + ssString
+ " is not currently running or one of the known completed snapshots.");
@@ -339,7 +350,7 @@ public class SnapshotManager implements
// pass on any failure we find in the sentinel
try {
- handler.rethrowException();
+ handler.rethrowExceptionIfFailed();
} catch (ForeignException e) {
// Give some procedure info on an exception.
String status;
@@ -364,32 +375,19 @@ public class SnapshotManager implements
}
/**
- * Check to see if there are any snapshots in progress currently. Currently we have a
- * limitation only allowing a single snapshot attempt at a time.
- * @return <tt>true</tt> if there any snapshots in progress, <tt>false</tt> otherwise
- * @throws SnapshotCreationException if the snapshot failed
- */
- synchronized boolean isTakingSnapshot() throws SnapshotCreationException {
- // TODO later when we handle multiple there would be a map with ssname to handler.
- return handler != null && !handler.isFinished();
- }
-
- /**
* Check to see if the specified table has a snapshot in progress. Currently we have a
- * limitation only allowing a single snapshot attempt at a time.
+ * limitation only allowing a single snapshot per table at a time.
* @param tableName name of the table being snapshotted.
* @return <tt>true</tt> if there is a snapshot in progress on the specified table.
*/
- private boolean isTakingSnapshot(final String tableName) {
- if (handler != null && handler.getSnapshot().getTable().equals(tableName)) {
- return !handler.isFinished();
- }
- return false;
+ synchronized boolean isTakingSnapshot(final String tableName) {
+ SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
+ return handler != null && !handler.isFinished();
}
/**
* Check to make sure that we are OK to run the passed snapshot. Checks to make sure that we
- * aren't already running a snapshot.
+ * aren't already running a snapshot or restore on the requested table.
* @param snapshot description of the snapshot we want to start
* @throws HBaseSnapshotException if the filesystem could not be prepared to start the snapshot
*/
@@ -399,19 +397,21 @@ public class SnapshotManager implements
Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
// make sure we aren't already running a snapshot
- if (isTakingSnapshot()) {
+ if (isTakingSnapshot(snapshot.getTable())) {
+ SnapshotSentinel handler = this.snapshotHandlers.get(snapshot.getTable());
throw new SnapshotCreationException("Rejected taking "
+ ClientSnapshotDescriptionUtils.toString(snapshot)
+ " because we are already running another snapshot "
- + ClientSnapshotDescriptionUtils.toString(this.handler.getSnapshot()), snapshot);
+ + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
}
// make sure we aren't running a restore on the same table
if (isRestoringTable(snapshot.getTable())) {
+ SnapshotSentinel handler = restoreHandlers.get(snapshot.getTable());
throw new SnapshotCreationException("Rejected taking "
+ ClientSnapshotDescriptionUtils.toString(snapshot)
+ " because we are already have a restore in progress on the same snapshot "
- + ClientSnapshotDescriptionUtils.toString(this.handler.getSnapshot()), snapshot);
+ + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
}
try {
@@ -433,31 +433,64 @@ public class SnapshotManager implements
}
/**
+ * Take a snapshot of a disabled table.
+ * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
+ * @throws HBaseSnapshotException if the snapshot could not be started
+ */
+ private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
+ throws HBaseSnapshotException {
+ // setup the snapshot
+ prepareToTakeSnapshot(snapshot);
+
+ // set the snapshot to be a disabled snapshot, since the client doesn't know about that
+ snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
+
+ // Take the snapshot of the disabled table
+ DisabledTableSnapshotHandler handler =
+ new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
+ snapshotTable(snapshot, handler);
+ }
+
+ /**
* Take a snapshot of an enabled table.
- * <p>
- * The thread limitation on the executorService's thread pool for snapshots ensures the
- * snapshot won't be started if there is another snapshot already running. Does
- * <b>not</b> check to see if another snapshot of the same name already exists.
* @param snapshot description of the snapshot to take.
* @throws HBaseSnapshotException if the snapshot could not be started
*/
private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
throws HBaseSnapshotException {
- TakeSnapshotHandler handler;
+ // setup the snapshot
+ prepareToTakeSnapshot(snapshot);
+
+ // Take the snapshot of the enabled table
+ EnabledTableSnapshotHandler handler =
+ new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
+ snapshotTable(snapshot, handler);
+ }
+
+ /**
+ * Take a snapshot using the specified handler.
+ * On failure the snapshot temporary working directory is removed.
+ * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
+ * snapshot request if the table is busy with another snapshot/restore operation.
+ * @param snapshot the snapshot description
+ * @param handler the snapshot handler
+ */
+ private synchronized void snapshotTable(SnapshotDescription snapshot,
+ final TakeSnapshotHandler handler) throws HBaseSnapshotException {
try {
- handler = new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster).prepare();
+ handler.prepare();
this.executorService.submit(handler);
- this.handler = handler;
+ this.snapshotHandlers.put(snapshot.getTable(), handler);
} catch (Exception e) {
// cleanup the working directory by trying to delete it from the fs.
Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
try {
if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
- LOG.warn("Couldn't delete working directory (" + workingDir + " for snapshot:"
- + ClientSnapshotDescriptionUtils.toString(snapshot));
+ LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+ ClientSnapshotDescriptionUtils.toString(snapshot));
}
} catch (IOException e1) {
- LOG.warn("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+ LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
ClientSnapshotDescriptionUtils.toString(snapshot));
}
// fail the snapshot
@@ -481,6 +514,9 @@ public class SnapshotManager implements
LOG.debug("No existing snapshot, attempting snapshot...");
+ // stop tracking "abandoned" handlers
+ cleanupSentinels();
+
// check to see if the table exists
HTableDescriptor desc = null;
try {
@@ -508,9 +544,6 @@ public class SnapshotManager implements
cpHost.preSnapshot(snapshot, desc);
}
- // setup the snapshot
- prepareToTakeSnapshot(snapshot);
-
// if the table is enabled, then have the RS run actually the snapshot work
AssignmentManager assignmentMgr = master.getAssignmentManager();
if (assignmentMgr.getZKTable().isEnabledTable(snapshot.getTable())) {
@@ -538,52 +571,21 @@ public class SnapshotManager implements
}
/**
- * Take a snapshot of a disabled table.
- * <p>
- * The thread limitation on the executorService's thread pool for snapshots ensures the
- * snapshot won't be started if there is another snapshot already running. Does
- * <b>not</b> check to see if another snapshot of the same name already exists.
- * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
- * @throws HBaseSnapshotException if the snapshot could not be started
- */
- private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
- throws HBaseSnapshotException {
-
- // set the snapshot to be a disabled snapshot, since the client doesn't know about that
- snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
-
- DisabledTableSnapshotHandler handler;
- try {
- handler = new DisabledTableSnapshotHandler(snapshot, master, metricsMaster).prepare();
- this.executorService.submit(handler);
- this.handler = handler;
- } catch (Exception e) {
- // cleanup the working directory by trying to delete it from the fs.
- Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
- try {
- if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
- LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
- ClientSnapshotDescriptionUtils.toString(snapshot));
- }
- } catch (IOException e1) {
- LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
- ClientSnapshotDescriptionUtils.toString(snapshot));
- }
- // fail the snapshot
- throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
- }
- }
-
- /**
* Set the handler for the current snapshot
* <p>
* Exposed for TESTING
+ * @param tableName
* @param handler handler the master should use
*
* TODO get rid of this if possible, repackaging, modify tests.
*/
- public synchronized void setSnapshotHandlerForTesting(TakeSnapshotHandler handler) {
- this.handler = handler;
+ public synchronized void setSnapshotHandlerForTesting(final String tableName,
+ final SnapshotSentinel handler) {
+ if (handler != null) {
+ this.snapshotHandlers.put(tableName, handler);
+ } else {
+ this.snapshotHandlers.remove(tableName);
+ }
}
/**
@@ -595,7 +597,9 @@ public class SnapshotManager implements
/**
* Check to see if the snapshot is one of the currently completed snapshots
- * @param expected snapshot to check
+ * Returns true if the snapshot exists in the "completed snapshots folder".
+ *
+ * @param snapshot expected snapshot to check
* @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
* not stored
* @throws IOException if the filesystem throws an unexpected exception,
@@ -619,7 +623,6 @@ public class SnapshotManager implements
*
* @param snapshot Snapshot Descriptor
* @param hTableDescriptor Table Descriptor of the table to create
- * @param waitTime timeout before considering the clone failed
*/
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
@@ -639,7 +642,7 @@ public class SnapshotManager implements
CloneSnapshotHandler handler =
new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster).prepare();
this.executorService.submit(handler);
- restoreHandlers.put(tableName, handler);
+ this.restoreHandlers.put(tableName, handler);
} catch (Exception e) {
String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
" on table=" + tableName;
@@ -669,8 +672,8 @@ public class SnapshotManager implements
HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
String tableName = reqSnapshot.getTable();
- // stop tracking completed restores
- cleanupRestoreSentinels();
+ // stop tracking "abandoned" handlers
+ cleanupSentinels();
// Execute the restore/clone operation
if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
@@ -710,14 +713,11 @@ public class SnapshotManager implements
*
* @param snapshot Snapshot Descriptor
* @param hTableDescriptor Table Descriptor
- * @param waitTime timeout before considering the restore failed
*/
private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
String tableName = hTableDescriptor.getNameAsString();
- // TODO: There is definite race condition for managing the single handler. We should fix
- // and remove the limitation of single snapshot / restore at a time.
// make sure we aren't running a snapshot on the same table
if (isTakingSnapshot(tableName)) {
throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
@@ -748,79 +748,107 @@ public class SnapshotManager implements
* @param tableName table under restore
* @return <tt>true</tt> if there is a restore in progress of the specified table.
*/
- private boolean isRestoringTable(final String tableName) {
- SnapshotSentinel sentinel = restoreHandlers.get(tableName);
+ private synchronized boolean isRestoringTable(final String tableName) {
+ SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
return(sentinel != null && !sentinel.isFinished());
}
/**
- * Returns status of a restore request, specifically comparing source snapshot and target table
- * names. Throws exception if not a known snapshot.
+ * Returns the status of a restore operation.
+ * If the in-progress restore is failed throws the exception that caused the failure.
+ *
* @param snapshot
- * @return true if in progress, false if snapshot is completed.
- * @throws UnknownSnapshotException if specified source snapshot does not exit.
- * @throws IOException if there was some sort of IO failure
+ * @return false if in progress, true if restore is completed or not requested.
+ * @throws IOException if there was a failure during the restore
*/
- public boolean isRestoringTable(final SnapshotDescription snapshot) throws IOException {
- // check to see if the snapshot is already on the fs
- if (!isSnapshotCompleted(snapshot)) {
- throw new UnknownSnapshotException("Snapshot:" + snapshot.getName()
- + " is not one of the known completed snapshots.");
- }
+ public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
+ // check to see if the sentinel exists,
+ // and if the task is complete removes it from the in-progress restore map.
+ SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
+
+ // stop tracking "abandoned" handlers
+ cleanupSentinels();
- SnapshotSentinel sentinel = getRestoreSnapshotSentinel(snapshot.getTable());
if (sentinel == null) {
// there is no sentinel so restore is not in progress.
- return false;
- }
- if (!sentinel.getSnapshot().getName().equals(snapshot.getName())) {
- // another handler is trying to restore to the table, but it isn't the same snapshot source.
- return false;
+ return true;
}
LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
+ sentinel.getSnapshot().getName() + " table=" + snapshot.getTable());
- ForeignException e = sentinel.getExceptionIfFailed();
- if (e != null) throw e;
+
+ // If the restore is failed, rethrow the exception
+ sentinel.rethrowExceptionIfFailed();
// check to see if we are done
if (sentinel.isFinished()) {
LOG.debug("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
" has completed. Notifying the client.");
- return false;
+ return true;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
ClientSnapshotDescriptionUtils.toString(snapshot));
}
- return true;
+ return false;
}
/**
- * Get the restore snapshot sentinel for the specified table
- * @param tableName table under restore
- * @return the restore snapshot handler
+ * Return the handler if it is currently live and has the same snapshot target name.
+ * The handler is removed from the sentinels map if completed.
+ * @param sentinels live handlers
+ * @param snapshot snapshot description
+ * @return null if doesn't match, else a live handler.
*/
- private synchronized SnapshotSentinel getRestoreSnapshotSentinel(final String tableName) {
- try {
- return restoreHandlers.get(tableName);
- } finally {
- cleanupRestoreSentinels();
+ private synchronized SnapshotSentinel removeSentinelIfFinished(
+ final Map<String, SnapshotSentinel> sentinels, final SnapshotDescription snapshot) {
+ SnapshotSentinel h = sentinels.get(snapshot.getTable());
+ if (h == null) {
+ return null;
+ }
+
+ if (!h.getSnapshot().getName().equals(snapshot.getName())) {
+ // specified snapshot is to the one currently running
+ return null;
}
+
+ // Remove from the "in-progress" list once completed
+ if (h.isFinished()) {
+ sentinels.remove(snapshot.getTable());
+ }
+
+ return h;
}
/**
- * Scan the restore handlers and remove the finished ones.
+ * Removes "abandoned" snapshot/restore requests.
+ * As part of the HBaseAdmin snapshot/restore API the operation status is checked until completed,
+ * and the in-progress maps are cleaned up when the status of a completed task is requested.
+ * To avoid having sentinels staying around for long time if something client side is failed,
+ * each operation tries to clean up the in-progress maps sentinels finished from a long time.
*/
- private synchronized void cleanupRestoreSentinels() {
- Iterator<Map.Entry<String, SnapshotSentinel>> it = restoreHandlers.entrySet().iterator();
+ private void cleanupSentinels() {
+ cleanupSentinels(this.snapshotHandlers);
+ cleanupSentinels(this.restoreHandlers);
+ }
+
+ /**
+ * Remove the sentinels that are marked as finished and the completion time
+ * has exceeded the removal timeout.
+ * @param sentinels map of sentinels to clean
+ */
+ private synchronized void cleanupSentinels(final Map<String, SnapshotSentinel> sentinels) {
+ long currentTime = EnvironmentEdgeManager.currentTimeMillis();
+ Iterator<Map.Entry<String, SnapshotSentinel>> it = sentinels.entrySet().iterator();
while (it.hasNext()) {
- Map.Entry<String, SnapshotSentinel> entry = it.next();
- SnapshotSentinel sentinel = entry.getValue();
- if (sentinel.isFinished()) {
- it.remove();
- }
+ Map.Entry<String, SnapshotSentinel> entry = it.next();
+ SnapshotSentinel sentinel = entry.getValue();
+ if (sentinel.isFinished() &&
+ (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
+ {
+ it.remove();
+ }
}
}
@@ -835,7 +863,9 @@ public class SnapshotManager implements
// make sure we get stop
this.stopped = true;
// pass the stop onto take snapshot handlers
- if (this.handler != null) this.handler.cancel(why);
+ for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
+ snapshotHandler.cancel(why);
+ }
// pass the stop onto all the restore handlers
for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
@@ -895,7 +925,7 @@ public class SnapshotManager implements
LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
}
-
+
// If the user has enabled the snapshot, we force the cleaners to be present
// otherwise we still need to check if cleaners are enabled or not and verify
// that there're no snapshot in the .snapshot folder.
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java Thu May 9 10:46:28 2013
@@ -87,10 +87,9 @@ public abstract class TakeSnapshotHandle
/**
* @param snapshot descriptor of the snapshot to take
* @param masterServices master services provider
- * @throws IOException on unexpected error
*/
public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices,
- final MetricsMaster metricsMaster) throws IOException {
+ final MetricsMaster metricsMaster) {
super(masterServices, EventType.C_M_SNAPSHOT_TABLE);
assert snapshot != null : "SnapshotDescription must not be nul1";
assert masterServices != null : "MasterServices must not be nul1";
@@ -264,6 +263,11 @@ public abstract class TakeSnapshotHandle
}
@Override
+ public long getCompletionTimestamp() {
+ return this.status.getCompletionTimestamp();
+ }
+
+ @Override
public SnapshotDescription getSnapshot() {
return snapshot;
}
@@ -274,6 +278,11 @@ public abstract class TakeSnapshotHandle
}
@Override
+ public void rethrowExceptionIfFailed() throws ForeignException {
+ monitor.rethrowException();
+ }
+
+ @Override
public void rethrowException() throws ForeignException {
monitor.rethrowException();
}
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java Thu May 9 10:46:28 2013
@@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.snapshot.
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.exceptions.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
import org.junit.AfterClass;
@@ -129,7 +130,7 @@ public class TestSnapshotFromMaster {
@Before
public void setup() throws Exception {
UTIL.createTable(TABLE_NAME, TEST_FAM);
- master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(null);
+ master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(STRING_TABLE_NAME, null);
}
@After
@@ -182,7 +183,8 @@ public class TestSnapshotFromMaster {
UnknownSnapshotException.class);
// and that we get the same issue, even if we specify a name
- SnapshotDescription desc = SnapshotDescription.newBuilder().setName(snapshotName).build();
+ SnapshotDescription desc = SnapshotDescription.newBuilder()
+ .setName(snapshotName).setTable(STRING_TABLE_NAME).build();
builder.setSnapshot(desc);
SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(),
UnknownSnapshotException.class);
@@ -192,8 +194,11 @@ public class TestSnapshotFromMaster {
Mockito.when(mockHandler.getException()).thenReturn(null);
Mockito.when(mockHandler.getSnapshot()).thenReturn(desc);
Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true));
+ Mockito.when(mockHandler.getCompletionTimestamp())
+ .thenReturn(EnvironmentEdgeManager.currentTimeMillis());
- master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(mockHandler);
+ master.getSnapshotManagerForTesting()
+ .setSnapshotHandlerForTesting(STRING_TABLE_NAME, mockHandler);
// if we do a lookup without a snapshot name, we should fail - you should always know your name
builder = IsSnapshotDoneRequest.newBuilder();
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java Thu May 9 10:46:28 2013
@@ -67,7 +67,8 @@ public class TestSnapshotManager {
return getNewManager(UTIL.getConfiguration());
}
- private SnapshotManager getNewManager(final Configuration conf) throws IOException, KeeperException {
+ private SnapshotManager getNewManager(final Configuration conf)
+ throws IOException, KeeperException {
Mockito.reset(services);
Mockito.when(services.getConfiguration()).thenReturn(conf);
Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
@@ -78,14 +79,18 @@ public class TestSnapshotManager {
@Test
public void testInProcess() throws KeeperException, IOException {
+ String tableName = "testTable";
SnapshotManager manager = getNewManager();
TakeSnapshotHandler handler = Mockito.mock(TakeSnapshotHandler.class);
- assertFalse("Manager is in process when there is no current handler", manager.isTakingSnapshot());
- manager.setSnapshotHandlerForTesting(handler);
+ assertFalse("Manager is in process when there is no current handler",
+ manager.isTakingSnapshot(tableName));
+ manager.setSnapshotHandlerForTesting(tableName, handler);
Mockito.when(handler.isFinished()).thenReturn(false);
- assertTrue("Manager isn't in process when handler is running", manager.isTakingSnapshot());
+ assertTrue("Manager isn't in process when handler is running",
+ manager.isTakingSnapshot(tableName));
Mockito.when(handler.isFinished()).thenReturn(true);
- assertFalse("Manager is process when handler isn't running", manager.isTakingSnapshot());
+ assertFalse("Manager is process when handler isn't running",
+ manager.isTakingSnapshot(tableName));
}
/**
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java?rev=1480587&r1=1480586&r2=1480587&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java Thu May 9 10:46:28 2013
@@ -260,11 +260,8 @@ public class TestFlushSnapshotFromClient
// load the table so we have some data
UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
// and wait until everything stabilizes
- HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
- List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
- for (HRegion region : onlineRegions) {
- region.waitForFlushesAndCompactions();
- }
+ waitForTableToBeOnline(TABLE_NAME);
+
String snapshotName = "flushSnapshotCreateListDestroy";
// test creating the snapshot
admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
@@ -314,32 +311,27 @@ public class TestFlushSnapshotFromClient
}
/**
- * Demonstrate that we reject snapshot requests if there is a snapshot currently running.
+ * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
+ * same table currently running and that concurrent snapshots on different tables can both
+ * succeed concurretly.
*/
@Test(timeout=60000)
public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
- int ssNum = 10;
+ final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
+ final byte[] TABLE2_NAME = Bytes.toBytes(STRING_TABLE2_NAME);
+
+ int ssNum = 20;
HBaseAdmin admin = UTIL.getHBaseAdmin();
// make sure we don't fail on listing snapshots
SnapshotTestingUtils.assertNoSnapshots(admin);
+ // create second testing table
+ UTIL.createTable(TABLE2_NAME, TEST_FAM);
// load the table so we have some data
UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
+ UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE2_NAME), TEST_FAM);
// and wait until everything stabilizes
- HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
- List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
- for (HRegion region : onlineRegions) {
- region.waitForFlushesAndCompactions();
- }
-
- // build descriptions
- SnapshotDescription[] descs = new SnapshotDescription[ssNum];
- for (int i = 0; i < ssNum; i++) {
- SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
- builder.setTable(STRING_TABLE_NAME);
- builder.setName("ss"+i);
- builder.setType(SnapshotDescription.Type.FLUSH);
- descs[i] = builder.build();
- }
+ waitForTableToBeOnline(TABLE_NAME);
+ waitForTableToBeOnline(TABLE2_NAME);
final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
// We'll have one of these per thread
@@ -365,6 +357,16 @@ public class TestFlushSnapshotFromClient
}
};
+ // build descriptions
+ SnapshotDescription[] descs = new SnapshotDescription[ssNum];
+ for (int i = 0; i < ssNum; i++) {
+ SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
+ builder.setTable((i % 2) == 0 ? STRING_TABLE_NAME : STRING_TABLE2_NAME);
+ builder.setName("ss"+i);
+ builder.setType(SnapshotDescription.Type.FLUSH);
+ descs[i] = builder.build();
+ }
+
// kick each off its own thread
for (int i=0 ; i < ssNum; i++) {
new Thread(new SSRunnable(descs[i])).start();
@@ -400,13 +402,36 @@ public class TestFlushSnapshotFromClient
LOG.info("Taken " + takenSize + " snapshots: " + taken);
assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
" issued many requests", takenSize < ssNum && takenSize > 0);
+
+ // Verify that there's at least one snapshot per table
+ int t1SnapshotsCount = 0;
+ int t2SnapshotsCount = 0;
+ for (SnapshotDescription ss : taken) {
+ if (ss.getTable().equals(STRING_TABLE_NAME)) {
+ t1SnapshotsCount++;
+ } else if (ss.getTable().equals(STRING_TABLE2_NAME)) {
+ t2SnapshotsCount++;
+ }
+ }
+ assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
+ assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
+
// delete snapshots so subsequent tests are clean.
for (SnapshotDescription ss : taken) {
admin.deleteSnapshot(ss.getName());
}
+ UTIL.deleteTable(TABLE2_NAME);
}
private void logFSTree(Path root) throws IOException {
FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
}
+
+ private void waitForTableToBeOnline(final byte[] tableName) throws IOException {
+ HRegionServer rs = UTIL.getRSForFirstRegionInTable(tableName);
+ List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
+ for (HRegion region : onlineRegions) {
+ region.waitForFlushesAndCompactions();
+ }
+ }
}