You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2013/05/09 12:45:32 UTC
svn commit: r1480586 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/master/snapshot/
test/java/org/apache/hadoop/hbase/master/cleaner/
test/java/org/apache/hadoop/hbase/master/snapshot/ test/j...
Author: mbertozzi
Date: Thu May 9 10:45:32 2013
New Revision: 1480586
URL: http://svn.apache.org/r1480586
Log:
HBASE-8446 Allow parallel snapshot of different tables
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu May 9 10:45:32 2013
@@ -192,7 +192,7 @@ Server {
private CatalogTracker catalogTracker;
// Cluster status zk tracker and local setter
private ClusterStatusTracker clusterStatusTracker;
-
+
// buffer for "fatal error" notices from region servers
// in the cluster. This is only used for assisting
// operations/debugging.
@@ -372,7 +372,7 @@ Server {
"(Also watching cluster state node)");
Thread.sleep(c.getInt("zookeeper.session.timeout", 180 * 1000));
}
-
+
}
/**
@@ -410,7 +410,7 @@ Server {
}
} catch (Throwable t) {
// HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
- if (t instanceof NoClassDefFoundError &&
+ if (t instanceof NoClassDefFoundError &&
t.getMessage().contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) {
// improved error message for this special case
abort("HBase is having a problem with its Hadoop jars. You may need to "
@@ -422,7 +422,7 @@ Server {
}
} finally {
startupStatus.cleanup();
-
+
stopChores();
// Wait for all the remaining region servers to report in IFF we were
// running a cluster shutdown AND we were NOT aborting.
@@ -445,7 +445,7 @@ Server {
/**
* Try becoming active master.
- * @param startupStatus
+ * @param startupStatus
* @return True if we could successfully become the active master.
* @throws InterruptedException
*/
@@ -526,7 +526,7 @@ Server {
* <li>Ensure assignment of root and meta regions<li>
* <li>Handle either fresh cluster start or master failover</li>
* </ol>
- * @param masterRecovery
+ * @param masterRecovery
*
* @throws IOException
* @throws InterruptedException
@@ -563,7 +563,7 @@ Server {
status.setStatus("Initializing ZK system trackers");
initializeZKBasedSystemTrackers();
-
+
if (!masterRecovery) {
// initialize master side coprocessors before we start handling requests
status.setStatus("Initializing master coprocessors");
@@ -667,7 +667,7 @@ Server {
// removing dead server with same hostname and port of rs which is trying to check in before
// master initialization. See HBASE-5916.
this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
-
+
if (!masterRecovery) {
if (this.cpHost != null) {
// don't let cp initialization errors kill the master
@@ -679,11 +679,11 @@ Server {
}
}
}
-
+
/**
* If ServerShutdownHandler is disabled, we enable it and expire those dead
* but not expired servers.
- *
+ *
* @throws IOException
*/
private void enableServerShutdownHandler() throws IOException {
@@ -692,7 +692,7 @@ Server {
this.serverManager.expireDeadNotExpiredServers();
}
}
-
+
/**
* Useful for testing purpose also where we have
* master restart scenarios.
@@ -863,7 +863,7 @@ Server {
fileSystemManager.splitMetaLog(sn);
fileSystemManager.splitLog(sn);
} else {
- fileSystemManager.splitAllLogs(sn);
+ fileSystemManager.splitAllLogs(sn);
}
serverManager.expireServer(sn);
}
@@ -937,7 +937,7 @@ Server {
* need to install an unexpected exception handler.
*/
private void startServiceThreads() throws IOException{
-
+
// Start the executor service pools
this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
conf.getInt("hbase.master.executor.openregion.threads", 5));
@@ -947,7 +947,7 @@ Server {
conf.getInt("hbase.master.executor.serverops.threads", 3));
this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
conf.getInt("hbase.master.executor.serverops.threads", 5));
-
+
// We depend on there being only one instance of this executor running
// at a time. To do concurrency, would need fencing of enable/disable of
// tables.
@@ -1204,11 +1204,11 @@ Server {
newValue = this.cpHost.preBalanceSwitch(newValue);
}
if (mode == BalanceSwitchMode.SYNC) {
- synchronized (this.balancer) {
+ synchronized (this.balancer) {
this.balanceSwitch = newValue;
}
} else {
- this.balanceSwitch = newValue;
+ this.balanceSwitch = newValue;
}
LOG.info("BalanceSwitch=" + newValue);
if (this.cpHost != null) {
@@ -1217,14 +1217,14 @@ Server {
} catch (IOException ioe) {
LOG.warn("Error flipping balance switch", ioe);
}
- return oldValue;
+ return oldValue;
}
-
+
@Override
public boolean synchronousBalanceSwitch(final boolean b) {
return switchBalancer(b, BalanceSwitchMode.SYNC);
}
-
+
@Override
public boolean balanceSwitch(final boolean b) {
return switchBalancer(b, BalanceSwitchMode.ASYNC);
@@ -1257,10 +1257,10 @@ Server {
} else {
dest = new ServerName(Bytes.toString(destServerName));
}
-
+
// Now we can do the move
RegionPlan rp = new RegionPlan(p.getFirst(), p.getSecond(), dest);
-
+
try {
checkInitialized();
if (this.cpHost != null) {
@@ -1347,7 +1347,7 @@ Server {
* @return Pair indicating the number of regions updated Pair.getFirst is the
* regions that are yet to be updated Pair.getSecond is the total number
* of regions of the table
- * @throws IOException
+ * @throws IOException
*/
public Pair<Integer, Integer> getAlterStatus(byte[] tableName)
throws IOException {
@@ -1699,7 +1699,7 @@ Server {
public AssignmentManager getAssignmentManager() {
return this.assignmentManager;
}
-
+
public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
return rsFatals;
}
@@ -1774,13 +1774,13 @@ Server {
public boolean isAborted() {
return this.abort;
}
-
+
void checkInitialized() throws PleaseHoldException {
if (!this.initialized) {
throw new PleaseHoldException("Master is initializing");
}
}
-
+
/**
* Report whether this master is currently the active master or not.
* If not active master, we are parked on ZK waiting to become active.
@@ -1850,8 +1850,8 @@ Server {
cpHost.postAssign(pair.getFirst());
}
}
-
-
+
+
public void assignRegion(HRegionInfo hri) {
assignmentManager.assign(hri, true);
@@ -1882,7 +1882,7 @@ Server {
}
/**
- * Get HTD array for given tables
+ * Get HTD array for given tables
* @param tableNames
* @return HTableDescriptor[]
*/
@@ -2184,7 +2184,7 @@ Server {
*/
@Override
public boolean isRestoreSnapshotDone(final HSnapshotDescription request) throws IOException {
- return !snapshotManager.isRestoringTable(request.getProto());
+ return snapshotManager.isRestoreDone(request.getProto());
}
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java Thu May 9 10:45:32 2013
@@ -37,6 +37,11 @@ public interface SnapshotSentinel {
public boolean isFinished();
/**
+ * @return -1 if the snapshot is in progress, otherwise the completion timestamp.
+ */
+ public long getCompletionTimestamp();
+
+ /**
* Actively cancel a running snapshot.
* @param why Reason for cancellation.
*/
@@ -54,4 +59,11 @@ public interface SnapshotSentinel {
*/
public ForeignException getExceptionIfFailed();
+ /**
+ * Rethrow the exception returned by {@link SnapshotSentinel#getExceptionIfFailed}.
+ * If there is no exception this is a no-op.
+ *
+ * @throws ForeignException all exceptions from remote sources are procedure exceptions
+ */
+ public void rethrowExceptionIfFailed() throws ForeignException;
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java Thu May 9 10:45:32 2013
@@ -151,6 +151,11 @@ public class CloneSnapshotHandler extend
}
@Override
+ public long getCompletionTimestamp() {
+ return this.status.getCompletionTimestamp();
+ }
+
+ @Override
public SnapshotDescription getSnapshot() {
return snapshot;
}
@@ -169,4 +174,9 @@ public class CloneSnapshotHandler extend
public ForeignException getExceptionIfFailed() {
return this.monitor.getException();
}
+
+ @Override
+ public void rethrowExceptionIfFailed() throws ForeignException {
+ monitor.rethrowException();
+ }
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java Thu May 9 10:45:32 2013
@@ -63,7 +63,7 @@ public class DisabledTableSnapshotHandle
* @throws IOException on unexpected error
*/
public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
- final MasterServices masterServices, final MasterMetrics metricsMaster) throws IOException {
+ final MasterServices masterServices, final MasterMetrics metricsMaster) {
super(snapshot, masterServices, metricsMaster);
// setup the timer
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java Thu May 9 10:45:32 2013
@@ -50,7 +50,7 @@ public class EnabledTableSnapshotHandler
private final ProcedureCoordinator coordinator;
public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master,
- final SnapshotManager manager, final MasterMetrics metricsMaster) throws IOException {
+ final SnapshotManager manager, final MasterMetrics metricsMaster) {
super(snapshot, master, metricsMaster);
this.coordinator = manager.getCoordinator();
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java Thu May 9 10:45:32 2013
@@ -159,6 +159,11 @@ public class RestoreSnapshotHandler exte
}
@Override
+ public long getCompletionTimestamp() {
+ return this.status.getCompletionTimestamp();
+ }
+
+ @Override
public SnapshotDescription getSnapshot() {
return snapshot;
}
@@ -174,7 +179,13 @@ public class RestoreSnapshotHandler exte
this.monitor.receive(new ForeignException(masterServices.getServerName().toString(), ce));
}
+ @Override
public ForeignException getExceptionIfFailed() {
return this.monitor.getException();
}
+
+ @Override
+ public void rethrowExceptionIfFailed() throws ForeignException {
+ monitor.rethrowException();
+ }
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java Thu May 9 10:45:32 2013
@@ -68,6 +68,7 @@ import org.apache.hadoop.hbase.snapshot.
import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.zookeeper.KeeperException;
@@ -89,6 +90,19 @@ public class SnapshotManager implements
/** By default, check to see if the snapshot is complete every WAKE MILLIS (ms) */
private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
+ /**
+ * Wait time before removing a finished sentinel from the in-progress map
+ *
+ * NOTE: This is used as a safety auto cleanup.
+ * The snapshot and restore handlers map entries are removed when a user asks if a snapshot or
+ * restore is completed. This operation is part of the HBaseAdmin snapshot/restore API flow.
+ * In case something fails on the client side and the snapshot/restore state is not reclaimed
+ * after a default timeout, the entry is removed from the in-progress map.
+ * At this point, if the user asks for the snapshot/restore status, the result will be
+ * snapshot done if exists or failed if it doesn't exists.
+ */
+ private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
+
/** Enable or disable snapshot support */
public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
@@ -110,10 +124,11 @@ public class SnapshotManager implements
/** Name of the operation to use in the controller */
public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
- // TODO - enable having multiple snapshots with multiple monitors/threads
- // this needs to be configuration based when running multiple snapshots is implemented
+ /** Conf key for # of threads used by the SnapshotManager thread pool */
+ private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
+
/** number of current operations running on the master */
- private static final int opThreads = 1;
+ private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
private boolean stopped;
private final long wakeFrequency;
@@ -124,18 +139,21 @@ public class SnapshotManager implements
// Is snapshot feature enabled?
private boolean isSnapshotSupported = false;
- // A reference to a handler. If the handler is non-null, then it is assumed that a snapshot is
- // in progress currently
- // TODO: this is a bad smell; likely replace with a collection in the future. Also this gets
- // reset by every operation.
- private TakeSnapshotHandler handler;
+ // Snapshot handlers map, with table name as key.
+ // The map is always accessed and modified under the object lock using synchronized.
+ // snapshotTable() will insert an Handler in the table.
+ // isSnapshotDone() will remove the handler requested if the operation is finished.
+ private Map<String, SnapshotSentinel> snapshotHandlers = new HashMap<String, SnapshotSentinel>();
+
+ // Restore Sentinels map, with table name as key.
+ // The map is always accessed and modified under the object lock using synchronized.
+ // restoreSnapshot()/cloneSnapshot() will insert an Handler in the table.
+ // isRestoreDone() will remove the handler requested if the operation is finished.
+ private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
private final Path rootDir;
private final ExecutorService executorService;
- // Restore Sentinels map, with table name as key
- private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
-
/**
* Construct a snapshot manager.
* @param master
@@ -152,6 +170,7 @@ public class SnapshotManager implements
Configuration conf = master.getConfiguration();
this.wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
long keepAliveTime = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
+ int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
// setup the default procedure coordinator
String name = master.getServerName().toString();
@@ -193,7 +212,7 @@ public class SnapshotManager implements
public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
}
-
+
/**
* Gets the list of all completed snapshots.
* @param snapshotDir snapshot directory
@@ -203,9 +222,8 @@ public class SnapshotManager implements
private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
// first create the snapshot root path and check to see if it exists
- if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
-
FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
// if there are no snapshots, return an empty list
if (!fs.exists(snapshotDir)) {
@@ -291,26 +309,8 @@ public class SnapshotManager implements
}
/**
- * Return the handler if it is currently running and has the same snapshot target name.
- * @param snapshot
- * @return null if doesn't match, else a live handler.
- */
- private synchronized TakeSnapshotHandler getTakeSnapshotHandler(SnapshotDescription snapshot) {
- TakeSnapshotHandler h = this.handler;
- if (h == null) {
- return null;
- }
-
- if (!h.getSnapshot().getName().equals(snapshot.getName())) {
- // specified snapshot is to the one currently running
- return null;
- }
-
- return h;
- }
-
- /**
* Check if the specified snapshot is done
+ *
* @param expected
* @return true if snapshot is ready to be restored, false if it is still being taken.
* @throws IOException IOException if error from HDFS or RPC
@@ -325,10 +325,20 @@ public class SnapshotManager implements
String ssString = SnapshotDescriptionUtils.toString(expected);
- // check to see if the sentinel exists
- TakeSnapshotHandler handler = getTakeSnapshotHandler(expected);
+ // check to see if the sentinel exists,
+ // and if the task is complete removes it from the in-progress snapshots map.
+ SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
+
+ // stop tracking "abandoned" handlers
+ cleanupSentinels();
+
if (handler == null) {
- // doesn't exist, check if it is already completely done.
+ // If there's no handler in the in-progress map, it means one of the following:
+ // - someone has already requested the snapshot state
+ // - the requested snapshot was completed long time ago (cleanupSentinels() timeout)
+ // - the snapshot was never requested
+ // In those cases returns to the user the "done state" if the snapshots exists on disk,
+ // otherwise raise an exception saying that the snapshot is not running and doesn't exist.
if (!isSnapshotCompleted(expected)) {
throw new UnknownSnapshotException("Snapshot " + ssString
+ " is not currently running or one of the known completed snapshots.");
@@ -339,7 +349,7 @@ public class SnapshotManager implements
// pass on any failure we find in the sentinel
try {
- handler.rethrowException();
+ handler.rethrowExceptionIfFailed();
} catch (ForeignException e) {
// Give some procedure info on an exception.
String status;
@@ -364,32 +374,19 @@ public class SnapshotManager implements
}
/**
- * Check to see if there are any snapshots in progress currently. Currently we have a
- * limitation only allowing a single snapshot attempt at a time.
- * @return <tt>true</tt> if there any snapshots in progress, <tt>false</tt> otherwise
- * @throws SnapshotCreationException if the snapshot failed
- */
- synchronized boolean isTakingSnapshot() throws SnapshotCreationException {
- // TODO later when we handle multiple there would be a map with ssname to handler.
- return handler != null && !handler.isFinished();
- }
-
- /**
* Check to see if the specified table has a snapshot in progress. Currently we have a
- * limitation only allowing a single snapshot attempt at a time.
+ * limitation only allowing a single snapshot per table at a time.
* @param tableName name of the table being snapshotted.
* @return <tt>true</tt> if there is a snapshot in progress on the specified table.
*/
- private boolean isTakingSnapshot(final String tableName) {
- if (handler != null && handler.getSnapshot().getTable().equals(tableName)) {
- return !handler.isFinished();
- }
- return false;
+ synchronized boolean isTakingSnapshot(final String tableName) {
+ SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
+ return handler != null && !handler.isFinished();
}
/**
* Check to make sure that we are OK to run the passed snapshot. Checks to make sure that we
- * aren't already running a snapshot.
+ * aren't already running a snapshot or restore on the requested table.
* @param snapshot description of the snapshot we want to start
* @throws HBaseSnapshotException if the filesystem could not be prepared to start the snapshot
*/
@@ -399,19 +396,21 @@ public class SnapshotManager implements
Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
// make sure we aren't already running a snapshot
- if (isTakingSnapshot()) {
+ if (isTakingSnapshot(snapshot.getTable())) {
+ SnapshotSentinel handler = this.snapshotHandlers.get(snapshot.getTable());
throw new SnapshotCreationException("Rejected taking "
+ SnapshotDescriptionUtils.toString(snapshot)
+ " because we are already running another snapshot "
- + SnapshotDescriptionUtils.toString(this.handler.getSnapshot()), snapshot);
+ + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
}
// make sure we aren't running a restore on the same table
if (isRestoringTable(snapshot.getTable())) {
+ SnapshotSentinel handler = restoreHandlers.get(snapshot.getTable());
throw new SnapshotCreationException("Rejected taking "
+ SnapshotDescriptionUtils.toString(snapshot)
+ " because we are already have a restore in progress on the same snapshot "
- + SnapshotDescriptionUtils.toString(this.handler.getSnapshot()), snapshot);
+ + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
}
try {
@@ -433,31 +432,64 @@ public class SnapshotManager implements
}
/**
+ * Take a snapshot of a disabled table.
+ * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
+ * @throws HBaseSnapshotException if the snapshot could not be started
+ */
+ private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
+ throws HBaseSnapshotException {
+ // setup the snapshot
+ prepareToTakeSnapshot(snapshot);
+
+ // set the snapshot to be a disabled snapshot, since the client doesn't know about that
+ snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
+
+ // Take the snapshot of the disabled table
+ DisabledTableSnapshotHandler handler =
+ new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
+ snapshotTable(snapshot, handler);
+ }
+
+ /**
* Take a snapshot of an enabled table.
- * <p>
- * The thread limitation on the executorService's thread pool for snapshots ensures the
- * snapshot won't be started if there is another snapshot already running. Does
- * <b>not</b> check to see if another snapshot of the same name already exists.
* @param snapshot description of the snapshot to take.
* @throws HBaseSnapshotException if the snapshot could not be started
*/
private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
throws HBaseSnapshotException {
- TakeSnapshotHandler handler;
+ // setup the snapshot
+ prepareToTakeSnapshot(snapshot);
+
+ // Take the snapshot of the enabled table
+ EnabledTableSnapshotHandler handler =
+ new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
+ snapshotTable(snapshot, handler);
+ }
+
+ /**
+ * Take a snapshot using the specified handler.
+ * On failure the snapshot temporary working directory is removed.
+ * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
+ * snapshot request if the table is busy with another snapshot/restore operation.
+ * @param snapshot the snapshot description
+ * @param handler the snapshot handler
+ */
+ private synchronized void snapshotTable(SnapshotDescription snapshot,
+ final TakeSnapshotHandler handler) throws HBaseSnapshotException {
try {
- handler = new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
+ handler.prepare();
this.executorService.submit(handler);
- this.handler = handler;
- } catch (IOException e) {
+ this.snapshotHandlers.put(snapshot.getTable(), handler);
+ } catch (Exception e) {
// cleanup the working directory by trying to delete it from the fs.
Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
try {
if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
- LOG.warn("Couldn't delete working directory (" + workingDir + " for snapshot:"
- + SnapshotDescriptionUtils.toString(snapshot));
+ LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+ SnapshotDescriptionUtils.toString(snapshot));
}
} catch (IOException e1) {
- LOG.warn("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+ LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
SnapshotDescriptionUtils.toString(snapshot));
}
// fail the snapshot
@@ -481,6 +513,9 @@ public class SnapshotManager implements
LOG.debug("No existing snapshot, attempting snapshot...");
+ // stop tracking "abandoned" handlers
+ cleanupSentinels();
+
// check to see if the table exists
HTableDescriptor desc = null;
try {
@@ -508,9 +543,6 @@ public class SnapshotManager implements
cpHost.preSnapshot(snapshot, desc);
}
- // setup the snapshot
- prepareToTakeSnapshot(snapshot);
-
// if the table is enabled, then have the RS run actually the snapshot work
AssignmentManager assignmentMgr = master.getAssignmentManager();
if (assignmentMgr.getZKTable().isEnabledTable(snapshot.getTable())) {
@@ -538,52 +570,21 @@ public class SnapshotManager implements
}
/**
- * Take a snapshot of a disabled table.
- * <p>
- * The thread limitation on the executorService's thread pool for snapshots ensures the
- * snapshot won't be started if there is another snapshot already running. Does
- * <b>not</b> check to see if another snapshot of the same name already exists.
- * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
- * @throws HBaseSnapshotException if the snapshot could not be started
- */
- private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
- throws HBaseSnapshotException {
-
- // set the snapshot to be a disabled snapshot, since the client doesn't know about that
- snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
-
- DisabledTableSnapshotHandler handler;
- try {
- handler = new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
- this.executorService.submit(handler);
- this.handler = handler;
- } catch (IOException e) {
- // cleanup the working directory by trying to delete it from the fs.
- Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
- try {
- if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
- LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
- SnapshotDescriptionUtils.toString(snapshot));
- }
- } catch (IOException e1) {
- LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
- SnapshotDescriptionUtils.toString(snapshot));
- }
- // fail the snapshot
- throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
- }
- }
-
- /**
* Set the handler for the current snapshot
* <p>
* Exposed for TESTING
+ * @param tableName
* @param handler handler the master should use
*
* TODO get rid of this if possible, repackaging, modify tests.
*/
- public synchronized void setSnapshotHandlerForTesting(TakeSnapshotHandler handler) {
- this.handler = handler;
+ public synchronized void setSnapshotHandlerForTesting(final String tableName,
+ final SnapshotSentinel handler) {
+ if (handler != null) {
+ this.snapshotHandlers.put(tableName, handler);
+ } else {
+ this.snapshotHandlers.remove(tableName);
+ }
}
/**
@@ -595,7 +596,9 @@ public class SnapshotManager implements
/**
* Check to see if the snapshot is one of the currently completed snapshots
- * @param expected snapshot to check
+ * Returns true if the snapshot exists in the "completed snapshots folder".
+ *
+ * @param snapshot expected snapshot to check
* @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
* not stored
* @throws IOException if the filesystem throws an unexpected exception,
@@ -619,7 +622,6 @@ public class SnapshotManager implements
*
* @param snapshot Snapshot Descriptor
* @param hTableDescriptor Table Descriptor of the table to create
- * @param waitTime timeout before considering the clone failed
*/
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
@@ -639,7 +641,7 @@ public class SnapshotManager implements
CloneSnapshotHandler handler =
new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
this.executorService.submit(handler);
- restoreHandlers.put(tableName, handler);
+ this.restoreHandlers.put(tableName, handler);
} catch (Exception e) {
String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
" on table=" + tableName;
@@ -669,8 +671,8 @@ public class SnapshotManager implements
HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
String tableName = reqSnapshot.getTable();
- // stop tracking completed restores
- cleanupRestoreSentinels();
+ // stop tracking "abandoned" handlers
+ cleanupSentinels();
// Execute the restore/clone operation
if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
@@ -710,7 +712,6 @@ public class SnapshotManager implements
*
* @param snapshot Snapshot Descriptor
* @param hTableDescriptor Table Descriptor
- * @param waitTime timeout before considering the restore failed
*/
private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
@@ -732,7 +733,8 @@ public class SnapshotManager implements
this.executorService.submit(handler);
restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
} catch (Exception e) {
- String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
+ String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(
+ snapshot) +
" on table=" + tableName;
LOG.error(msg, e);
throw new RestoreSnapshotException(msg, e);
@@ -745,79 +747,107 @@ public class SnapshotManager implements
* @param tableName table under restore
* @return <tt>true</tt> if there is a restore in progress of the specified table.
*/
- private boolean isRestoringTable(final String tableName) {
- SnapshotSentinel sentinel = restoreHandlers.get(tableName);
+ private synchronized boolean isRestoringTable(final String tableName) {
+ SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
return(sentinel != null && !sentinel.isFinished());
}
/**
- * Returns status of a restore request, specifically comparing source snapshot and target table
- * names. Throws exception if not a known snapshot.
+ * Returns the status of a restore operation.
+ * If the in-progress restore is failed throws the exception that caused the failure.
+ *
* @param snapshot
- * @return true if in progress, false if snapshot is completed.
- * @throws UnknownSnapshotException if specified source snapshot does not exit.
- * @throws IOException if there was some sort of IO failure
+ * @return false if in progress, true if restore is completed or not requested.
+ * @throws IOException if there was a failure during the restore
*/
- public boolean isRestoringTable(final SnapshotDescription snapshot) throws IOException {
- // check to see if the snapshot is already on the fs
- if (!isSnapshotCompleted(snapshot)) {
- throw new UnknownSnapshotException("Snapshot:" + snapshot.getName()
- + " is not one of the known completed snapshots.");
- }
+ public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
+ // check to see if the sentinel exists,
+ // and if the task is complete removes it from the in-progress restore map.
+ SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
+
+ // stop tracking "abandoned" handlers
+ cleanupSentinels();
- SnapshotSentinel sentinel = getRestoreSnapshotSentinel(snapshot.getTable());
if (sentinel == null) {
// there is no sentinel so restore is not in progress.
- return false;
- }
- if (!sentinel.getSnapshot().getName().equals(snapshot.getName())) {
- // another handler is trying to restore to the table, but it isn't the same snapshot source.
- return false;
+ return true;
}
LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
+ sentinel.getSnapshot().getName() + " table=" + snapshot.getTable());
- ForeignException e = sentinel.getExceptionIfFailed();
- if (e != null) throw e;
+
+ // If the restore is failed, rethrow the exception
+ sentinel.rethrowExceptionIfFailed();
// check to see if we are done
if (sentinel.isFinished()) {
LOG.debug("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
" has completed. Notifying the client.");
- return false;
+ return true;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
SnapshotDescriptionUtils.toString(snapshot));
}
- return true;
+ return false;
}
/**
- * Get the restore snapshot sentinel for the specified table
- * @param tableName table under restore
- * @return the restore snapshot handler
+ * Return the handler if it is currently live and has the same snapshot target name.
+ * The handler is removed from the sentinels map if completed.
+ * @param sentinels live handlers
+ * @param snapshot snapshot description
+ * @return null if doesn't match, else a live handler.
*/
- private synchronized SnapshotSentinel getRestoreSnapshotSentinel(final String tableName) {
- try {
- return restoreHandlers.get(tableName);
- } finally {
- cleanupRestoreSentinels();
+ private synchronized SnapshotSentinel removeSentinelIfFinished(
+ final Map<String, SnapshotSentinel> sentinels, final SnapshotDescription snapshot) {
+ SnapshotSentinel h = sentinels.get(snapshot.getTable());
+ if (h == null) {
+ return null;
}
+
+ if (!h.getSnapshot().getName().equals(snapshot.getName())) {
+ // specified snapshot is to the one currently running
+ return null;
+ }
+
+ // Remove from the "in-progress" list once completed
+ if (h.isFinished()) {
+ sentinels.remove(snapshot.getTable());
+ }
+
+ return h;
}
/**
- * Scan the restore handlers and remove the finished ones.
+ * Removes "abandoned" snapshot/restore requests.
+ * As part of the HBaseAdmin snapshot/restore API the operation status is checked until completed,
+ * and the in-progress maps are cleaned up when the status of a completed task is requested.
+ * To avoid having sentinels staying around for long time if something client side is failed,
+ * each operation tries to clean up the in-progress maps sentinels finished from a long time.
*/
- private synchronized void cleanupRestoreSentinels() {
- Iterator<Map.Entry<String, SnapshotSentinel>> it = restoreHandlers.entrySet().iterator();
+ private void cleanupSentinels() {
+ cleanupSentinels(this.snapshotHandlers);
+ cleanupSentinels(this.restoreHandlers);
+ }
+
+ /**
+ * Remove the sentinels that are marked as finished and the completion time
+ * has exceeded the removal timeout.
+ * @param sentinels map of sentinels to clean
+ */
+ private synchronized void cleanupSentinels(final Map<String, SnapshotSentinel> sentinels) {
+ long currentTime = EnvironmentEdgeManager.currentTimeMillis();
+ Iterator<Map.Entry<String, SnapshotSentinel>> it = sentinels.entrySet().iterator();
while (it.hasNext()) {
- Map.Entry<String, SnapshotSentinel> entry = it.next();
- SnapshotSentinel sentinel = entry.getValue();
- if (sentinel.isFinished()) {
- it.remove();
- }
+ Map.Entry<String, SnapshotSentinel> entry = it.next();
+ SnapshotSentinel sentinel = entry.getValue();
+ if (sentinel.isFinished() &&
+ (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
+ {
+ it.remove();
+ }
}
}
@@ -832,7 +862,9 @@ public class SnapshotManager implements
// make sure we get stop
this.stopped = true;
// pass the stop onto take snapshot handlers
- if (this.handler != null) this.handler.cancel(why);
+ for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
+ snapshotHandler.cancel(why);
+ }
// pass the stop onto all the restore handlers
for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
@@ -892,7 +924,7 @@ public class SnapshotManager implements
LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
}
-
+
// If the user has enabled the snapshot, we force the cleaners to be present
// otherwise we still need to check if cleaners are enabled or not and verify
// that there're no snapshot in the .snapshot folder.
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java Thu May 9 10:45:32 2013
@@ -84,7 +84,7 @@ public abstract class TakeSnapshotHandle
* @throws IOException on unexpected error
*/
public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices,
- final MasterMetrics metricsMaster) throws IOException {
+ final MasterMetrics metricsMaster) {
super(masterServices, EventType.C_M_SNAPSHOT_TABLE);
assert snapshot != null : "SnapshotDescription must not be nul1";
assert masterServices != null : "MasterServices must not be nul1";
@@ -99,8 +99,6 @@ public abstract class TakeSnapshotHandle
this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
this.monitor = new ForeignExceptionDispatcher();
- loadTableDescriptor(); // check that .tableinfo is present
-
// prepare the verify
this.verifier = new MasterSnapshotVerifier(masterServices, snapshot, rootDir);
// update the running tasks
@@ -119,6 +117,11 @@ public abstract class TakeSnapshotHandle
return htd;
}
+ public TakeSnapshotHandler prepare() throws Exception {
+ loadTableDescriptor(); // check that .tableinfo is present
+ return this;
+ }
+
/**
* Execute the core common portions of taking a snapshot. The {@link #snapshotRegions(List)}
* call should get implemented for each snapshot flavor.
@@ -228,6 +231,11 @@ public abstract class TakeSnapshotHandle
}
@Override
+ public long getCompletionTimestamp() {
+ return this.status.getCompletionTimestamp();
+ }
+
+ @Override
public SnapshotDescription getSnapshot() {
return snapshot;
}
@@ -238,6 +246,11 @@ public abstract class TakeSnapshotHandle
}
@Override
+ public void rethrowExceptionIfFailed() throws ForeignException {
+ monitor.rethrowException();
+ }
+
+ @Override
public void rethrowException() throws ForeignException {
monitor.rethrowException();
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java Thu May 9 10:45:32 2013
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.snapshot.
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.junit.After;
@@ -126,7 +127,7 @@ public class TestSnapshotFromMaster {
@Before
public void setup() throws Exception {
UTIL.createTable(TABLE_NAME, TEST_FAM);
- master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(null);
+ master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(STRING_TABLE_NAME, null);
}
@After
@@ -179,7 +180,7 @@ public class TestSnapshotFromMaster {
// and that we get the same issue, even if we specify a name
SnapshotDescription desc = SnapshotDescription.newBuilder()
- .setName(snapshotName).build();
+ .setName(snapshotName).setTable(STRING_TABLE_NAME).build();
SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(desc),
UnknownSnapshotException.class);
@@ -188,8 +189,11 @@ public class TestSnapshotFromMaster {
Mockito.when(mockHandler.getException()).thenReturn(null);
Mockito.when(mockHandler.getSnapshot()).thenReturn(desc);
Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true));
+ Mockito.when(mockHandler.getCompletionTimestamp())
+ .thenReturn(EnvironmentEdgeManager.currentTimeMillis());
- master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(mockHandler);
+ master.getSnapshotManagerForTesting()
+ .setSnapshotHandlerForTesting(STRING_TABLE_NAME, mockHandler);
// if we do a lookup without a snapshot name, we should fail - you should always know your name
SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(),
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java Thu May 9 10:45:32 2013
@@ -67,7 +67,8 @@ public class TestSnapshotManager {
return getNewManager(UTIL.getConfiguration());
}
- private SnapshotManager getNewManager(final Configuration conf) throws IOException, KeeperException {
+ private SnapshotManager getNewManager(final Configuration conf)
+ throws IOException, KeeperException {
Mockito.reset(services);
Mockito.when(services.getConfiguration()).thenReturn(conf);
Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
@@ -78,14 +79,18 @@ public class TestSnapshotManager {
@Test
public void testInProcess() throws KeeperException, IOException {
+ String tableName = "testTable";
SnapshotManager manager = getNewManager();
TakeSnapshotHandler handler = Mockito.mock(TakeSnapshotHandler.class);
- assertFalse("Manager is in process when there is no current handler", manager.isTakingSnapshot());
- manager.setSnapshotHandlerForTesting(handler);
+ assertFalse("Manager is in process when there is no current handler",
+ manager.isTakingSnapshot(tableName));
+ manager.setSnapshotHandlerForTesting(tableName, handler);
Mockito.when(handler.isFinished()).thenReturn(false);
- assertTrue("Manager isn't in process when handler is running", manager.isTakingSnapshot());
+ assertTrue("Manager isn't in process when handler is running",
+ manager.isTakingSnapshot(tableName));
Mockito.when(handler.isFinished()).thenReturn(true);
- assertFalse("Manager is process when handler isn't running", manager.isTakingSnapshot());
+ assertFalse("Manager is process when handler isn't running",
+ manager.isTakingSnapshot(tableName));
}
/**
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java?rev=1480586&r1=1480585&r2=1480586&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java Thu May 9 10:45:32 2013
@@ -251,11 +251,8 @@ public class TestFlushSnapshotFromClient
// load the table so we have some data
UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
// and wait until everything stabilizes
- HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
- List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
- for (HRegion region : onlineRegions) {
- region.waitForFlushesAndCompactions();
- }
+ waitForTableToBeOnline(TABLE_NAME);
+
String snapshotName = "flushSnapshotCreateListDestroy";
// test creating the snapshot
admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
@@ -305,32 +302,27 @@ public class TestFlushSnapshotFromClient
}
/**
- * Demonstrate that we reject snapshot requests if there is a snapshot currently running.
+ * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
+ * same table currently running and that concurrent snapshots on different tables can both
+ * succeed concurretly.
*/
@Test(timeout=60000)
public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
- int ssNum = 10;
+ final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
+ final byte[] TABLE2_NAME = Bytes.toBytes(STRING_TABLE2_NAME);
+
+ int ssNum = 20;
HBaseAdmin admin = UTIL.getHBaseAdmin();
// make sure we don't fail on listing snapshots
SnapshotTestingUtils.assertNoSnapshots(admin);
+ // create second testing table
+ UTIL.createTable(TABLE2_NAME, TEST_FAM);
// load the table so we have some data
UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
+ UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE2_NAME), TEST_FAM);
// and wait until everything stabilizes
- HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
- List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
- for (HRegion region : onlineRegions) {
- region.waitForFlushesAndCompactions();
- }
-
- // build descriptions
- SnapshotDescription[] descs = new SnapshotDescription[ssNum];
- for (int i = 0; i < ssNum; i++) {
- SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
- builder.setTable(STRING_TABLE_NAME);
- builder.setName("ss"+i);
- builder.setType(SnapshotDescription.Type.FLUSH);
- descs[i] = builder.build();
- }
+ waitForTableToBeOnline(TABLE_NAME);
+ waitForTableToBeOnline(TABLE2_NAME);
final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
// We'll have one of these per thread
@@ -355,6 +347,16 @@ public class TestFlushSnapshotFromClient
}
};
+ // build descriptions
+ SnapshotDescription[] descs = new SnapshotDescription[ssNum];
+ for (int i = 0; i < ssNum; i++) {
+ SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
+ builder.setTable((i % 2) == 0 ? STRING_TABLE_NAME : STRING_TABLE2_NAME);
+ builder.setName("ss"+i);
+ builder.setType(SnapshotDescription.Type.FLUSH);
+ descs[i] = builder.build();
+ }
+
// kick each off its own thread
for (int i=0 ; i < ssNum; i++) {
new Thread(new SSRunnable(descs[i])).start();
@@ -390,13 +392,36 @@ public class TestFlushSnapshotFromClient
LOG.info("Taken " + takenSize + " snapshots: " + taken);
assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
" issued many requests", takenSize < ssNum && takenSize > 0);
+
+ // Verify that there's at least one snapshot per table
+ int t1SnapshotsCount = 0;
+ int t2SnapshotsCount = 0;
+ for (SnapshotDescription ss : taken) {
+ if (ss.getTable().equals(STRING_TABLE_NAME)) {
+ t1SnapshotsCount++;
+ } else if (ss.getTable().equals(STRING_TABLE2_NAME)) {
+ t2SnapshotsCount++;
+ }
+ }
+ assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
+ assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
+
// delete snapshots so subsequent tests are clean.
for (SnapshotDescription ss : taken) {
admin.deleteSnapshot(ss.getName());
}
+ UTIL.deleteTable(TABLE2_NAME);
}
private void logFSTree(Path root) throws IOException {
FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
}
+
+ private void waitForTableToBeOnline(final byte[] tableName) throws IOException {
+ HRegionServer rs = UTIL.getRSForFirstRegionInTable(tableName);
+ List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
+ for (HRegion region : onlineRegions) {
+ region.waitForFlushesAndCompactions();
+ }
+ }
}