You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2013/02/13 19:36:05 UTC
svn commit: r1445813 [2/2] - in /hbase/branches/hbase-7290/hbase-server/src:
main/java/org/apache/hadoop/hbase/errorhandling/
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/master/handler/
main/java/org/apache/hadoop/hbase/...
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/manage/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/manage/SnapshotManager.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/manage/SnapshotManager.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/manage/SnapshotManager.java Wed Feb 13 18:36:03 2013
@@ -17,79 +17,175 @@
*/
package org.apache.hadoop.hbase.master.snapshot.manage;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.SnapshotSentinel;
import org.apache.hadoop.hbase.master.snapshot.CloneSnapshotHandler;
import org.apache.hadoop.hbase.master.snapshot.DisabledTableSnapshotHandler;
import org.apache.hadoop.hbase.master.snapshot.RestoreSnapshotHandler;
+import org.apache.hadoop.hbase.master.snapshot.TakeSnapshotHandler;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.exception.HBaseSnapshotException;
-import org.apache.hadoop.hbase.snapshot.exception.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.exception.RestoreSnapshotException;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
+import org.apache.hadoop.hbase.snapshot.exception.SnapshotCreationException;
+import org.apache.hadoop.hbase.snapshot.exception.SnapshotDoesNotExistException;
+import org.apache.hadoop.hbase.snapshot.exception.SnapshotExistsException;
+import org.apache.hadoop.hbase.snapshot.exception.TablePartiallyOpenException;
+import org.apache.hadoop.hbase.snapshot.exception.UnknownSnapshotException;
+import org.apache.hadoop.hbase.snapshot.restore.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+
+import com.google.protobuf.ServiceException;
/**
- * This class monitors the whole process of snapshots via ZooKeeper. There is only one
- * SnapshotMonitor for the master.
+ * This class manages the procedure of taking and restoring snapshots. There is only one
+ * SnapshotManager for the master.
* <p>
- * Start monitoring a snapshot by calling method monitor() before the snapshot is started across the
- * cluster via ZooKeeper. SnapshotMonitor would stop monitoring this snapshot only if it is finished
- * or aborted.
+ * The class provides methods for monitoring in-progress snapshot actions.
* <p>
- * Note: There could be only one snapshot being processed and monitored at a time over the cluster.
- * Start monitoring a snapshot only when the previous one reaches an end status.
+ * Note: Currently there can only one snapshot being taken at a time over the cluster. This is a
+ * simplification in the current implementation.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class SnapshotManager implements Stoppable {
private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
- // TODO - enable having multiple snapshots with multiple monitors
+ /** By default, check to see if the snapshot is complete every WAKE MILLIS (ms) */
+ public static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
- // Restore Sentinels map, with table name as key
- private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
+ /**
+ * Conf key for # of ms elapsed between checks for snapshot errors while waiting for
+ * completion.
+ */
+ public static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
- private final MasterServices master;
- private SnapshotSentinel handler;
- private ExecutorService pool;
- private final Path rootDir;
+ /** By default, check to see if the snapshot is complete (ms) */
+ public static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 5000;
+
+ /**
+ * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for
+ * completion.
+ */
+ public static final String SNAPSHOT_TIMEMOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
+
+ /** Name of the operation to use in the controller */
+ public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
+
+ // TODO - enable having multiple snapshots with multiple monitors/threads
+ // this needs to be configuration based when running multiple snapshots is implemented
+ /** number of current operations running on the master */
+ private static final int opThreads = 1;
private boolean stopped;
+ private final long wakeFrequency;
+ private final MasterServices master; // Needed by TableEventHandlers
+
+ // A reference to a handler. If the handler is non-null, then it is assumed that a snapshot is
+ // in progress currently
+ // TODO: this is a bad smell; likely replace with a collection in the future. Also this gets
+ // reset by every operation.
+ private TakeSnapshotHandler handler;
+
+ private final Path rootDir;
+ private final ExecutorService executorService;
+
+ // Restore Sentinels map, with table name as key
+ private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
- public SnapshotManager(final MasterServices master, final ZooKeeperWatcher watcher,
- final ExecutorService executorService) throws KeeperException {
+ /**
+ * Construct a snapshot manager.
+ * @param master
+ * @param comms
+ */
+ public SnapshotManager(final MasterServices master) throws IOException {
this.master = master;
- this.pool = executorService;
+
+ // get the configuration for the coordinator
+ Configuration conf = master.getConfiguration();
+ this.wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
this.rootDir = master.getMasterFileSystem().getRootDir();
+ this.executorService = master.getExecutorService();
+ resetTempDir();
}
/**
- * Start running the manager.
- * <p>
- * <ol>
- * <li>Cleans up any snapshots in the snapshot/.tmp directory that were left from failed
- * snapshot/export attempts</li>
- * </ol>
+ * Gets the list of all completed snapshots.
+ * @return list of SnapshotDescriptions
+ * @throws IOException File system exception
+ */
+ public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
+ List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
+ // first create the snapshot root path and check to see if it exists
+ Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+ FileSystem fs = master.getMasterFileSystem().getFileSystem();
+
+ // if there are no snapshots, return an empty list
+ if (!fs.exists(snapshotDir)) {
+ return snapshotDescs;
+ }
+
+ // ignore all the snapshots in progress
+ FileStatus[] snapshots = fs.listStatus(snapshotDir,
+ new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
+ // loop through all the completed snapshots
+ for (FileStatus snapshot : snapshots) {
+ Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
+ // if the snapshot is bad
+ if (!fs.exists(info)) {
+ LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
+ continue;
+ }
+ FSDataInputStream in = null;
+ try {
+ in = fs.open(info);
+ SnapshotDescription desc = SnapshotDescription.parseFrom(in);
+ snapshotDescs.add(desc);
+ } catch (IOException e) {
+ LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
+ } finally {
+ if (in != null) {
+ in.close();
+ }
+ }
+ }
+ return snapshotDescs;
+ }
+
+ /**
+ * Cleans up any snapshots in the snapshot/.tmp directory that were left from failed
+ * snapshot attempts.
+ *
* @throws IOException if we can't reach the filesystem
*/
- public void start() throws IOException {
+ void resetTempDir() throws IOException {
// cleanup any existing snapshots.
Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
if (master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
@@ -98,16 +194,122 @@ public class SnapshotManager implements
}
/**
- * @return <tt>true</tt> if there is a snapshot currently being taken, <tt>false</tt> otherwise
+ * Delete the specified snapshot
+ * @param snapshot
+ * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
+ * @throws IOException For filesystem IOExceptions
+ */
+ public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
+
+ // call coproc pre hook
+ MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.preDeleteSnapshot(snapshot);
+ }
+
+ // check to see if it is completed
+ if (!isSnapshotCompleted(snapshot)) {
+ throw new SnapshotDoesNotExistException(snapshot);
+ }
+
+ String snapshotName = snapshot.getName();
+ LOG.debug("Deleting snapshot: " + snapshotName);
+ // first create the snapshot description and check to see if it exists
+ MasterFileSystem fs = master.getMasterFileSystem();
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+
+ // delete the existing snapshot
+ if (!fs.getFileSystem().delete(snapshotDir, true)) {
+ throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
+ }
+
+ // call coproc post hook
+ if (cpHost != null) {
+ cpHost.postDeleteSnapshot(snapshot);
+ }
+
+ }
+
+ /**
+ * Return the handler if it is currently running and has the same snapshot target name.
+ * @param snapshot
+ * @return null if doesn't match, else a live handler.
+ */
+ TakeSnapshotHandler getTakeSnapshotHandler(SnapshotDescription snapshot) {
+ TakeSnapshotHandler h = this.handler;
+ if (h == null) {
+ return null;
+ }
+
+ if (!h.getSnapshot().getName().equals(snapshot.getName())) {
+ // specified snapshot is to the one currently running
+ return null;
+ }
+
+ return h;
+ }
+
+ /**
+ * Check if the specified snapshot is done
+ * @param expected
+ * @return true if snapshot is ready to be restored, false if it is still being taken.
+ * @throws IOException IOException if error from HDFS or RPC
+ * @throws UnknownSnapshotException if snapshot is invalid or does not exist.
*/
- public boolean isTakingSnapshot() {
+ public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
+ // check the request to make sure it has a snapshot
+ if (expected == null) {
+ throw new UnknownSnapshotException(
+ "No snapshot name passed in request, can't figure out which snapshot you want to check.");
+ }
+
+ // check to see if the sentinel exists
+ TakeSnapshotHandler handler = getTakeSnapshotHandler(expected);
+ if (handler == null) {
+ // doesn't exist, check if it is already completely done.
+ if (!isSnapshotCompleted(expected)) {
+ throw new UnknownSnapshotException("Snapshot:" + expected.getName()
+ + " is not currently running or one of the known completed snapshots.");
+ }
+ // was done, return true;
+ return true;
+ }
+
+ // pass on any failure we find in the sentinel
+ try {
+ handler.rethrowException();
+ } catch (ForeignException e) {
+ throw new HBaseSnapshotException("Snapshot error from RS", e, expected);
+ }
+
+ // check to see if we are done
+ if (handler.isFinished()) {
+ LOG.debug("Snapshot '" + expected.getName() + "' has completed, notifying client.");
+ return true;
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug("Sentinel isn't finished with snapshot '" + expected.getName() + "'!");
+ }
+ return false;
+ }
+
+ /**
+ * Check to see if there are any snapshots in progress currently. Currently we have a
+ * limitation only allowing a single snapshot attempt at a time.
+ * @return <tt>true</tt> if there any snapshots in progress, <tt>false</tt> otherwise
+ * @throws SnapshotCreationException if the snapshot failed
+ */
+ synchronized boolean isTakingSnapshot() throws SnapshotCreationException {
+ // TODO later when we handle multiple there would be a map with ssname to handler.
return handler != null && !handler.isFinished();
}
- /*
+ /**
+ * Check to see if the specified table has a snapshot in progress. Currently we have a
+ * limitation only allowing a single snapshot attempt at a time.
+ * @param tableName name of the table being snapshotted.
* @return <tt>true</tt> if there is a snapshot in progress on the specified table.
*/
- public boolean isTakingSnapshot(final String tableName) {
+ private boolean isTakingSnapshot(final String tableName) {
if (handler != null && handler.getSnapshot().getTable().equals(tableName)) {
return !handler.isFinished();
}
@@ -138,7 +340,8 @@ public class SnapshotManager implements
}
try {
- // delete the working directory, since we aren't running the snapshot
+ // delete the working directory, since we aren't running the snapshot. Likely leftovers
+ // from a failed attempt.
fs.delete(workingDir, true);
// recreate the working directory for the snapshot
@@ -155,6 +358,77 @@ public class SnapshotManager implements
}
/**
+ * Take a snapshot based on the enabled/disabled state of the table.
+ *
+ * @param snapshot
+ * @throws HBaseSnapshotException when a snapshot specific exception occurs.
+ * @throws IOException when some sort of generic IO exception occurs.
+ */
+ public void takeSnapshot(SnapshotDescription snapshot) throws HBaseSnapshotException, IOException {
+ // check to see if we already completed the snapshot
+ if (isSnapshotCompleted(snapshot)) {
+ throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
+ + "' already stored on the filesystem.", snapshot);
+ }
+
+ LOG.debug("No existing snapshot, attempting snapshot...");
+
+ // check to see if the table exists
+ HTableDescriptor desc = null;
+ try {
+ desc = master.getTableDescriptors().get(snapshot.getTable());
+ } catch (FileNotFoundException e) {
+ String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
+ LOG.error(msg);
+ throw new SnapshotCreationException(msg, e, snapshot);
+ } catch (IOException e) {
+ throw new SnapshotCreationException("Error while geting table description for table "
+ + snapshot.getTable(), e, snapshot);
+ }
+ if (desc == null) {
+ throw new SnapshotCreationException("Table '" + snapshot.getTable()
+ + "' doesn't exist, can't take snapshot.", snapshot);
+ }
+
+ // set the snapshot version, now that we are ready to take it
+ snapshot = snapshot.toBuilder().setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
+ .build();
+
+ // call pre coproc hook
+ MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.preSnapshot(snapshot, desc);
+ }
+
+ // setup the snapshot
+ prepareToTakeSnapshot(snapshot);
+
+ // if the table is enabled, then have the RS run actually the snapshot work
+ AssignmentManager assignmentMgr = master.getAssignmentManager();
+ if (assignmentMgr.getZKTable().isEnabledTable(snapshot.getTable())) {
+ LOG.debug("Table enabled, starting distributed snapshot.");
+ throw new UnsupportedOperationException("Snapshots of enabled tables is not yet supported");
+ }
+ // For disabled table, snapshot is created by the master
+ else if (assignmentMgr.getZKTable().isDisabledTable(snapshot.getTable())) {
+ LOG.debug("Table is disabled, running snapshot entirely on master.");
+ snapshotDisabledTable(snapshot);
+ LOG.debug("Started snapshot: " + snapshot);
+ } else {
+ LOG.error("Can't snapshot table '" + snapshot.getTable()
+ + "', isn't open or closed, we don't know what to do!");
+ TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
+ + " isn't fully open.");
+ throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
+ }
+
+ // call post coproc hook
+ if (cpHost != null) {
+ cpHost.postSnapshot(snapshot, desc);
+ }
+ }
+
+ /**
* Take a snapshot of a disabled table.
* <p>
* Ensures the snapshot won't be started if there is another snapshot already running. Does
@@ -162,21 +436,19 @@ public class SnapshotManager implements
* @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
* @throws HBaseSnapshotException if the snapshot could not be started
*/
- public synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
+ private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
throws HBaseSnapshotException {
- // setup the snapshot
- prepareToTakeSnapshot(snapshot);
// set the snapshot to be a disabled snapshot, since the client doesn't know about that
snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
DisabledTableSnapshotHandler handler;
try {
- handler = new DisabledTableSnapshotHandler(snapshot, this.master, this.master);
+ handler = new DisabledTableSnapshotHandler(snapshot, this.master);
this.handler = handler;
- this.pool.submit(handler);
+ this.executorService.submit(handler);
} catch (IOException e) {
- // cleanup the working directory
+ // cleanup the working directory by trying to delete it from the fs.
Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
try {
if (this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
@@ -192,10 +464,35 @@ public class SnapshotManager implements
}
/**
- * @return the current handler for the snapshot
+ * Set the handler for the current snapshot
+ * <p>
+ * Exposed for TESTING
+ * @param handler handler the master should use
+ *
+ * TODO get rid of this if possible, repackaging, modify tests.
+ */
+ public synchronized void setSnapshotHandlerForTesting(TakeSnapshotHandler handler) {
+ this.handler = handler;
+ }
+
+ /**
+ * Check to see if the snapshot is one of the currently completed snapshots
+ * @param expected snapshot to check
+ * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
+ * not stored
+ * @throws IOException if the filesystem throws an unexpected exception,
+ * @throws IllegalArgumentException if snapshot name is invalid.
*/
- public SnapshotSentinel getCurrentSnapshotSentinel() {
- return this.handler;
+ private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
+ try {
+ final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+ FileSystem fs = master.getMasterFileSystem().getFileSystem();
+
+ // check to see if the snapshot already exists
+ return fs.exists(snapshotDir);
+ } catch (IllegalArgumentException iae) {
+ throw new UnknownSnapshotException("Unexpected exception thrown", iae);
+ }
}
/**
@@ -206,10 +503,9 @@ public class SnapshotManager implements
* @param hTableDescriptor Table Descriptor of the table to create
* @param waitTime timeout before considering the clone failed
*/
- public synchronized void cloneSnapshot(final SnapshotDescription snapshot,
+ synchronized void cloneSnapshot(final SnapshotDescription snapshot,
final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
String tableName = hTableDescriptor.getNameAsString();
- cleanupRestoreSentinels();
// make sure we aren't running a snapshot on the same table
if (isTakingSnapshot(tableName)) {
@@ -224,7 +520,7 @@ public class SnapshotManager implements
try {
CloneSnapshotHandler handler =
new CloneSnapshotHandler(master, snapshot, hTableDescriptor);
- this.pool.submit(handler);
+ this.executorService.submit(handler);
restoreHandlers.put(tableName, handler);
} catch (Exception e) {
String msg = "Couldn't clone the snapshot=" + snapshot + " on table=" + tableName;
@@ -234,6 +530,62 @@ public class SnapshotManager implements
}
/**
+ * Restore the specified snapshot
+ * @param reqSnapshot
+ * @throws IOException
+ */
+ public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
+ FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
+ MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+
+ // check if the snapshot exists
+ if (!fs.exists(snapshotDir)) {
+ LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
+ throw new SnapshotDoesNotExistException(reqSnapshot);
+ }
+
+ // read snapshot information
+ SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+ String tableName = reqSnapshot.getTable();
+
+ // stop tracking completed restores
+ cleanupRestoreSentinels();
+
+ // Execute the restore/clone operation
+ if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
+ if (master.getAssignmentManager().getZKTable().isEnabledTable(fsSnapshot.getTable())) {
+ throw new UnsupportedOperationException("Table '" +
+ fsSnapshot.getTable() + "' must be disabled in order to perform a restore operation.");
+ }
+
+ // call coproc pre hook
+ if (cpHost != null) {
+ cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
+ }
+ restoreSnapshot(fsSnapshot, snapshotTableDesc);
+ LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
+
+ if (cpHost != null) {
+ cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
+ }
+ } else {
+ HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc,
+ Bytes.toBytes(tableName));
+ if (cpHost != null) {
+ cpHost.preCloneSnapshot(reqSnapshot, htd);
+ }
+ cloneSnapshot(fsSnapshot, htd);
+ LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
+
+ if (cpHost != null) {
+ cpHost.postCloneSnapshot(reqSnapshot, htd);
+ }
+ }
+ }
+
+ /**
* Restore the specified snapshot.
* The restore will fail if the destination table has a snapshot or restore in progress.
*
@@ -241,10 +593,9 @@ public class SnapshotManager implements
* @param hTableDescriptor Table Descriptor
* @param waitTime timeout before considering the restore failed
*/
- public synchronized void restoreSnapshot(final SnapshotDescription snapshot,
+ private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
String tableName = hTableDescriptor.getNameAsString();
- cleanupRestoreSentinels();
// make sure we aren't running a snapshot on the same table
if (isTakingSnapshot(tableName)) {
@@ -259,7 +610,7 @@ public class SnapshotManager implements
try {
RestoreSnapshotHandler handler =
new RestoreSnapshotHandler(master, snapshot, hTableDescriptor);
- this.pool.submit(handler);
+ this.executorService.submit(handler);
restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
} catch (Exception e) {
String msg = "Couldn't restore the snapshot=" + snapshot + " on table=" + tableName;
@@ -274,17 +625,59 @@ public class SnapshotManager implements
* @param tableName table under restore
* @return <tt>true</tt> if there is a restore in progress of the specified table.
*/
- public boolean isRestoringTable(final String tableName) {
+ private boolean isRestoringTable(final String tableName) {
SnapshotSentinel sentinel = restoreHandlers.get(tableName);
return(sentinel != null && !sentinel.isFinished());
}
/**
+ * Returns status of a restore request, specifically comparing source snapshot and target table
+ * names. Throws exception if not a known snapshot.
+ * @param snapshot
+ * @return true if in progress, false if is not.
+ * @throws UnknownSnapshotException if specified source snapshot does not exit.
+ * @throws IOException if there was some sort of IO failure
+ */
+ public boolean isRestoringTable(final SnapshotDescription snapshot) throws IOException {
+ // check to see if the snapshot is already on the fs
+ if (!isSnapshotCompleted(snapshot)) {
+ throw new UnknownSnapshotException("Snapshot:" + snapshot.getName()
+ + " is not one of the known completed snapshots.");
+ }
+
+ SnapshotSentinel sentinel = getRestoreSnapshotSentinel(snapshot.getTable());
+ if (sentinel == null) {
+ // there is no sentinel so restore is not in progress.
+ return false;
+ }
+ if (!sentinel.getSnapshot().getName().equals(snapshot.getName())) {
+ // another handler is trying to restore to the table, but it isn't the same snapshot source.
+ return false;
+ }
+
+ LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
+ + sentinel.getSnapshot().getName() + " table=" + snapshot.getTable());
+ ForeignException e = sentinel.getExceptionIfFailed();
+ if (e != null) throw e;
+
+ // check to see if we are done
+ if (sentinel.isFinished()) {
+ LOG.debug("Restore snapshot=" + snapshot + " has completed. Notifying the client.");
+ return false;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sentinel is not yet finished with restoring snapshot=" + snapshot);
+ }
+ return true;
+ }
+
+ /**
* Get the restore snapshot sentinel for the specified table
* @param tableName table under restore
* @return the restore snapshot handler
*/
- public synchronized SnapshotSentinel getRestoreSnapshotSentinel(final String tableName) {
+ private synchronized SnapshotSentinel getRestoreSnapshotSentinel(final String tableName) {
try {
return restoreHandlers.get(tableName);
} finally {
@@ -306,17 +699,22 @@ public class SnapshotManager implements
}
}
+ //
+ // Implementing Stoppable interface
+ //
+
@Override
public void stop(String why) {
// short circuit
if (this.stopped) return;
// make sure we get stop
this.stopped = true;
- // pass the stop onto all the listeners
- if (this.handler != null) this.handler.stop(why);
+ // pass the stop onto take snapshot handlers
+ if (this.handler != null) this.handler.cancel(why);
+
// pass the stop onto all the restore handlers
for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
- restoreHandler.stop(why);
+ restoreHandler.cancel(why);
}
}
@@ -324,14 +722,4 @@ public class SnapshotManager implements
public boolean isStopped() {
return this.stopped;
}
-
- /**
- * Set the handler for the current snapshot
- * <p>
- * Exposed for TESTING
- * @param handler handler the master should use
- */
- public void setSnapshotHandlerForTesting(SnapshotSentinel handler) {
- this.handler = handler;
- }
}
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Wed Feb 13 18:36:03 2013
@@ -2526,8 +2526,8 @@ public class HRegion implements HeapSize
}
/**
- * Replaces any KV timestamps set to {@link HConstants#LATEST_TIMESTAMP}
- * with the provided current timestamp.
+ * Replaces any KV timestamps set to {@link HConstants#LATEST_TIMESTAMP} with the provided current
+ * timestamp.
*/
void updateKVTimestamps(
final Iterable<List<KeyValue>> keyLists, final byte[] now) {
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/TakeSnapshotUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/TakeSnapshotUtils.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/TakeSnapshotUtils.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/TakeSnapshotUtils.java Wed Feb 13 18:36:03 2013
@@ -32,19 +32,17 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
+import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.server.errorhandling.ExceptionListener;
-import org.apache.hadoop.hbase.server.errorhandling.OperationAttemptTimer;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.exception.CorruptedSnapshotException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSUtils.DirFilter;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
@@ -118,11 +116,11 @@ public class TakeSnapshotUtils {
* @return the timer to use update to signal the start and end of the snapshot
*/
@SuppressWarnings("rawtypes")
- public static OperationAttemptTimer getMasterTimerAndBindToMonitor(SnapshotDescription snapshot,
- Configuration conf, ExceptionListener monitor) {
+ public static TimeoutExceptionInjector getMasterTimerAndBindToMonitor(SnapshotDescription snapshot,
+ Configuration conf, ForeignExceptionListener monitor) {
long maxTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(),
SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME);
- return new OperationAttemptTimer(monitor, maxTime, snapshot);
+ return new TimeoutExceptionInjector(monitor, maxTime);
}
/**
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/CopyRecoveredEditsTask.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/CopyRecoveredEditsTask.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/CopyRecoveredEditsTask.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/CopyRecoveredEditsTask.java Wed Feb 13 18:36:03 2013
@@ -28,9 +28,9 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.server.snapshot.error.SnapshotExceptionSnare;
/**
* Copy over each of the files in a region's recovered.edits directory to the region's snapshot
@@ -56,18 +56,18 @@ public class CopyRecoveredEditsTask exte
* @param regionDir directory for the region to examine for edits
* @param snapshotRegionDir directory for the region in the snapshot
*/
- public CopyRecoveredEditsTask(SnapshotDescription snapshot, SnapshotExceptionSnare monitor,
+ public CopyRecoveredEditsTask(SnapshotDescription snapshot, ForeignExceptionDispatcher monitor,
FileSystem fs, Path regionDir, Path snapshotRegionDir) {
- super(snapshot, monitor, "Copy recovered.edits for region:" + regionDir.getName());
+ super(snapshot, monitor);
this.fs = fs;
this.regiondir = regionDir;
this.outputDir = HLogUtil.getRegionDirRecoveredEditsDir(snapshotRegionDir);
}
@Override
- public void process() throws IOException {
+ public Void call() throws IOException {
NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(this.fs, regiondir);
- if (files == null || files.size() == 0) return;
+ if (files == null || files.size() == 0) return null;
// copy over each file.
// this is really inefficient (could be trivially parallelized), but is
@@ -83,7 +83,8 @@ public class CopyRecoveredEditsTask exte
FileUtil.copy(fs, source, fs, out, true, fs.getConf());
// check for errors to the running operation after each file
- this.failOnError();
+ this.rethrowException();
}
+ return null;
}
}
\ No newline at end of file
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/ReferenceRegionHFilesTask.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/ReferenceRegionHFilesTask.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/ReferenceRegionHFilesTask.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/ReferenceRegionHFilesTask.java Wed Feb 13 18:36:03 2013
@@ -26,9 +26,9 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.server.snapshot.TakeSnapshotUtils;
-import org.apache.hadoop.hbase.server.snapshot.error.SnapshotExceptionSnare;
import org.apache.hadoop.hbase.util.FSUtils;
/**
@@ -54,8 +54,8 @@ public class ReferenceRegionHFilesTask e
* @param regionSnapshotDir directory in the snapshot to store region files
*/
public ReferenceRegionHFilesTask(final SnapshotDescription snapshot,
- SnapshotExceptionSnare monitor, Path regionDir, final FileSystem fs, Path regionSnapshotDir) {
- super(snapshot, monitor, "Reference hfiles for region:" + regionDir.getName());
+ ForeignExceptionDispatcher monitor, Path regionDir, final FileSystem fs, Path regionSnapshotDir) {
+ super(snapshot, monitor);
this.regiondir = regionDir;
this.fs = fs;
@@ -76,14 +76,14 @@ public class ReferenceRegionHFilesTask e
}
@Override
- public void process() throws IOException {
+ public Void call() throws IOException {
FileStatus[] families = FSUtils.listStatus(fs, regiondir, new FSUtils.FamilyDirFilter(fs));
// if no families, then we are done again
if (families == null || families.length == 0) {
LOG.info("No families under region directory:" + regiondir
+ ", not attempting to add references.");
- return;
+ return null;
}
// snapshot directories to store the hfile reference
@@ -109,6 +109,7 @@ public class ReferenceRegionHFilesTask e
// create a reference for each hfile
for (FileStatus hfile : hfiles) {
+ // references are 0-length files, relying on file name.
Path referenceFile = new Path(snapshotFamilyDir, hfile.getPath().getName());
LOG.debug("Creating reference for:" + hfile.getPath() + " at " + referenceFile);
if (!fs.createNewFile(referenceFile)) {
@@ -122,5 +123,6 @@ public class ReferenceRegionHFilesTask e
LOG.debug("and the snapshot directory:");
FSUtils.logFileSystemState(fs, snapshotDir, LOG);
}
+ return null;
}
}
\ No newline at end of file
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/ReferenceServerWALsTask.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/ReferenceServerWALsTask.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/ReferenceServerWALsTask.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/ReferenceServerWALsTask.java Wed Feb 13 18:36:03 2013
@@ -28,9 +28,10 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.server.snapshot.TakeSnapshotUtils;
-import org.apache.hadoop.hbase.server.snapshot.error.SnapshotExceptionSnare;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -41,7 +42,6 @@ import org.apache.hadoop.hbase.util.FSUt
@InterfaceStability.Evolving
public class ReferenceServerWALsTask extends SnapshotTask {
private static final Log LOG = LogFactory.getLog(ReferenceServerWALsTask.class);
- // XXX does this need to be HasThread?
private final FileSystem fs;
private final Configuration conf;
private final String serverName;
@@ -53,23 +53,28 @@ public class ReferenceServerWALsTask ext
* propagate errors found while running the task
* @param logDir log directory for the server. Name of the directory is taken as the name of the
* server
- * @param conf {@link Configuration} to extract fileystem information
+ * @param conf {@link Configuration} to extract filesystem information
* @param fs filesystem where the log files are stored and should be referenced
- * @throws IOException
*/
public ReferenceServerWALsTask(SnapshotDescription snapshot,
- SnapshotExceptionSnare failureListener, final Path logDir, final Configuration conf,
- final FileSystem fs) throws IOException {
- super(snapshot, failureListener, "Reference WALs for server:" + logDir.getName());
+ ForeignExceptionDispatcher failureListener, final Path logDir, final Configuration conf,
+ final FileSystem fs) {
+ super(snapshot, failureListener);
this.fs = fs;
this.conf = conf;
this.serverName = logDir.getName();
this.logDir = logDir;
}
+ /**
+ * Create reference files (empty files with the same path and file name as original).
+ * @throws IOException exception from hdfs or network problems
+ * @throws ForeignException exception from an external procedure
+ */
@Override
- public void process() throws IOException {
+ public Void call() throws IOException, ForeignException {
// TODO switch to using a single file to reference all required WAL files
+
// Iterate through each of the log files and add a reference to it.
// assumes that all the files under the server's logs directory is a log
FileStatus[] serverLogs = FSUtils.listStatus(fs, logDir, null);
@@ -80,12 +85,9 @@ public class ReferenceServerWALsTask ext
+ Arrays.toString(serverLogs));
for (FileStatus file : serverLogs) {
- this.failOnError();
+ this.rethrowException();
- // TODO - switch to using MonitoredTask
- // add the reference to the file
- // 0. Build a reference path based on the file name
- // get the current snapshot directory
+ // add the reference to the file. ex: hbase/.snapshots/.logs/<serverName>/<hlog>
Path rootDir = FSUtils.getRootDir(conf);
Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(this.snapshot, rootDir);
Path snapshotLogDir = TakeSnapshotUtils.getSnapshotHLogsDir(snapshotDir, serverName);
@@ -98,6 +100,8 @@ public class ReferenceServerWALsTask ext
}
LOG.debug("Completed WAL referencing for: " + file.getPath() + " to " + ref);
}
+
LOG.debug("Successfully completed WAL referencing for ALL files");
+ return null;
}
}
\ No newline at end of file
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/SnapshotTask.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/SnapshotTask.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/SnapshotTask.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/SnapshotTask.java Wed Feb 13 18:36:03 2013
@@ -17,65 +17,49 @@
*/
package org.apache.hadoop.hbase.server.snapshot.task;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.concurrent.Callable;
+
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.server.errorhandling.ExceptionCheckable;
-import org.apache.hadoop.hbase.server.snapshot.error.SnapshotExceptionSnare;
-import org.apache.hadoop.hbase.snapshot.exception.HBaseSnapshotException;
/**
* General snapshot operation taken on a regionserver
*/
-public abstract class SnapshotTask implements ExceptionCheckable<HBaseSnapshotException>, Runnable {
-
- private static final Log LOG = LogFactory.getLog(SnapshotTask.class);
-
- private final SnapshotExceptionSnare errorMonitor;
- private final String desc;
+public abstract class SnapshotTask implements ForeignExceptionSnare, Callable<Void>{
protected final SnapshotDescription snapshot;
+ protected final ForeignExceptionDispatcher errorMonitor;
/**
* @param snapshot Description of the snapshot we are going to operate on
* @param monitor listener interested in failures to the snapshot caused by this operation
- * @param description description of the task being run, for logging
*/
- public SnapshotTask(SnapshotDescription snapshot, SnapshotExceptionSnare monitor,
- String description) {
+ public SnapshotTask(SnapshotDescription snapshot, ForeignExceptionDispatcher monitor) {
+ assert monitor != null : "ForeignExceptionDispatcher must not be null!";
+ assert snapshot != null : "SnapshotDescription must not be null!";
this.snapshot = snapshot;
this.errorMonitor = monitor;
- this.desc = description;
}
- protected final void snapshotFailure(String message, Exception e) {
- this.errorMonitor.snapshotFailure(message, this.snapshot, e);
+ public void snapshotFailure(String message, Exception e) {
+ ForeignException ee = new ForeignException(message, e);
+ errorMonitor.receive(ee);
}
@Override
- public void failOnError() throws HBaseSnapshotException {
- this.errorMonitor.failOnError();
+ public void rethrowException() throws ForeignException {
+ this.errorMonitor.rethrowException();
}
@Override
- public boolean checkForError() {
- return this.errorMonitor.checkForError();
+ public boolean hasException() {
+ return this.errorMonitor.hasException();
}
@Override
- public void run() {
- try {
- LOG.debug("Running: " + desc);
- this.process();
- } catch (Exception e) {
- this.snapshotFailure("Failed to run " + this.desc, e);
- }
+ public ForeignException getException() {
+ return this.errorMonitor.getException();
}
-
- /**
- * Run the task for the snapshot.
- * @throws Exception if the task fails. Will be propagated to any other tasks watching the same
- * {@link SnapshotErrorListener}.
- */
- protected abstract void process() throws Exception;
}
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/TableInfoCopyTask.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/TableInfoCopyTask.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/TableInfoCopyTask.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/server/snapshot/task/TableInfoCopyTask.java Wed Feb 13 18:36:03 2013
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hbase.server.snapshot.task;
-import java.io.IOException;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -26,8 +24,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.server.snapshot.error.SnapshotExceptionSnare;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
@@ -45,31 +43,31 @@ public class TableInfoCopyTask extends S
/**
* Copy the table info for the given table into the snapshot
- * @param failureListener listen for errors while running the snapshot
+ * @param monitor listen for errors while running the snapshot
* @param snapshot snapshot for which we are copying the table info
* @param fs {@link FileSystem} where the tableinfo is stored (and where the copy will be written)
* @param rootDir root of the {@link FileSystem} where the tableinfo is stored
*/
- public TableInfoCopyTask(SnapshotExceptionSnare failureListener, SnapshotDescription snapshot,
- FileSystem fs, Path rootDir) {
- super(snapshot, failureListener, "Copy table info for table: " + snapshot.getTable());
+ public TableInfoCopyTask(ForeignExceptionDispatcher monitor,
+ SnapshotDescription snapshot, FileSystem fs, Path rootDir) {
+ super(snapshot, monitor);
this.rootDir = rootDir;
this.fs = fs;
}
@Override
- public void process() throws IOException {
+ public Void call() throws Exception {
LOG.debug("Running table info copy.");
- this.failOnError();
+ this.rethrowException();
LOG.debug("Attempting to copy table info for snapshot:" + this.snapshot);
// get the HTable descriptor
HTableDescriptor orig = FSTableDescriptors.getTableDescriptor(fs, rootDir,
Bytes.toBytes(this.snapshot.getTable()));
-
- this.failOnError();
+ this.rethrowException();
// write a copy of descriptor to the snapshot directory
Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
FSTableDescriptors.createTableDescriptorForTableDirectory(fs, snapshotDir, orig, false);
LOG.debug("Finished copying tableinfo.");
+ return null;
}
}
\ No newline at end of file
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java Wed Feb 13 18:36:03 2013
@@ -20,24 +20,17 @@ package org.apache.hadoop.hbase.snapshot
import java.io.IOException;
import java.util.HashSet;
-import java.util.TreeMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.TreeMap;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileStatus;
-
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.FSVisitor;
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/CorruptedSnapshotException.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/CorruptedSnapshotException.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/CorruptedSnapshotException.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/CorruptedSnapshotException.java Wed Feb 13 18:36:03 2013
@@ -46,4 +46,11 @@ public class CorruptedSnapshotException
public CorruptedSnapshotException(String message, SnapshotDescription snapshot) {
super(message, snapshot);
}
+
+ /**
+ * @param message message describing the exception
+ */
+ public CorruptedSnapshotException(String message) {
+ super(message, (SnapshotDescription)null);
+ }
}
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/HBaseSnapshotException.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/HBaseSnapshotException.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/HBaseSnapshotException.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/HBaseSnapshotException.java Wed Feb 13 18:36:03 2013
@@ -28,7 +28,7 @@ import org.apache.hadoop.hbase.protobuf.
@SuppressWarnings("serial")
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public abstract class HBaseSnapshotException extends HBaseIOException {
+public class HBaseSnapshotException extends HBaseIOException {
private SnapshotDescription description;
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/SnapshotDoesNotExistException.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/SnapshotDoesNotExistException.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/SnapshotDoesNotExistException.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/SnapshotDoesNotExistException.java Wed Feb 13 18:36:03 2013
@@ -40,6 +40,6 @@ public class SnapshotDoesNotExistExcepti
* @param desc expected snapshot to find
*/
public SnapshotDoesNotExistException(SnapshotDescription desc) {
- super("Snapshot doesn't exist on the filesystem", desc);
+ super("Snapshot '" + desc.getName() +"' doesn't exist on the filesystem", desc);
}
}
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/UnknownSnapshotException.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/UnknownSnapshotException.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/UnknownSnapshotException.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/exception/UnknownSnapshotException.java Wed Feb 13 18:36:03 2013
@@ -28,11 +28,15 @@ import org.apache.hadoop.classification.
@InterfaceStability.Evolving
public class UnknownSnapshotException extends HBaseSnapshotException {
-
/**
- * @param msg full infomration about the failure
+ * @param msg full information about the failure
*/
public UnknownSnapshotException(String msg) {
super(msg);
}
+
+ public UnknownSnapshotException(String msg, Exception e) {
+ super(msg, e);
+ }
+
}
\ No newline at end of file
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/restore/RestoreSnapshotHelper.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/restore/RestoreSnapshotHelper.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/restore/RestoreSnapshotHelper.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/restore/RestoreSnapshotHelper.java Wed Feb 13 18:36:03 2013
@@ -18,46 +18,35 @@
package org.apache.hadoop.hbase.snapshot.restore;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.TreeMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
-
+import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.catalog.MetaEditor;
-import org.apache.hadoop.hbase.server.snapshot.error.SnapshotExceptionSnare;
-import org.apache.hadoop.hbase.snapshot.exception.RestoreSnapshotException;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
-import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.FSVisitor;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
@@ -110,7 +99,7 @@ public class RestoreSnapshotHelper {
private final Map<byte[], byte[]> regionsMap =
new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
- private final SnapshotExceptionSnare monitor;
+ private final ForeignExceptionDispatcher monitor;
private final SnapshotDescription snapshotDesc;
private final Path snapshotDir;
@@ -126,7 +115,7 @@ public class RestoreSnapshotHelper {
final CatalogTracker catalogTracker,
final SnapshotDescription snapshotDescription, final Path snapshotDir,
final HTableDescriptor tableDescriptor, final Path tableDir,
- final SnapshotExceptionSnare monitor)
+ final ForeignExceptionDispatcher monitor)
{
this.fs = fs;
this.conf = conf;
@@ -155,7 +144,7 @@ public class RestoreSnapshotHelper {
// NOTE: we rely upon the region name as: "table name, start key, end key"
List<HRegionInfo> tableRegions = getTableRegions();
if (tableRegions != null) {
- monitor.failOnError();
+ monitor.rethrowException();
List<HRegionInfo> regionsToRestore = new LinkedList<HRegionInfo>();
List<HRegionInfo> regionsToRemove = new LinkedList<HRegionInfo>();
@@ -172,11 +161,11 @@ public class RestoreSnapshotHelper {
}
// Restore regions using the snapshot data
- monitor.failOnError();
+ monitor.rethrowException();
restoreRegions(regionsToRestore);
// Remove regions from the current table
- monitor.failOnError();
+ monitor.rethrowException();
ModifyRegionUtils.deleteRegions(fs, catalogTracker, regionsToRemove);
}
@@ -184,7 +173,7 @@ public class RestoreSnapshotHelper {
if (snapshotRegionNames.size() > 0) {
List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
- monitor.failOnError();
+ monitor.rethrowException();
for (String regionName: snapshotRegionNames) {
LOG.info("region to add: " + regionName);
Path regionDir = new Path(snapshotDir, regionName);
@@ -192,12 +181,12 @@ public class RestoreSnapshotHelper {
}
// Create new regions cloning from the snapshot
- monitor.failOnError();
+ monitor.rethrowException();
cloneRegions(regionsToAdd);
}
// Restore WALs
- monitor.failOnError();
+ monitor.rethrowException();
restoreWALs();
}
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Wed Feb 13 18:36:03 2013
@@ -952,13 +952,14 @@ public abstract class FSUtils {
this.fs = fs;
}
+ @Override
public boolean accept(Path p) {
boolean isValid = false;
try {
if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(p.toString())) {
isValid = false;
} else {
- isValid = this.fs.getFileStatus(p).isDir();
+ isValid = fs.getFileStatus(p).isDir();
}
} catch (IOException e) {
LOG.warn("An error occurred while verifying if [" + p.toString() +
@@ -969,6 +970,21 @@ public abstract class FSUtils {
}
/**
+ * Filter out paths that are hidden (start with '.') and are not directories.
+ */
+ public static class VisibleDirectory extends DirFilter {
+
+ public VisibleDirectory(FileSystem fs) {
+ super(fs);
+ }
+
+ @Override
+ public boolean accept(Path file) {
+ return super.accept(file) && !file.getName().startsWith(".");
+ }
+ }
+
+ /**
* Heuristic to determine whether is safe or not to open a file for append
* Looks both for dfs.support.append and use reflection to search
* for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()
@@ -1307,19 +1323,6 @@ public abstract class FSUtils {
}
/**
- * Log the current state of the filesystem from a certain root directory
- * @param fs filesystem to investigate
- * @param root root file/directory to start logging from
- * @param LOG log to output information
- * @throws IOException if an unexpected exception occurs
- */
- public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG)
- throws IOException {
- LOG.debug("Current file system:");
- logFSTree(LOG, fs, root, "|-");
- }
-
- /**
* Throw an exception if an action is not permitted by a user on a file.
*
* @param ugi
@@ -1356,6 +1359,19 @@ public abstract class FSUtils {
}
/**
+ * Log the current state of the filesystem from a certain root directory
+ * @param fs filesystem to investigate
+ * @param root root file/directory to start logging from
+ * @param LOG log to output information
+ * @throws IOException if an unexpected exception occurs
+ */
+ public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG)
+ throws IOException {
+ LOG.debug("Current file system:");
+ logFSTree(LOG, fs, root, "|-");
+ }
+
+ /**
* Recursive helper to log the state of the FS
*
* @see #logFileSystemState(FileSystem, Path, Log)
Modified: hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java Wed Feb 13 18:36:03 2013
@@ -38,8 +38,6 @@ import org.apache.hadoop.hbase.MediumTes
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
-import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.master.snapshot.DisabledTableSnapshotHandler;
import org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -51,8 +49,6 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
-import org.apache.hadoop.hbase.snapshot.exception.HBaseSnapshotException;
-import org.apache.hadoop.hbase.snapshot.exception.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.exception.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -79,10 +75,11 @@ public class TestSnapshotFromMaster {
private static final int NUM_RS = 2;
private static Path rootDir;
private static Path snapshots;
- private static Path archiveDir;
private static FileSystem fs;
private static HMaster master;
+ // for hfile archiving test.
+ private static Path archiveDir;
private static final String STRING_TABLE_NAME = "test";
private static final byte[] TEST_FAM = Bytes.toBytes("fam");
private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
@@ -118,7 +115,7 @@ public class TestSnapshotFromMaster {
conf.setInt("hbase.client.retries.number", 1);
// set the only HFile cleaner as the snapshot cleaner
conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
- SnapshotHFileCleaner.class.getCanonicalName());
+ SnapshotHFileCleaner.class.getCanonicalName());
conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod);
}
@@ -130,7 +127,6 @@ public class TestSnapshotFromMaster {
@After
public void tearDown() throws Exception {
-
UTIL.deleteTable(TABLE_NAME);
// delete the archive directory, if its exists
@@ -186,7 +182,7 @@ public class TestSnapshotFromMaster {
// set a mock handler to simulate a snapshot
DisabledTableSnapshotHandler mockHandler = Mockito.mock(DisabledTableSnapshotHandler.class);
- Mockito.when(mockHandler.getExceptionIfFailed()).thenReturn(null);
+ Mockito.when(mockHandler.getException()).thenReturn(null);
Mockito.when(mockHandler.getSnapshot()).thenReturn(desc);
Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true));
@@ -218,15 +214,6 @@ public class TestSnapshotFromMaster {
builder.setSnapshot(desc);
response = master.isSnapshotDone(null, builder.build());
assertTrue("Completed, on-disk snapshot not found", response.getDone());
-
- HBaseSnapshotException testException = new SnapshotCreationException("test fail", desc);
- Mockito.when(mockHandler.getExceptionIfFailed()).thenReturn(testException);
- try {
- master.isSnapshotDone(null, builder.build());
- fail("Master should have passed along snapshot error, but didn't");
- }catch(ServiceException e) {
- LOG.debug("Correctly got exception back from the master on failure: " + e.getMessage());
- }
}
@Test
Modified: hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/manage/TestSnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/manage/TestSnapshotManager.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/manage/TestSnapshotManager.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/manage/TestSnapshotManager.java Wed Feb 13 18:36:03 2013
@@ -18,27 +18,23 @@
package org.apache.hadoop.hbase.master.snapshot.manage;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.master.SnapshotSentinel;
-import org.apache.hadoop.hbase.master.snapshot.DisabledTableSnapshotHandler;
+import org.apache.hadoop.hbase.master.snapshot.TakeSnapshotHandler;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.snapshot.exception.SnapshotCreationException;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -49,11 +45,9 @@ import org.mockito.Mockito;
*/
@Category(SmallTests.class)
public class TestSnapshotManager {
- private static final Log LOG = LogFactory.getLog(TestSnapshotManager.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
MasterServices services = Mockito.mock(MasterServices.class);
- ZooKeeperWatcher watcher = Mockito.mock(ZooKeeperWatcher.class);
ExecutorService pool = Mockito.mock(ExecutorService.class);
MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class);
FileSystem fs;
@@ -63,23 +57,21 @@ public class TestSnapshotManager {
} catch (IOException e) {
throw new RuntimeException("Couldn't get test filesystem", e);
}
-
}
- private SnapshotManager getNewManager() throws KeeperException {
- Mockito.reset(services, watcher, pool);
+ private SnapshotManager getNewManager() throws KeeperException, IOException {
+ Mockito.reset(services);
+ Mockito.when(services.getConfiguration()).thenReturn(UTIL.getConfiguration());
Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
Mockito.when(mfs.getFileSystem()).thenReturn(fs);
Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir());
- return new SnapshotManager(services, watcher, pool);
+ return new SnapshotManager(services);
}
-
-
@Test
- public void testInProcess() throws KeeperException, SnapshotCreationException {
+ public void testInProcess() throws KeeperException, IOException {
SnapshotManager manager = getNewManager();
- SnapshotSentinel handler = Mockito.mock(SnapshotSentinel.class);
+ TakeSnapshotHandler handler = Mockito.mock(TakeSnapshotHandler.class);
assertFalse("Manager is in process when there is no current handler", manager.isTakingSnapshot());
manager.setSnapshotHandlerForTesting(handler);
Mockito.when(handler.isFinished()).thenReturn(false);
@@ -87,46 +79,4 @@ public class TestSnapshotManager {
Mockito.when(handler.isFinished()).thenReturn(true);
assertFalse("Manager is process when handler isn't running", manager.isTakingSnapshot());
}
-
- /**
- * Test that we stop the running disabled table snapshot by passing along an error to the error
- * handler.
- * @throws Exception
- */
- @Test
- public void testStopPropagation() throws Exception {
- // create a new orchestrator and hook up a listener
- SnapshotManager manager = getNewManager();
- FSUtils.setRootDir(UTIL.getConfiguration(), UTIL.getDataTestDir());
-
- // setup a mock snapshot to run
- String tableName = "some table";
- SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("testAbort")
- .setTable(tableName).build();
- // mock out all the expected call to the master services
- // this allows us to run must faster and without using a minicluster
-
- // ensure the table exists when we ask for it
- TableDescriptors tables = Mockito.mock(TableDescriptors.class);
- Mockito.when(services.getTableDescriptors()).thenReturn(tables);
- HTableDescriptor descriptor = Mockito.mock(HTableDescriptor.class);
- Mockito.when(tables.get(Mockito.anyString())).thenReturn(descriptor);
-
- // return the local file system as the backing to the MasterFileSystem
- MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class);
- Mockito.when(mfs.getFileSystem()).thenReturn(UTIL.getTestFileSystem());
- Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
- Mockito.when(services.getConfiguration()).thenReturn(UTIL.getConfiguration());
-
- // create a new handler that we will check for errors
- manager.snapshotDisabledTable(snapshot);
- // make sure we submitted the handler, but because its mocked, it doesn't run it.
- Mockito.verify(pool, Mockito.times(1)).submit(Mockito.any(DisabledTableSnapshotHandler.class));
-
- // pass along the stop notification
- manager.stop("stopping for test");
- SnapshotSentinel handler = manager.getCurrentSnapshotSentinel();
- assertNotNull("Snare didn't receive error notification from snapshot manager.",
- handler.getExceptionIfFailed());
- }
}
\ No newline at end of file
Modified: hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestCopyRecoveredEditsTask.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestCopyRecoveredEditsTask.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestCopyRecoveredEditsTask.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestCopyRecoveredEditsTask.java Wed Feb 13 18:36:03 2013
@@ -26,11 +26,11 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.server.snapshot.error.SnapshotExceptionSnare;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.exception.HBaseSnapshotException;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -48,7 +48,7 @@ public class TestCopyRecoveredEditsTask
public void testCopyFiles() throws Exception {
SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build();
- SnapshotExceptionSnare monitor = Mockito.mock(SnapshotExceptionSnare.class);
+ ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
FileSystem fs = UTIL.getTestFileSystem();
Path root = UTIL.getDataTestDir();
String regionName = "regionA";
@@ -75,7 +75,8 @@ public class TestCopyRecoveredEditsTask
CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir,
snapshotRegionDir);
- task.run();
+ CopyRecoveredEditsTask taskSpy = Mockito.spy(task);
+ taskSpy.call();
Path snapshotEdits = HLogUtil.getRegionDirRecoveredEditsDir(snapshotRegionDir);
FileStatus[] snapshotEditFiles = FSUtils.listStatus(fs, snapshotEdits);
@@ -83,12 +84,10 @@ public class TestCopyRecoveredEditsTask
FileStatus file = snapshotEditFiles[0];
assertEquals("Didn't copy expected file", file1.getName(), file.getPath().getName());
- Mockito.verify(monitor, Mockito.never()).receiveError(Mockito.anyString(),
- Mockito.any(HBaseSnapshotException.class));
- Mockito.verify(monitor, Mockito.never()).snapshotFailure(Mockito.anyString(),
- Mockito.any(SnapshotDescription.class));
- Mockito.verify(monitor, Mockito.never()).snapshotFailure(Mockito.anyString(),
- Mockito.any(SnapshotDescription.class), Mockito.any(Exception.class));
+ Mockito.verify(monitor, Mockito.never()).receive(Mockito.any(ForeignException.class));
+ Mockito.verify(taskSpy, Mockito.never()).snapshotFailure(Mockito.anyString(),
+ Mockito.any(Exception.class));
+
} finally {
// cleanup the working directory
FSUtils.delete(fs, regionDir, true);
@@ -103,7 +102,7 @@ public class TestCopyRecoveredEditsTask
@Test
public void testNoEditsDir() throws Exception {
SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build();
- SnapshotExceptionSnare monitor = Mockito.mock(SnapshotExceptionSnare.class);
+ ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
FileSystem fs = UTIL.getTestFileSystem();
Path root = UTIL.getDataTestDir();
String regionName = "regionA";
@@ -118,7 +117,7 @@ public class TestCopyRecoveredEditsTask
CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir,
snapshotRegionDir);
- task.run();
+ task.call();
} finally {
// cleanup the working directory
FSUtils.delete(fs, regionDir, true);
Modified: hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestReferenceRegionHFilesTask.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestReferenceRegionHFilesTask.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestReferenceRegionHFilesTask.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestReferenceRegionHFilesTask.java Wed Feb 13 18:36:03 2013
@@ -29,8 +29,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.server.snapshot.error.SnapshotExceptionSnare;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -63,16 +63,15 @@ public class TestReferenceRegionHFilesTa
SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("name")
.setTable("table").build();
- SnapshotExceptionSnare monitor = Mockito.mock(SnapshotExceptionSnare.class);
+ ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
ReferenceRegionHFilesTask task = new ReferenceRegionHFilesTask(snapshot, monitor, regionDir,
fs, snapshotRegionDir);
- task.run();
+ ReferenceRegionHFilesTask taskSpy = Mockito.spy(task);
+ task.call();
// make sure we never get an error
- Mockito.verify(monitor, Mockito.never()).snapshotFailure(Mockito.anyString(),
- Mockito.eq(snapshot));
- Mockito.verify(monitor, Mockito.never()).snapshotFailure(Mockito.anyString(),
- Mockito.eq(snapshot), Mockito.any(Exception.class));
+ Mockito.verify(taskSpy, Mockito.never()).snapshotFailure(Mockito.anyString(),
+ Mockito.any(Exception.class));
// verify that all the hfiles get referenced
List<String> hfiles = new ArrayList<String>(2);
Modified: hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestSnapshotTask.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestSnapshotTask.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestSnapshotTask.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestSnapshotTask.java Wed Feb 13 18:36:03 2013
@@ -17,9 +17,15 @@
*/
package org.apache.hadoop.hbase.server.snapshot.task;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.server.snapshot.error.SnapshotExceptionSnare;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.mockito.Mockito;
@@ -31,22 +37,21 @@ public class TestSnapshotTask {
* Check that errors from running the task get propagated back to the error listener.
*/
@Test
- public void testErrorPropagationg() {
- SnapshotExceptionSnare error = Mockito.mock(SnapshotExceptionSnare.class);
+ public void testErrorPropagation() throws Exception {
+ ForeignExceptionDispatcher error = mock(ForeignExceptionDispatcher.class);
SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot")
.setTable("table").build();
final Exception thrown = new Exception("Failed!");
- SnapshotTask fail = new SnapshotTask(snapshot, error, "always fails") {
-
+ SnapshotTask fail = new SnapshotTask(snapshot, error) {
@Override
- protected void process() throws Exception {
- throw thrown;
+ public Void call() {
+ snapshotFailure("Injected failure", thrown);
+ return null;
}
};
- fail.run();
+ fail.call();
- Mockito.verify(error, Mockito.times(1)).snapshotFailure(Mockito.anyString(),
- Mockito.eq(snapshot), Mockito.eq(thrown));
+ verify(error, Mockito.times(1)).receive(any(ForeignException.class));
}
}
Modified: hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestWALReferenceTask.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestWALReferenceTask.java?rev=1445813&r1=1445812&r2=1445813&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestWALReferenceTask.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/server/snapshot/task/TestWALReferenceTask.java Wed Feb 13 18:36:03 2013
@@ -29,9 +29,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.server.snapshot.TakeSnapshotUtils;
-import org.apache.hadoop.hbase.server.snapshot.error.SnapshotExceptionSnare;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.Test;
@@ -76,16 +76,16 @@ public class TestWALReferenceTask {
FSUtils.setRootDir(conf, testDir);
SnapshotDescription snapshot = SnapshotDescription.newBuilder()
.setName("testWALReferenceSnapshot").build();
- SnapshotExceptionSnare listener = Mockito.mock(SnapshotExceptionSnare.class);
+ ForeignExceptionDispatcher listener = Mockito.mock(ForeignExceptionDispatcher.class);
// reference all the files in the first server directory
ReferenceServerWALsTask task = new ReferenceServerWALsTask(snapshot, listener, server1Dir,
conf, fs);
- task.run();
+ task.call();
// reference all the files in the first server directory
task = new ReferenceServerWALsTask(snapshot, listener, server2Dir, conf, fs);
- task.run();
+ task.call();
// verify that we got everything
FSUtils.logFileSystemState(fs, testDir, LOG);
@@ -96,7 +96,7 @@ public class TestWALReferenceTask {
TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logDir, servers, snapshot, snapshotLogDir);
// make sure we never got an error
- Mockito.verify(listener, Mockito.atLeastOnce()).failOnError();
+ Mockito.verify(listener, Mockito.atLeastOnce()).rethrowException();
Mockito.verifyNoMoreInteractions(listener);
}
}
\ No newline at end of file