You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2013/04/05 13:42:31 UTC
svn commit: r1464931 - in /hbase/branches/0.94/src:
main/jamon/org/apache/hadoop/hbase/tmpl/master/
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/master/metrics/
main/java/org/apache/hadoop/hbase/master/snapshot/ main/java...
Author: mbertozzi
Date: Fri Apr 5 11:42:31 2013
New Revision: 1464931
URL: http://svn.apache.org/r1464931
Log:
HBASE-7615 Add metrics for snapshots
Added:
hbase/branches/0.94/src/main/resources/hbase-webapps/master/snapshot.jsp
Modified:
hbase/branches/0.94/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
Modified: hbase/branches/0.94/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon (original)
+++ hbase/branches/0.94/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon Fri Apr 5 11:42:31 2013
@@ -212,7 +212,7 @@ org.apache.hadoop.hbase.protobuf.generat
</tr>
<%for SnapshotDescription snapshotDesc : snapshots%>
<tr>
- <td><% snapshotDesc.getName() %></td>
+ <td><a href="snapshot.jsp?name=<% snapshotDesc.getName() %>"><% snapshotDesc.getName() %></a></td>
<td><a href="table.jsp?name=<% snapshotDesc.getTable() %>"><% snapshotDesc.getTable() %></a></td>
<td><% new Date(snapshotDesc.getCreationTime()) %></td>
<td><% snapshotDesc.getType() %></td>
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Fri Apr 5 11:42:31 2013
@@ -492,7 +492,7 @@ Server {
", cluster-up flag was=" + wasUp);
// create the snapshot manager
- this.snapshotManager = new SnapshotManager(this);
+ this.snapshotManager = new SnapshotManager(this, this.metrics);
}
// Check if we should stop every second.
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java Fri Apr 5 11:42:31 2013
@@ -50,7 +50,8 @@ public class MasterMetrics implements Up
private long lastUpdate = System.currentTimeMillis();
private long lastExtUpdate = System.currentTimeMillis();
private long extendedPeriod = 0;
-/*
+
+ /*
* Count of requests to the cluster since last call to metrics update
*/
private final MetricsRate cluster_requests =
@@ -64,6 +65,18 @@ public class MasterMetrics implements Up
final PersistentMetricsTimeVaryingRate splitSize =
new PersistentMetricsTimeVaryingRate("splitSize", registry);
+ /** Time it takes to finish snapshot() */
+ final PersistentMetricsTimeVaryingRate snapshotTime =
+ new PersistentMetricsTimeVaryingRate("snapshotTime", registry);
+
+ /** Time it takes to finish restoreSnapshot() */
+ final PersistentMetricsTimeVaryingRate snapshotRestoreTime =
+ new PersistentMetricsTimeVaryingRate("snapshotRestoreTime", registry);
+
+ /** Time it takes to finish cloneSnapshotTime() */
+ final PersistentMetricsTimeVaryingRate snapshotCloneTime =
+ new PersistentMetricsTimeVaryingRate("snapshotCloneTime", registry);
+
public MasterMetrics(final String name) {
MetricsContext context = MetricsUtil.getContext("hbase");
metricsRecord = MetricsUtil.createRecord(context, "master");
@@ -146,4 +159,28 @@ public class MasterMetrics implements Up
public void incrementRequests(final int inc) {
this.cluster_requests.inc(inc);
}
+
+ /**
+ * Record a single instance of a snapshot
+ * @param time time that the snapshot took
+ */
+ public void addSnapshot(long time) {
+ snapshotTime.inc(time);
+ }
+
+ /**
+ * Record a single instance of a snapshot
+ * @param time time that the snapshot restore took
+ */
+ public void addSnapshotRestore(long time) {
+ snapshotRestoreTime.inc(time);
+ }
+
+ /**
+ * Record a single instance of a snapshot cloned table
+ * @param time time that the snapshot clone took
+ */
+ public void addSnapshotClone(long time) {
+ snapshotCloneTime.inc(time);
+ }
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java Fri Apr 5 11:42:31 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.errorhand
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.SnapshotSentinel;
import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -63,17 +64,20 @@ public class CloneSnapshotHandler extend
private final SnapshotDescription snapshot;
private final ForeignExceptionDispatcher monitor;
+ private final MasterMetrics metricsMaster;
private final MonitoredTask status;
private volatile boolean stopped = false;
public CloneSnapshotHandler(final MasterServices masterServices,
- final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+ final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor,
+ final MasterMetrics metricsMaster)
throws NotAllMetaRegionsOnlineException, TableExistsException, IOException {
- super(masterServices, masterServices.getMasterFileSystem(),
+ super(masterServices, masterServices.getMasterFileSystem(),
masterServices.getServerManager(), hTableDescriptor,
masterServices.getConfiguration(), null, masterServices.getCatalogTracker(),
masterServices.getAssignmentManager());
+ this.metricsMaster = metricsMaster;
// Snapshot information
this.snapshot = snapshot;
@@ -137,6 +141,7 @@ public class CloneSnapshotHandler extend
} else {
status.markComplete("Snapshot '"+ snapshot.getName() +"' clone completed and table enabled!");
}
+ metricsMaster.addSnapshotClone(status.getCompletionTimestamp() - status.getStartTime());
super.completed(exception);
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java Fri Apr 5 11:42:31 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.ServerNam
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -62,8 +63,8 @@ public class DisabledTableSnapshotHandle
* @throws IOException on unexpected error
*/
public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
- final MasterServices masterServices) throws IOException {
- super(snapshot, masterServices);
+ final MasterServices masterServices, final MasterMetrics metricsMaster) throws IOException {
+ super(snapshot, masterServices, metricsMaster);
// setup the timer
timeoutInjector = TakeSnapshotUtils.getMasterTimerAndBindToMonitor(snapshot, conf, monitor);
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java Fri Apr 5 11:42:31 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HRegionIn
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
import org.apache.hadoop.hbase.procedure.Procedure;
import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -49,8 +50,8 @@ public class EnabledTableSnapshotHandler
private final ProcedureCoordinator coordinator;
public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master,
- SnapshotManager manager) throws IOException {
- super(snapshot, master);
+ final SnapshotManager manager, final MasterMetrics metricsMaster) throws IOException {
+ super(snapshot, master, metricsMaster);
this.coordinator = manager.getCoordinator();
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java Fri Apr 5 11:42:31 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.Ma
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.SnapshotSentinel;
import org.apache.hadoop.hbase.master.handler.TableEventHandler;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -60,14 +61,16 @@ public class RestoreSnapshotHandler exte
private final SnapshotDescription snapshot;
private final ForeignExceptionDispatcher monitor;
+ private final MasterMetrics metricsMaster;
private final MonitoredTask status;
private volatile boolean stopped = false;
public RestoreSnapshotHandler(final MasterServices masterServices,
- final SnapshotDescription snapshot, final HTableDescriptor htd)
- throws IOException {
+ final SnapshotDescription snapshot, final HTableDescriptor htd,
+ final MasterMetrics metricsMaster) throws IOException {
super(EventType.C_M_RESTORE_SNAPSHOT, htd.getName(), masterServices, masterServices);
+ this.metricsMaster = metricsMaster;
// Snapshot information
this.snapshot = snapshot;
@@ -146,6 +149,8 @@ public class RestoreSnapshotHandler exte
} else {
status.markComplete("Restore snapshot '"+ snapshot.getName() +"' completed!");
}
+ metricsMaster.addSnapshotRestore(status.getCompletionTimestamp() - status.getStartTime());
+ super.completed(exception);
}
@Override
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java Fri Apr 5 11:42:31 2013
@@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.master.Ma
import org.apache.hadoop.hbase.master.SnapshotSentinel;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
import org.apache.hadoop.hbase.procedure.Procedure;
import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
@@ -117,6 +118,7 @@ public class SnapshotManager implements
private boolean stopped;
private final long wakeFrequency;
private final MasterServices master; // Needed by TableEventHandlers
+ private final MasterMetrics metricsMaster;
private final ProcedureCoordinator coordinator;
// Is snapshot feature enabled?
@@ -138,9 +140,11 @@ public class SnapshotManager implements
* Construct a snapshot manager.
* @param master
*/
- public SnapshotManager(final MasterServices master) throws KeeperException, IOException,
- UnsupportedOperationException {
+ public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster)
+ throws KeeperException, IOException, UnsupportedOperationException {
this.master = master;
+ this.metricsMaster = metricsMaster;
+
checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
// get the configuration for the coordinator
@@ -165,9 +169,12 @@ public class SnapshotManager implements
* @param coordinator procedure coordinator instance. exposed for testing.
* @param pool HBase ExecutorServcie instance, exposed for testing.
*/
- public SnapshotManager(final MasterServices master, ProcedureCoordinator coordinator, ExecutorService pool)
+ public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster,
+ ProcedureCoordinator coordinator, ExecutorService pool)
throws IOException, UnsupportedOperationException {
this.master = master;
+ this.metricsMaster = metricsMaster;
+
checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
this.wakeFrequency = master.getConfiguration().getInt(SNAPSHOT_WAKE_MILLIS_KEY,
@@ -427,7 +434,7 @@ public class SnapshotManager implements
throws HBaseSnapshotException {
TakeSnapshotHandler handler;
try {
- handler = new EnabledTableSnapshotHandler(snapshot, master, this);
+ handler = new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
this.executorService.submit(handler);
this.handler = handler;
} catch (IOException e) {
@@ -536,7 +543,7 @@ public class SnapshotManager implements
DisabledTableSnapshotHandler handler;
try {
- handler = new DisabledTableSnapshotHandler(snapshot, this.master);
+ handler = new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
this.executorService.submit(handler);
this.handler = handler;
} catch (IOException e) {
@@ -619,7 +626,7 @@ public class SnapshotManager implements
try {
CloneSnapshotHandler handler =
- new CloneSnapshotHandler(master, snapshot, hTableDescriptor);
+ new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
this.executorService.submit(handler);
restoreHandlers.put(tableName, handler);
} catch (Exception e) {
@@ -710,7 +717,7 @@ public class SnapshotManager implements
try {
RestoreSnapshotHandler handler =
- new RestoreSnapshotHandler(master, snapshot, hTableDescriptor);
+ new RestoreSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
this.executorService.submit(handler);
restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
} catch (Exception e) {
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java Fri Apr 5 11:42:31 2013
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.errorhand
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -66,6 +67,7 @@ public abstract class TakeSnapshotHandle
// none of these should ever be null
protected final MasterServices master;
+ protected final MasterMetrics metricsMaster;
protected final SnapshotDescription snapshot;
protected final Configuration conf;
protected final FileSystem fs;
@@ -81,13 +83,14 @@ public abstract class TakeSnapshotHandle
* @param masterServices master services provider
* @throws IOException on unexpected error
*/
- public TakeSnapshotHandler(SnapshotDescription snapshot,
- final MasterServices masterServices) throws IOException {
+ public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices,
+ final MasterMetrics metricsMaster) throws IOException {
super(masterServices, EventType.C_M_SNAPSHOT_TABLE);
assert snapshot != null : "SnapshotDescription must not be nul1";
assert masterServices != null : "MasterServices must not be nul1";
this.master = masterServices;
+ this.metricsMaster = metricsMaster;
this.snapshot = snapshot;
this.conf = this.master.getConfiguration();
this.fs = this.master.getMasterFileSystem().getFileSystem();
@@ -156,6 +159,7 @@ public abstract class TakeSnapshotHandle
completeSnapshot(this.snapshotDir, this.workingDir, this.fs);
status.markComplete("Snapshot " + snapshot.getName() + " of table " + snapshot.getTable()
+ " completed");
+ metricsMaster.addSnapshot(status.getCompletionTimestamp() - status.getStartTime());
} catch (Exception e) {
status.abort("Failed to complete snapshot " + snapshot.getName() + " on table " +
snapshot.getTable() + " because " + e.getMessage());
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java Fri Apr 5 11:42:31 2013
@@ -63,6 +63,187 @@ import org.apache.hadoop.hbase.util.FSTa
public final class SnapshotInfo extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
+ /**
+ * Statistics about the snapshot
+ * <ol>
+ * <li> How many store files and logs are in the archive
+ * <li> How many store files and logs are shared with the table
+ * <li> Total store files and logs size and shared amount
+ * </ol>
+ */
+ public static class SnapshotStats {
+ /** Information about the file referenced by the snapshot */
+ static class FileInfo {
+ private final boolean inArchive;
+ private final long size;
+
+ FileInfo(final boolean inArchive, final long size) {
+ this.inArchive = inArchive;
+ this.size = size;
+ }
+
+ /** @return true if the file is in the archive */
+ public boolean inArchive() {
+ return this.inArchive;
+ }
+
+ /** @return true if the file is missing */
+ public boolean isMissing() {
+ return this.size < 0;
+ }
+
+ /** @return the file size */
+ public long getSize() {
+ return this.size;
+ }
+ }
+
+ private int hfileArchiveCount = 0;
+ private int hfilesMissing = 0;
+ private int hfilesCount = 0;
+ private int logsMissing = 0;
+ private int logsCount = 0;
+ private long hfileArchiveSize = 0;
+ private long hfileSize = 0;
+ private long logSize = 0;
+
+ private final SnapshotDescription snapshot;
+ private final Configuration conf;
+ private final FileSystem fs;
+
+ SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
+ {
+ this.snapshot = snapshot;
+ this.conf = conf;
+ this.fs = fs;
+ }
+
+ /** @return the snapshot descriptor */
+ public SnapshotDescription getSnapshotDescription() {
+ return this.snapshot;
+ }
+
+ /** @return true if the snapshot is corrupted */
+ public boolean isSnapshotCorrupted() {
+ return hfilesMissing > 0 || logsMissing > 0;
+ }
+
+ /** @return the number of available store files */
+ public int getStoreFilesCount() {
+ return hfilesCount + hfileArchiveCount;
+ }
+
+ /** @return the number of available store files in the archive */
+ public int getArchivedStoreFilesCount() {
+ return hfileArchiveCount;
+ }
+
+ /** @return the number of available log files */
+ public int getLogsCount() {
+ return logsCount;
+ }
+
+ /** @return the number of missing store files */
+ public int getMissingStoreFilesCount() {
+ return hfilesMissing;
+ }
+
+ /** @return the number of missing log files */
+ public int getMissingLogsCount() {
+ return logsMissing;
+ }
+
+ /** @return the total size of the store files referenced by the snapshot */
+ public long getStoreFilesSize() {
+ return hfileSize + hfileArchiveSize;
+ }
+
+ /** @return the total size of the store files shared */
+ public long getSharedStoreFilesSize() {
+ return hfileSize;
+ }
+
+ /** @return the total size of the store files in the archive */
+ public long getArchivedStoreFileSize() {
+ return hfileArchiveSize;
+ }
+
+ /** @return the percentage of the shared store files */
+ public float getSharedStoreFilePercentage() {
+ return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
+ }
+
+ /** @return the total log size */
+ public long getLogsSize() {
+ return logSize;
+ }
+
+ /**
+ * Add the specified store file to the stats
+ * @param region region encoded Name
+ * @param family family name
+ * @param hfile store file name
+ * @return the store file information
+ */
+ FileInfo addStoreFile(final String region, final String family, final String hfile)
+ throws IOException {
+ String table = this.snapshot.getTable();
+ Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
+ HFileLink link = new HFileLink(conf, path);
+ boolean inArchive = false;
+ long size = -1;
+ try {
+ if ((inArchive = fs.exists(link.getArchivePath()))) {
+ size = fs.getFileStatus(link.getArchivePath()).getLen();
+ hfileArchiveSize += size;
+ hfileArchiveCount++;
+ } else {
+ size = link.getFileStatus(fs).getLen();
+ hfileSize += size;
+ hfilesCount++;
+ }
+ } catch (FileNotFoundException e) {
+ hfilesMissing++;
+ }
+ return new FileInfo(inArchive, size);
+ }
+
+ /**
+ * Add the specified recovered.edits file to the stats
+ * @param region region encoded name
+ * @param logfile log file name
+ * @return the recovered.edits information
+ */
+ FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
+ Path rootDir = FSUtils.getRootDir(conf);
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+ Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
+ long size = fs.getFileStatus(path).getLen();
+ logSize += size;
+ logsCount++;
+ return new FileInfo(true, size);
+ }
+
+ /**
+ * Add the specified log file to the stats
+ * @param server server name
+ * @param logfile log file name
+ * @return the log information
+ */
+ FileInfo addLogFile(final String server, final String logfile) throws IOException {
+ HLogLink logLink = new HLogLink(conf, server, logfile);
+ long size = -1;
+ try {
+ size = logLink.getFileStatus(fs).getLen();
+ logSize += size;
+ logsCount++;
+ } catch (FileNotFoundException e) {
+ logsMissing++;
+ }
+ return new FileInfo(false, size);
+ }
+ }
+
private FileSystem fs;
private Path rootDir;
@@ -170,104 +351,68 @@ public final class SnapshotInfo extends
* dump the file list if requested and the collected information.
*/
private void printFiles(final boolean showFiles) throws IOException {
- final String table = snapshotDesc.getTable();
- final Configuration conf = getConf();
-
if (showFiles) {
System.out.println("Snapshot Files");
System.out.println("----------------------------------------");
}
// Collect information about hfiles and logs in the snapshot
- final AtomicInteger hfileArchiveCount = new AtomicInteger();
- final AtomicInteger hfilesMissing = new AtomicInteger();
- final AtomicInteger hfilesCount = new AtomicInteger();
- final AtomicInteger logsMissing = new AtomicInteger();
- final AtomicInteger logsCount = new AtomicInteger();
- final AtomicLong hfileArchiveSize = new AtomicLong();
- final AtomicLong hfileSize = new AtomicLong();
- final AtomicLong logSize = new AtomicLong();
+ final String table = this.snapshotDesc.getTable();
+ final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
new SnapshotReferenceUtil.FileVisitor() {
public void storeFile (final String region, final String family, final String hfile)
throws IOException {
- Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
- HFileLink link = new HFileLink(conf, path);
- boolean inArchive = false;
- long size = -1;
- try {
- if ((inArchive = fs.exists(link.getArchivePath()))) {
- size = fs.getFileStatus(link.getArchivePath()).getLen();
- hfileArchiveSize.addAndGet(size);
- hfileArchiveCount.addAndGet(1);
- } else {
- size = link.getFileStatus(fs).getLen();
- hfileSize.addAndGet(size);
- hfilesCount.addAndGet(1);
- }
- } catch (FileNotFoundException e) {
- hfilesMissing.addAndGet(1);
- }
+ SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
if (showFiles) {
System.out.printf("%8s %s/%s/%s/%s %s%n",
- (size < 0 ? "-" : StringUtils.humanReadableInt(size)),
+ (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
table, region, family, hfile,
- (inArchive ? "(archive)" : (size < 0) ? "(NOT FOUND)" : ""));
+ (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
}
}
public void recoveredEdits (final String region, final String logfile)
throws IOException {
- Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
- long size = fs.getFileStatus(path).getLen();
- logSize.addAndGet(size);
- logsCount.addAndGet(1);
+ SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
if (showFiles) {
System.out.printf("%8s recovered.edits %s on region %s%n",
- StringUtils.humanReadableInt(size), logfile, region);
+ StringUtils.humanReadableInt(info.getSize()), logfile, region);
}
}
public void logFile (final String server, final String logfile)
throws IOException {
- HLogLink logLink = new HLogLink(conf, server, logfile);
- long size = -1;
- try {
- size = logLink.getFileStatus(fs).getLen();
- logSize.addAndGet(size);
- logsCount.addAndGet(1);
- } catch (FileNotFoundException e) {
- logsMissing.addAndGet(1);
- }
+ SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
if (showFiles) {
System.out.printf("%8s log %s on server %s %s%n",
- (size < 0 ? "-" : StringUtils.humanReadableInt(size)),
+ (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
logfile, server,
- (size < 0 ? "(NOT FOUND)" : ""));
+ (info.isMissing() ? "(NOT FOUND)" : ""));
}
}
});
// Dump the stats
System.out.println();
- if (hfilesMissing.get() > 0 || logsMissing.get() > 0) {
+ if (stats.isSnapshotCorrupted()) {
System.out.println("**************************************************************");
System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
- hfilesMissing.get(), logsMissing.get());
+ stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
System.out.println("**************************************************************");
}
System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
- hfilesCount.get() + hfileArchiveCount.get(), hfileArchiveCount.get(),
- StringUtils.humanReadableInt(hfileSize.get() + hfileArchiveSize.get()),
- ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100,
- StringUtils.humanReadableInt(hfileSize.get())
+ stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
+ StringUtils.humanReadableInt(stats.getStoreFilesSize()),
+ stats.getSharedStoreFilePercentage(),
+ StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
);
System.out.printf("%d Logs, total size %s%n",
- logsCount.get(), StringUtils.humanReadableInt(logSize.get()));
+ stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
System.out.println();
}
@@ -287,6 +432,36 @@ public final class SnapshotInfo extends
}
/**
+ * Returns the snapshot stats
+ * @param conf the {@link Configuration} to use
+ * @param snapshot {@link SnapshotDescription} to get stats from
+ * @return the snapshot stats
+ */
+ public static SnapshotStats getSnapshotStats(final Configuration conf,
+ final SnapshotDescription snapshot) throws IOException {
+ Path rootDir = FSUtils.getRootDir(conf);
+ FileSystem fs = FileSystem.get(conf);
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+ final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
+ SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
+ new SnapshotReferenceUtil.FileVisitor() {
+ public void storeFile (final String region, final String family, final String hfile)
+ throws IOException {
+ stats.addStoreFile(region, family, hfile);
+ }
+
+ public void recoveredEdits (final String region, final String logfile) throws IOException {
+ stats.addRecoveredEdits(region, logfile);
+ }
+
+ public void logFile (final String server, final String logfile) throws IOException {
+ stats.addLogFile(server, logfile);
+ }
+ });
+ return stats;
+ }
+
+ /**
* The guts of the {@link #main} method.
* Call this method to avoid the {@link #main(String[])} System.exit.
* @param args
Added: hbase/branches/0.94/src/main/resources/hbase-webapps/master/snapshot.jsp
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/resources/hbase-webapps/master/snapshot.jsp?rev=1464931&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/resources/hbase-webapps/master/snapshot.jsp (added)
+++ hbase/branches/0.94/src/main/resources/hbase-webapps/master/snapshot.jsp Fri Apr 5 11:42:31 2013
@@ -0,0 +1,189 @@
+<%--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--%>
+<%@ page contentType="text/html;charset=UTF-8"
+ import="java.util.Date"
+ import="java.util.HashMap"
+ import="org.apache.hadoop.conf.Configuration"
+ import="org.apache.hadoop.hbase.client.HBaseAdmin"
+ import="org.apache.hadoop.hbase.client.HConnectionManager"
+ import="org.apache.hadoop.hbase.HRegionInfo"
+ import="org.apache.hadoop.hbase.master.HMaster"
+ import="org.apache.hadoop.hbase.util.Bytes"
+ import="org.apache.hadoop.hbase.util.FSUtils"
+ import="org.apache.hadoop.hbase.protobuf.ProtobufUtil"
+ import="org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription"
+ import="org.apache.hadoop.hbase.snapshot.SnapshotInfo"
+ import="org.apache.hadoop.util.StringUtils"
+ import="java.util.List"
+ import="java.util.Map"
+ import="org.apache.hadoop.hbase.HConstants"%><%
+ HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
+ Configuration conf = master.getConfiguration();
+ HBaseAdmin hbadmin = new HBaseAdmin(conf);
+ boolean readOnly = conf.getBoolean("hbase.master.ui.readonly", false);
+ String snapshotName = request.getParameter("name");
+ SnapshotDescription snapshot = null;
+ SnapshotInfo.SnapshotStats stats = null;
+ for (SnapshotDescription snapshotDesc: hbadmin.listSnapshots()) {
+ if (snapshotName.equals(snapshotDesc.getName())) {
+ snapshot = snapshotDesc;
+ stats = SnapshotInfo.getSnapshotStats(conf, snapshot);
+ break;
+ }
+ }
+
+ String action = request.getParameter("action");
+ String cloneName = request.getParameter("cloneName");
+ boolean isActionResultPage = (!readOnly && action != null);
+%>
+
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Commenting out DOCTYPE so our blue outline shows on hadoop 0.20.205.0, etc.
+ See tail of HBASE-2110 for explaination.
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+-->
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
+<link rel="stylesheet" type="text/css" href="/static/hbase.css" />
+<% if (isActionResultPage) { %>
+ <title>HBase Master: <%= master.getServerName() %></title>
+ <meta http-equiv="refresh" content="5,javascript:history.back()" />
+<% } else { %>
+ <title>Snapshot: <%= snapshotName %></title>
+<% } %>
+</head>
+<body>
+<a id="logo" href="http://wiki.apache.org/lucene-hadoop/Hbase"><img src="/static/hbase_logo.png" alt="HBase Logo" title="HBase Logo" /></a>
+<% if (isActionResultPage) { %>
+ <h1>Snapshot action request...</h1>
+<%
+ if (action.equals("restore")) {
+ hbadmin.restoreSnapshot(snapshotName);
+ %> Restore Snapshot request accepted. <%
+ } else if (action.equals("clone")) {
+ if (cloneName != null && cloneName.length() > 0) {
+ hbadmin.cloneSnapshot(snapshotName, cloneName);
+ %> Clone from Snapshot request accepted. <%
+ } else {
+ %> Clone from Snapshot request failed, No table name specified. <%
+ }
+ }
+%>
+ <p>Go <a href="javascript:history.back()">Back</a>, or wait for the redirect.
+</div>
+<% } else if (snapshot == null) { %>
+ <h1>Snapshot "<%= snapshotName %>" does not exists</h1>
+ <p id="links_menu"><a href="/master.jsp">Master</a>, <a href="/logs/">Local logs</a>, <a href="/stacks">Thread Dump</a>, <a href="/logLevel">Log Level</a></p>
+<hr id="head_rule" />
+ <p>Go <a href="javascript:history.back()">Back</a>, or wait for the redirect.
+<% } else { %>
+ <h1>Snapshot: <%= snapshotName %></h1>
+ <p id="links_menu"><a href="/master.jsp">Master</a>, <a href="/logs/">Local logs</a>, <a href="/stacks">Thread Dump</a>, <a href="/logLevel">Log Level</a></p>
+ <hr id="head_rule" />
+ <h2>Snapshot Attributes</h2>
+ <table class="table" width="90%" >
+ <tr>
+ <th>Table</th>
+ <th>Creation Time</th>
+ <th>Type</th>
+ <th>Format Version</th>
+ <th>State</th>
+ </tr>
+ <tr>
+ <td><a href="table.jsp?name=<%= snapshot.getTable() %>"><%= snapshot.getTable() %></a></td>
+ <td><%= new Date(snapshot.getCreationTime()) %></td>
+ <td><%= snapshot.getType() %></td>
+ <td><%= snapshot.getVersion() %></td>
+ <% if (stats.isSnapshotCorrupted()) { %>
+ <td style="font-weight: bold; color: #dd0000;">CORRUPTED</td>
+ <% } else { %>
+ <td>ok</td>
+ <% } %>
+ </tr>
+ </table>
+ <p>
+ <%= stats.getStoreFilesCount() %> HFiles (<%= stats.getArchivedStoreFilesCount() %> in archive),
+ total size <%= StringUtils.humanReadableInt(stats.getStoreFilesSize()) %>
+ (<%= stats.getSharedStoreFilePercentage() %>%
+ <%= StringUtils.humanReadableInt(stats.getSharedStoreFilesSize()) %> shared with the source
+ table)
+ </p>
+ <p>
+ <%= stats.getLogsCount() %> Logs, total size
+ <%= StringUtils.humanReadableInt(stats.getLogsSize()) %>
+ </p>
+ <% if (stats.isSnapshotCorrupted()) { %>
+ <h3>CORRUPTED Snapshot</h3>
+ <p>
+ <%= stats.getMissingStoreFilesCount() %> hfile(s) and
+ <%= stats.getMissingLogsCount() %> log(s) missing.
+ <p>
+ <% } %>
+<%
+ } // end else
+
+HConnectionManager.deleteConnection(hbadmin.getConfiguration());
+%>
+
+
+<% if (!readOnly && action == null && snapshot != null) { %>
+<p><hr><p>
+Actions:
+<p>
+<center>
+<table style="border-style: none" width="90%">
+<tr>
+ <form method="get">
+ <input type="hidden" name="action" value="clone">
+ <input type="hidden" name="name" value="<%= snapshotName %>">
+ <td style="border-style: none; text-align: center">
+ <input style="font-size: 12pt; width: 10em" type="submit" value="Clone" class="btn"></td>
+ <td style="border-style: none" width="5%"> </td>
+ <td style="border-style: none">New Table Name (clone):<input type="text" name="cloneName" size="40"></td>
+ <td style="border-style: none">
+ This action will create a new table by cloning the snapshot content.
+ There are no copies of data involved.
+ And writing on the newly created table will not influence the snapshot data.
+ </td>
+ </form>
+</tr>
+<tr><td style="border-style: none" colspan="4"> </td></tr>
+<tr>
+ <form method="get">
+ <input type="hidden" name="action" value="restore">
+ <input type="hidden" name="name" value="<%= snapshotName %>">
+ <td style="border-style: none; text-align: center">
+ <input style="font-size: 12pt; width: 10em" type="submit" value="Restore" class="btn"></td>
+ <td style="border-style: none" width="5%"> </td>
+ <td style="border-style: none"> </td>
+ <td style="border-style: none">Restore a specified snapshot.
+ The restore will replace the content of the original table,
+ bringing back the content to the snapshot state.
+ The table must be disabled.</td>
+ </form>
+</tr>
+</table>
+</center>
+<p>
+</div>
+<% } %>
+</body>
+</html>
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java?rev=1464931&r1=1464930&r2=1464931&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java Fri Apr 5 11:42:31 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.master.Ma
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.zookeeper.KeeperException;
@@ -49,6 +50,7 @@ public class TestSnapshotManager {
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
MasterServices services = Mockito.mock(MasterServices.class);
+ MasterMetrics metrics = Mockito.mock(MasterMetrics.class);
ProcedureCoordinator coordinator = Mockito.mock(ProcedureCoordinator.class);
ExecutorService pool = Mockito.mock(ExecutorService.class);
MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class);
@@ -71,7 +73,7 @@ public class TestSnapshotManager {
Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
Mockito.when(mfs.getFileSystem()).thenReturn(fs);
Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir());
- return new SnapshotManager(services, coordinator, pool);
+ return new SnapshotManager(services, metrics, coordinator, pool);
}
@Test