You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/02/14 03:18:28 UTC
[3/5] git commit: ACCUMULO-1764 Iterate over each volume and compute
an overall summary of usage.
ACCUMULO-1764 Iterate over each volume and compute an overall summary of usage.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/75aac15d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/75aac15d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/75aac15d
Branch: refs/heads/master
Commit: 75aac15d568d5a2d0a50b9bbe813551615d14d1f
Parents: ec4d4d8
Author: Josh Elser <el...@apache.org>
Authored: Thu Feb 13 19:25:11 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Feb 13 19:26:56 2014 -0500
----------------------------------------------------------------------
.../monitor/servlets/DefaultServlet.java | 70 ++++++++++++++------
1 file changed, 51 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/75aac15d/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index 4a7838a..75aa400 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@ -33,10 +33,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.file.FileUtil;
import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Duration;
import org.apache.accumulo.core.util.NumUtil;
import org.apache.accumulo.core.util.Pair;
@@ -44,9 +41,10 @@ import org.apache.accumulo.monitor.Monitor;
import org.apache.accumulo.monitor.ZooKeeperStatus;
import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState;
import org.apache.accumulo.monitor.util.celltypes.NumberType;
+import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.trace.TraceFileSystem;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -259,31 +257,65 @@ public class DefaultServlet extends BasicServlet {
private void doAccumuloTable(StringBuilder sb) throws IOException {
// Accumulo
- Configuration conf = CachedConfiguration.getInstance();
- FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration()));
+ VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
MasterMonitorInfo info = Monitor.getMmi();
sb.append("<table>\n");
sb.append("<tr><th colspan='2'><a href='/master'>Accumulo Master</a></th></tr>\n");
if (info == null) {
sb.append("<tr><td colspan='2'><span class='error'>Master is Down</span></td></tr>\n");
} else {
- String consumed = "Unknown";
- String diskUsed = "Unknown";
+ long totalAcuBytesUsed = 0l;
+ long totalHdfsBytesUsed = 0l;
+
try {
- Path path = new Path(Monitor.getSystemConfiguration().get(Property.INSTANCE_DFS_DIR));
- log.debug("Reading the content summary for " + path);
- try {
- ContentSummary acu = fs.getContentSummary(path);
- diskUsed = bytes(acu.getSpaceConsumed());
- ContentSummary rootSummary = fs.getContentSummary(new Path("/"));
- consumed = String.format("%.2f%%", acu.getSpaceConsumed() * 100. / rootSummary.getSpaceConsumed());
- } catch (Exception ex) {
- log.trace("Unable to get disk usage information from hdfs", ex);
+ for (String baseDir : ServerConstants.getConfiguredBaseDirs()) {
+ final Path basePath = new Path(baseDir);
+ final FileSystem fs = vm.getFileSystemByPath(basePath);
+
+ try {
+ // Calculate the amount of space used by Accumulo on the FileSystem
+ ContentSummary accumuloSummary = fs.getContentSummary(basePath);
+ long bytesUsedByAcuOnFs = accumuloSummary.getSpaceConsumed();
+ totalAcuBytesUsed += bytesUsedByAcuOnFs;
+
+ // Catch the overflow -- this is big data
+ if (totalAcuBytesUsed < bytesUsedByAcuOnFs) {
+ log.debug("Overflowed long in bytes used by Accumulo for " + baseDir);
+ totalAcuBytesUsed = 0l;
+ break;
+ }
+
+ // Calculate the total amount of space used on the FileSystem
+ ContentSummary volumeSummary = fs.getContentSummary(new Path("/"));
+ long bytesUsedOnVolume = volumeSummary.getSpaceConsumed();
+ totalHdfsBytesUsed += bytesUsedOnVolume;
+
+ // Catch the overflow -- this is big data
+ if (totalHdfsBytesUsed < bytesUsedOnVolume) {
+ log.debug("Overflowed long in bytes used in HDFS for " + baseDir);
+ totalHdfsBytesUsed = 0;
+ break;
+ }
+ } catch (Exception ex) {
+ log.trace("Unable to get disk usage information for " + baseDir, ex);
+ }
+ }
+
+ String diskUsed = "Unknown";
+ String consumed = null;
+ if (totalAcuBytesUsed > 0) {
+ // Convert Accumulo usage to a readable String
+ diskUsed = bytes(totalAcuBytesUsed);
+
+ if (totalHdfsBytesUsed > 0) {
+ // Compute amount of space used by Accumulo as a percentage of total space usage.
+ consumed = String.format("%.2f%%", totalAcuBytesUsed * 100. / totalHdfsBytesUsed);
+ }
}
boolean highlight = false;
tableRow(sb, (highlight = !highlight), "Disk Used", diskUsed);
- if (fs.getUsed() != 0)
+ if (null != consumed)
tableRow(sb, (highlight = !highlight), "% of Used DFS", consumed);
tableRow(sb, (highlight = !highlight), "<a href='/tables'>Tables</a>", NumberType.commas(Monitor.getTotalTables()));
tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Tablet Servers</a>", NumberType.commas(info.tServerInfo.size(), 1, Long.MAX_VALUE));