You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cn...@apache.org on 2013/07/12 07:10:39 UTC
svn commit: r1502426 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/main/webapps/hdfs/ src/main/webapps/static/
src/test/java/org/apache/hadoop/hdfs/server/namenode/
Author: cnauroth
Date: Fri Jul 12 05:10:38 2013
New Revision: 1502426
URL: http://svn.apache.org/r1502426
Log:
HDFS-4374. Merging change r1502331 from trunk to branch-2.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1502426&r1=1502425&r2=1502426&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Jul 12 05:10:38 2013
@@ -97,6 +97,8 @@ Release 2.1.0-beta - 2013-07-02
HDFS-4372. Track NameNode startup progress. (cnauroth)
HDFS-4373. Add HTTP API for querying NameNode startup progress. (cnauroth)
+
+ HDFS-4374. Display NameNode startup progress in UI. (cnauroth)
IMPROVEMENTS
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java?rev=1502426&r1=1502425&r2=1502426&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java Fri Jul 12 05:10:38 2013
@@ -52,6 +52,12 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgressView;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Status;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.Text;
@@ -76,7 +82,7 @@ class NamenodeJspHelper {
}
static String getSafeModeText(FSNamesystem fsn) {
- if (!fsn.isInSafeMode())
+ if (fsn == null || !fsn.isInSafeMode())
return "";
return "Safe mode is ON. <em>" + fsn.getSafeModeTip() + "</em><br>";
}
@@ -94,6 +100,10 @@ class NamenodeJspHelper {
}
static String getInodeLimitText(FSNamesystem fsn) {
+ if (fsn == null) {
+ return "";
+ }
+
long inodes = fsn.dir.totalInodes();
long blocks = fsn.getBlocksTotal();
long maxobjects = fsn.getMaxObjects();
@@ -133,15 +143,21 @@ class NamenodeJspHelper {
/** Return a table containing version information. */
static String getVersionTable(FSNamesystem fsn) {
- return "<div class='dfstable'><table>"
- + "\n <tr><td class='col1'>Started:</td><td>" + fsn.getStartTime()
- + "</td></tr>\n" + "\n <tr><td class='col1'>Version:</td><td>"
- + VersionInfo.getVersion() + ", " + VersionInfo.getRevision()
- + "</td></tr>\n" + "\n <tr><td class='col1'>Compiled:</td><td>" + VersionInfo.getDate()
- + " by " + VersionInfo.getUser() + " from " + VersionInfo.getBranch()
- + "</td></tr>\n <tr><td class='col1'>Cluster ID:</td><td>" + fsn.getClusterId()
- + "</td></tr>\n <tr><td class='col1'>Block Pool ID:</td><td>" + fsn.getBlockPoolId()
- + "</td></tr>\n</table></div>";
+ StringBuilder sb = new StringBuilder();
+ sb.append("<div class='dfstable'><table>");
+ if (fsn != null) {
+ sb.append("\n <tr><td class='col1'>Started:</td><td>" + fsn.getStartTime());
+ }
+ sb.append("</td></tr>\n" + "\n <tr><td class='col1'>Version:</td><td>");
+ sb.append(VersionInfo.getVersion() + ", " + VersionInfo.getRevision());
+ sb.append("</td></tr>\n" + "\n <tr><td class='col1'>Compiled:</td><td>" + VersionInfo.getDate());
+ sb.append(" by " + VersionInfo.getUser() + " from " + VersionInfo.getBranch());
+ if (fsn != null) {
+ sb.append("</td></tr>\n <tr><td class='col1'>Cluster ID:</td><td>" + fsn.getClusterId());
+ sb.append("</td></tr>\n <tr><td class='col1'>Block Pool ID:</td><td>" + fsn.getBlockPoolId());
+ }
+ sb.append("</td></tr>\n</table></div>");
+ return sb.toString();
}
/**
@@ -149,6 +165,10 @@ class NamenodeJspHelper {
* @return a warning if files are corrupt, otherwise return an empty string.
*/
static String getCorruptFilesWarning(FSNamesystem fsn) {
+ if (fsn == null) {
+ return "";
+ }
+
long missingBlocks = fsn.getMissingBlocksCount();
if (missingBlocks > 0) {
StringBuilder result = new StringBuilder();
@@ -196,6 +216,9 @@ class NamenodeJspHelper {
void generateConfReport(JspWriter out, NameNode nn,
HttpServletRequest request) throws IOException {
FSNamesystem fsn = nn.getNamesystem();
+ if (fsn == null) {
+ return;
+ }
FSImage fsImage = fsn.getFSImage();
List<Storage.StorageDirectory> removedStorageDirs
= fsImage.getStorage().getRemovedStorageDirs();
@@ -233,6 +256,9 @@ class NamenodeJspHelper {
*/
void generateJournalReport(JspWriter out, NameNode nn,
HttpServletRequest request) throws IOException {
+ if (nn.getNamesystem() == null) {
+ return;
+ }
FSEditLog log = nn.getFSImage().getEditLog();
Preconditions.checkArgument(log != null, "no edit log set in %s", nn);
@@ -297,6 +323,9 @@ class NamenodeJspHelper {
void generateHealthReport(JspWriter out, NameNode nn,
HttpServletRequest request) throws IOException {
FSNamesystem fsn = nn.getNamesystem();
+ if (fsn == null) {
+ return;
+ }
final DatanodeManager dm = fsn.getBlockManager().getDatanodeManager();
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
@@ -426,6 +455,142 @@ class NamenodeJspHelper {
out.print("There are no datanodes in the cluster.");
}
}
+
+ /**
+ * Generates the Startup Progress report.
+ *
+ * @param out JspWriter to receive output
+ * @param prog StartupProgress tracking NameNode startup progress
+ * @throws IOException thrown if there is an I/O error
+ */
+ void generateStartupProgress(JspWriter out, StartupProgress prog)
+ throws IOException {
+ StartupProgressView view = prog.createView();
+ FormattedWriter fout = new FormattedWriter(out);
+ fout.println("<div id=\"startupprogress\">");
+ fout.println("<div><span>Elapsed Time:</span> %s</div>",
+ StringUtils.formatTime(view.getElapsedTime()));
+ fout.println("<div><span>Percent Complete:</span> %s</div>",
+ StringUtils.formatPercent(view.getPercentComplete(), 2));
+ fout.println("<table>");
+ fout.println("<tr>");
+ fout.println("<th>Phase</th>");
+ fout.println("<th>Completion</th>");
+ fout.println("<th>Elapsed Time</th>");
+ fout.println("</tr>");
+ for (Phase phase: view.getPhases()) {
+ final String timeClass;
+ Status status = view.getStatus(phase);
+ if (status == Status.PENDING) {
+ timeClass = "later";
+ } else if (status == Status.RUNNING) {
+ timeClass = "current";
+ } else {
+ timeClass = "prior";
+ }
+
+ fout.println("<tr class=\"phase %s\">", timeClass);
+ printPhase(fout, view, phase);
+ fout.println("</tr>");
+
+ for (Step step: view.getSteps(phase)) {
+ fout.println("<tr class=\"step %s\">", timeClass);
+ printStep(fout, view, phase, step);
+ fout.println("</tr>");
+ }
+ }
+ fout.println("</table>");
+ fout.println("</div>");
+ }
+
+ /**
+ * Prints one line of content for a phase in the Startup Progress report.
+ *
+ * @param fout FormattedWriter to receive output
+ * @param view StartupProgressView containing information to print
+ * @param phase Phase to print
+ * @throws IOException thrown if there is an I/O error
+ */
+ private void printPhase(FormattedWriter fout, StartupProgressView view,
+ Phase phase) throws IOException {
+ StringBuilder phaseLine = new StringBuilder();
+ phaseLine.append(phase.getDescription());
+ String file = view.getFile(phase);
+ if (file != null) {
+ phaseLine.append(" ").append(file);
+ }
+ long size = view.getSize(phase);
+ if (size != Long.MIN_VALUE) {
+ phaseLine.append(" (").append(StringUtils.byteDesc(size)).append(")");
+ }
+ fout.println("<td class=\"startupdesc\">%s</td>", phaseLine.toString());
+ fout.println("<td>%s</td>", StringUtils.formatPercent(
+ view.getPercentComplete(phase), 2));
+ fout.println("<td>%s</td>", view.getStatus(phase) == Status.PENDING ? "" :
+ StringUtils.formatTime(view.getElapsedTime(phase)));
+ }
+
+ /**
+ * Prints one line of content for a step in the Startup Progress report.
+ *
+ * @param fout FormattedWriter to receive output
+ * @param view StartupProgressView containing information to print
+ * @param phase Phase to print
+ * @param step Step to print
+ * @throws IOException thrown if there is an I/O error
+ */
+ private void printStep(FormattedWriter fout, StartupProgressView view,
+ Phase phase, Step step) throws IOException {
+ StringBuilder stepLine = new StringBuilder();
+ String file = step.getFile();
+ if (file != null) {
+ stepLine.append(file);
+ }
+ long size = step.getSize();
+ if (size != Long.MIN_VALUE) {
+ stepLine.append(" (").append(StringUtils.byteDesc(size)).append(")");
+ }
+ StepType type = step.getType();
+ if (type != null) {
+ stepLine.append(" ").append(type.getDescription());
+ }
+
+ fout.println("<td class=\"startupdesc\">%s (%d/%d)</td>",
+ stepLine.toString(), view.getCount(phase, step),
+ view.getTotal(phase, step));
+ fout.println("<td>%s</td>", StringUtils.formatPercent(
+ view.getPercentComplete(phase), 2));
+ fout.println("<td>%s</td>", view.getStatus(phase) == Status.PENDING ? "" :
+ StringUtils.formatTime(view.getElapsedTime(phase)));
+ }
+
+ /**
+ * JspWriter wrapper that helps simplify printing formatted lines.
+ */
+ private static class FormattedWriter {
+ private final JspWriter out;
+
+ /**
+ * Creates a new FormattedWriter that delegates to the given JspWriter.
+ *
+ * @param out JspWriter to wrap
+ */
+ FormattedWriter(JspWriter out) {
+ this.out = out;
+ }
+
+ /**
+ * Prints one formatted line, followed by line terminator, using the
+ * English locale.
+ *
+ * @param format String format
+ * @param args Object... any number of arguments to match format
+ * @throws IOException thrown if there is an I/O error
+ */
+ void println(String format, Object... args) throws IOException {
+ out.println(StringUtils.format(format, args));
+ }
+ }
}
static String getDelegationToken(final NamenodeProtocols nn,
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp?rev=1502426&r1=1502425&r2=1502426&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp Fri Jul 12 05:10:38 2013
@@ -34,17 +34,26 @@
boolean isActive = (nnHAState == HAServiceState.ACTIVE);
String namenodeRole = nn.getRole().toString();
String namenodeState = nnHAState.toString();
- String namenodeLabel = nn.getNameNodeAddressHostPortString();
+ String namenodeLabel = nn.getRpcServer() != null ?
+ nn.getNameNodeAddressHostPortString() : null;
%>
<!DOCTYPE html>
<html>
<head>
<link rel="stylesheet" type="text/css" href="/static/hadoop.css">
+<% if (namenodeLabel != null) { %>
<title>Hadoop <%=namenodeRole%> <%=namenodeLabel%></title>
+<% } else { %>
+<title>Hadoop <%=namenodeRole%></title>
+<% } %>
</head>
<body>
+<% if (namenodeLabel != null) { %>
<h1><%=namenodeRole%> '<%=namenodeLabel%>' (<%=namenodeState%>)</h1>
+<% } else { %>
+<h1><%=namenodeRole%> (<%=namenodeState%>)</h1>
+<% } %>
<%= NamenodeJspHelper.getVersionTable(fsn) %>
<br />
<% if (isActive) { %>
@@ -64,6 +73,8 @@
<hr/>
<% healthjsp.generateConfReport(out, nn, request); %>
<hr>
+<h3>Startup Progress</h3>
+<% healthjsp.generateStartupProgress(out, nn.getStartupProgress()); %>
<%
out.println(ServletUtil.htmlFooter());
%>
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css?rev=1502426&r1=1502425&r2=1502426&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css Fri Jul 12 05:10:38 2013
@@ -155,3 +155,36 @@ span.failed {
div.security {
width:100%;
}
+
+#startupprogress table, #startupprogress th, #startupprogress td {
+ border-collapse: collapse;
+ border-left: 1px solid black;
+ border-right: 1px solid black;
+ padding: 5px;
+ text-align: left;
+}
+
+#startupprogress table {
+ border: 1px solid black;
+}
+
+.phase {
+ border-top: 1px solid black;
+ font-weight: bold;
+}
+
+.current {
+ font-style: italic;
+}
+
+.later {
+ color: gray;
+}
+
+.step .startupdesc {
+ text-indent: 20px;
+}
+
+#startupprogress span {
+ font-weight: bold;
+}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java?rev=1502426&r1=1502425&r2=1502426&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java Fri Jul 12 05:10:38 2013
@@ -18,22 +18,28 @@
package org.apache.hadoop.hdfs.server.namenode;
-import static org.mockito.Mockito.mock;
+import static org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase.*;
+import static org.mockito.Mockito.*;
import java.io.IOException;
+import java.util.List;
+import java.util.regex.Pattern;
import javax.servlet.http.HttpServletRequest;
+import javax.servlet.jsp.JspWriter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
public class TestNameNodeJspHelper {
@@ -79,4 +85,42 @@ public class TestNameNodeJspHelper {
Assert.assertTrue("security mode doesn't match. Should be OFF",
securityOnOff.contains("OFF"));
}
+
+ @Test
+ public void testGenerateStartupProgress() throws Exception {
+ cluster.waitClusterUp();
+ NamenodeJspHelper.HealthJsp jsp = new NamenodeJspHelper.HealthJsp();
+ StartupProgress prog = NameNode.getStartupProgress();
+ JspWriter out = mock(JspWriter.class);
+ jsp.generateStartupProgress(out, prog);
+ ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
+ verify(out, atLeastOnce()).println(captor.capture());
+ List<String> contents = captor.getAllValues();
+
+ // Verify 100% overall completion and all phases mentioned in output.
+ Assert.assertTrue(containsMatch(contents, "Elapsed Time\\:"));
+ Assert.assertTrue(containsMatch(contents, "Percent Complete\\:.*?100\\.00%"));
+ Assert.assertTrue(containsMatch(contents, LOADING_FSIMAGE.getDescription()));
+ Assert.assertTrue(containsMatch(contents, LOADING_EDITS.getDescription()));
+ Assert.assertTrue(containsMatch(contents,
+ SAVING_CHECKPOINT.getDescription()));
+ Assert.assertTrue(containsMatch(contents, SAFEMODE.getDescription()));
+ }
+
+ /**
+ * Checks if the list contains any string that partially matches the regex.
+ *
+ * @param list List<String> containing strings to check
+ * @param regex String regex to check
+ * @return boolean true if some string in list partially matches regex
+ */
+ private static boolean containsMatch(List<String> list, String regex) {
+ Pattern pattern = Pattern.compile(regex);
+ for (String str: list) {
+ if (pattern.matcher(str).find()) {
+ return true;
+ }
+ }
+ return false;
+ }
}