You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2013/08/15 22:35:28 UTC
svn commit: r1514474 - in
/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/qjournal/server/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/qjour...
Author: jing9
Date: Thu Aug 15 20:35:28 2013
New Revision: 1514474
URL: http://svn.apache.org/r1514474
Log:
HDFS-5076. Merge change r1514429 from branch-2.
Added:
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeMXBean.java
- copied unchanged from r1514429, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeMXBean.java
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java
- copied unchanged from r1514429, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1514474&r1=1514473&r2=1514474&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Aug 15 20:35:28 2013
@@ -12,6 +12,9 @@ Release 2.1.1-beta - UNRELEASED
HDFS-4763 Add script changes/utility for starting NFS gateway (brandonli)
+ HDFS-5076 Add MXBean methods to query NN's transaction information and
+ JournalNode's journal status. (jing9)
+
IMPROVEMENTS
HDFS-4513. Clarify in the WebHDFS REST API that all JSON respsonses may
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java?rev=1514474&r1=1514473&r2=1514474&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java Thu Aug 15 20:35:28 2013
@@ -18,8 +18,10 @@
package org.apache.hadoop.hdfs.qjournal.server;
import java.io.File;
+import java.io.FileFilter;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.HashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -34,11 +36,13 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.mortbay.util.ajax.JSON;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
@@ -51,7 +55,7 @@ import com.google.common.collect.Maps;
* in the quorum protocol.
*/
@InterfaceAudience.Private
-public class JournalNode implements Tool, Configurable {
+public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
public static final Log LOG = LogFactory.getLog(JournalNode.class);
private Configuration conf;
private JournalNodeRpcServer rpcServer;
@@ -128,6 +132,8 @@ public class JournalNode implements Tool
SecurityUtil.login(conf, DFSConfigKeys.DFS_JOURNALNODE_KEYTAB_FILE_KEY,
DFSConfigKeys.DFS_JOURNALNODE_USER_NAME_KEY, socAddr.getHostName());
+ registerJNMXBean();
+
httpServer = new JournalNodeHttpServer(conf, this);
httpServer.start();
@@ -208,6 +214,50 @@ public class JournalNode implements Tool
return new File(new File(dir), jid);
}
+ @Override // JournalNodeMXBean
+ public String getJournalsStatus() {
+ // jid:{Formatted:True/False}
+ Map<String, Map<String, String>> status =
+ new HashMap<String, Map<String, String>>();
+ synchronized (this) {
+ for (Map.Entry<String, Journal> entry : journalsById.entrySet()) {
+ Map<String, String> jMap = new HashMap<String, String>();
+ jMap.put("Formatted", Boolean.toString(entry.getValue().isFormatted()));
+ status.put(entry.getKey(), jMap);
+ }
+ }
+
+ // It is possible that some journals have been formatted before, while the
+ // corresponding journals are not in journalsById yet (because of restarting
+ // JN, e.g.). For simplicity, let's just assume a journal is formatted if
+ // there is a directory for it. We can also call analyzeStorage method for
+ // these directories if necessary.
+ // Also note that we do not need to check localDir here since
+ // validateAndCreateJournalDir has been called before we register the
+ // MXBean.
+ File[] journalDirs = localDir.listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File file) {
+ return file.isDirectory();
+ }
+ });
+ for (File journalDir : journalDirs) {
+ String jid = journalDir.getName();
+ if (!status.containsKey(jid)) {
+ Map<String, String> jMap = new HashMap<String, String>();
+ jMap.put("Formatted", "true");
+ status.put(jid, jMap);
+ }
+ }
+ return JSON.toString(status);
+ }
+
+ /**
+ * Register JournalNodeMXBean
+ */
+ private void registerJNMXBean() {
+ MBeans.register("JournalNode", "JournalNodeInfo", this);
+ }
private class ErrorReporter implements StorageErrorReporter {
@Override
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1514474&r1=1514473&r2=1514474&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu Aug 15 20:35:28 2013
@@ -6240,6 +6240,16 @@ public class FSNamesystem implements Nam
return JSON.toString(statusMap);
}
+ @Override // NameNodeMxBean
+ public String getJournalTransactionInfo() {
+ Map<String, String> txnIdMap = new HashMap<String, String>();
+ txnIdMap.put("LastAppliedOrWrittenTxId",
+ Long.toString(this.getFSImage().getLastAppliedOrWrittenTxId()));
+ txnIdMap.put("MostRecentCheckpointTxId",
+ Long.toString(this.getFSImage().getMostRecentCheckpointTxId()));
+ return JSON.toString(txnIdMap);
+ }
+
/** @return the block manager. */
public BlockManager getBlockManager() {
return blockManager;
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java?rev=1514474&r1=1514473&r2=1514474&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java Thu Aug 15 20:35:28 2013
@@ -174,4 +174,10 @@ public interface NameNodeMXBean {
* @return the name dir status information, as a JSON string.
*/
public String getNameDirStatuses();
+
+ /**
+ * Get information about the transaction ID, including the last applied
+ * transaction ID and the most recent checkpoint's transaction ID
+ */
+ public String getJournalTransactionInfo();
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java?rev=1514474&r1=1514473&r2=1514474&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java Thu Aug 15 20:35:28 2013
@@ -112,6 +112,11 @@ public class TestNameNodeMXBean {
String deadnodeinfo = (String) (mbs.getAttribute(mxbeanName,
"DeadNodes"));
assertEquals(fsn.getDeadNodes(), deadnodeinfo);
+ // get attribute JournalTransactionInfo
+ String journalTxnInfo = (String) mbs.getAttribute(mxbeanName,
+ "JournalTransactionInfo");
+ assertEquals("Bad value for NameTxnIds", fsn.getJournalTransactionInfo(),
+ journalTxnInfo);
// get attribute NameDirStatuses
String nameDirStatuses = (String) (mbs.getAttribute(mxbeanName,
"NameDirStatuses"));