You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/08/15 02:52:24 UTC
svn commit: r1373178 - in
/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/qjournal/server/
src/test/java/org/apache/hadoop/hdfs/qjournal/
src/test/java/org/apache/hadoop/hdfs/qjournal/server/
Author: todd
Date: Wed Aug 15 00:52:24 2012
New Revision: 1373178
URL: http://svn.apache.org/viewvc?rev=1373178&view=rev
Log:
HDFS-3795. QJM: validate journal dir at startup. Contributed by Todd Lipcon.
Modified:
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt?rev=1373178&r1=1373177&r2=1373178&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt Wed Aug 15 00:52:24 2012
@@ -16,3 +16,5 @@ HDFS-3741. Exhaustive failure injection
HDFS-3773. TestNNWithQJM fails after HDFS-3741. (atm)
HDFS-3793. Implement genericized format() in QJM (todd)
+
+HDFS-3795. QJM: validate journal dir at startup (todd)
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java?rev=1373178&r1=1373177&r2=1373178&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java Wed Aug 15 00:52:24 2012
@@ -55,6 +55,8 @@ public class JournalNode implements Tool
private JournalNodeHttpServer httpServer;
private Map<String, Journal> journalsById = Maps.newHashMap();
+ private File localDir;
+
static {
HdfsConfiguration.init();
}
@@ -82,8 +84,32 @@ public class JournalNode implements Tool
@Override
public void setConf(Configuration conf) {
this.conf = conf;
+ this.localDir = new File(
+ conf.get(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY,
+ DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_DEFAULT).trim());
+ }
+
+ private static void validateAndCreateJournalDir(File dir) throws IOException {
+ if (!dir.isAbsolute()) {
+ throw new IllegalArgumentException(
+ "Journal dir '" + dir + "' should be an absolute path");
+ }
+
+ if (!dir.exists() && !dir.mkdirs()) {
+ throw new IOException("Could not create journal dir '" +
+ dir + "'");
+ } else if (!dir.isDirectory()) {
+ throw new IOException("Journal directory '" + dir + "' is not " +
+ "a directory");
+ }
+
+ if (!dir.canWrite()) {
+ throw new IOException("Unable to write to journal dir '" +
+ dir + "'");
+ }
}
+
@Override
public Configuration getConf() {
return conf;
@@ -101,6 +127,8 @@ public class JournalNode implements Tool
public void start() throws IOException {
Preconditions.checkState(!isStarted(), "JN already running");
+ validateAndCreateJournalDir(localDir);
+
DefaultMetricsSystem.initialize("JournalNode");
JvmMetrics.create("JournalNode",
conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY),
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java?rev=1373178&r1=1373177&r2=1373178&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java Wed Aug 15 00:52:24 2012
@@ -156,7 +156,7 @@ public class MiniJournalCluster {
}
public File getStorageDir(int idx) {
- return new File(baseDir, "journalnode-" + idx);
+ return new File(baseDir, "journalnode-" + idx).getAbsoluteFile();
}
public File getCurrentDir(int idx, String jid) {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java?rev=1373178&r1=1373177&r2=1373178&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java Wed Aug 15 00:52:24 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.qjournal;
import static org.junit.Assert.*;
+import java.io.File;
import java.io.IOException;
import java.net.URI;
@@ -42,7 +43,9 @@ public class TestMiniJournalCluster {
JournalNode node = c.getJournalNode(0);
String dir = node.getConf().get(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY);
- assertEquals(MiniDFSCluster.getBaseDirectory() + "journalnode-0",
+ assertEquals(
+ new File(MiniDFSCluster.getBaseDirectory() + "journalnode-0")
+ .getAbsolutePath(),
dir);
} finally {
c.shutdown();
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java?rev=1373178&r1=1373177&r2=1373178&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java Wed Aug 15 00:52:24 2012
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.URL;
@@ -234,6 +235,33 @@ public class TestJournalNode {
}
}
+ @Test
+ public void testFailToStartWithBadConfig() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY, "non-absolute-path");
+ assertJNFailsToStart(conf, "should be an absolute path");
+
+ // Existing file which is not a directory
+ conf.set(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY, "/dev/null");
+ assertJNFailsToStart(conf, "is not a directory");
+
+ // Directory which cannot be created
+ conf.set(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY, "/proc/does-not-exist");
+ assertJNFailsToStart(conf, "Could not create");
+
+ }
+
+ private static void assertJNFailsToStart(Configuration conf,
+ String errString) {
+ try {
+ JournalNode jn = new JournalNode();
+ jn.setConf(conf);
+ jn.start();
+ } catch (Exception e) {
+ GenericTestUtils.assertExceptionContains(errString, e);
+ }
+ }
+
// TODO:
// - add test that checks formatting behavior
// - add test that checks rejects newEpoch if nsinfo doesn't match