You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/03/28 20:14:03 UTC
svn commit: r389566 - in /lucene/hadoop/trunk/src:
java/org/apache/hadoop/dfs/FSDirectory.java
java/org/apache/hadoop/dfs/NameNode.java java/overview.html
test/org/apache/hadoop/dfs/MiniDFSCluster.java
Author: cutting
Date: Tue Mar 28 10:14:02 2006
New Revision: 389566
URL: http://svn.apache.org/viewcvs?rev=389566&view=rev
Log:
Fix for HADOOP-19. A namenode must now be formatted before it may be used. Attempts to start a namenode in an unformatted directory will fail, rather than automatically creating a new, empty filesystem, causing existing datanodes to delete all blocks. Thus a mis-configured dfs.data.dir should no longer cause data loss.
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java
lucene/hadoop/trunk/src/java/overview.html
lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java?rev=389566&r1=389565&r2=389566&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java Tue Mar 28 10:14:02 2006
@@ -245,14 +245,11 @@
DataOutputStream editlog = null;
boolean ready = false;
- /**
- * Create a FileSystem directory, and load its info
- * from the indicated place.
- */
+ /** Access an existing dfs name directory. */
public FSDirectory(File dir) throws IOException {
File fullimage = new File(dir, "image");
if (! fullimage.exists()) {
- fullimage.mkdirs();
+ throw new IOException("NameNode not formatted: " + dir);
}
File edits = new File(dir, "edits");
if (loadFSImage(fullimage, edits)) {
@@ -263,6 +260,19 @@
this.ready = true;
this.notifyAll();
this.editlog = new DataOutputStream(new FileOutputStream(edits));
+ }
+ }
+
+ /** Create a new dfs name directory. Caution: this destroys all files
+ * in this filesystem. */
+ public static void format(File dir) throws IOException {
+ File image = new File(dir, "image");
+ File edits = new File(dir, "edits");
+
+ if (!((!image.exists() || image.delete()) &&
+ (!edits.exists() || edits.delete()) &&
+ image.mkdirs())) {
+ throw new IOException("Unable to format: "+dir);
}
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java?rev=389566&r1=389565&r2=389566&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java Tue Mar 28 10:14:02 2006
@@ -65,12 +65,17 @@
/** only used for testing purposes */
private boolean stopRequested = false;
+ /** Format a new filesystem. Destroys any filesystem that may already
+ * exist at this location. **/
+ public static void format(Configuration conf) throws IOException {
+ FSDirectory.format(getDir(conf));
+ }
+
/**
* Create a NameNode at the default location
*/
public NameNode(Configuration conf) throws IOException {
- this(new File(conf.get("dfs.name.dir",
- "/tmp/hadoop/dfs/name")),
+ this(getDir(conf),
DataNode.createSocketAddr
(conf.get("fs.default.name", "local")).getPort(), conf);
}
@@ -85,6 +90,11 @@
this.server.start();
}
+ /** Return the configured directory where name data is stored. */
+ private static File getDir(Configuration conf) {
+ return new File(conf.get("dfs.name.dir", "/tmp/hadoop/dfs/name"));
+ }
+
/**
* Wait for service to finish.
* (Normally, it runs forever.)
@@ -364,8 +374,24 @@
/**
*/
- public static void main(String argv[]) throws IOException, InterruptedException {
- NameNode namenode = new NameNode(new Configuration());
+ public static void main(String argv[]) throws Exception {
+ Configuration conf = new Configuration();
+
+ if (argv.length == 1 && argv[0].equals("-format")) {
+ File dir = getDir(conf);
+ if (dir.exists()) {
+ System.err.print("Re-format filesystem in " + dir +" ? (Y or N) ");
+ if (!(System.in.read() == 'Y')) {
+ System.err.println("Format aborted.");
+ System.exit(1);
+ }
+ }
+ format(conf);
+ System.err.println("Formatted "+dir);
+ System.exit(0);
+ }
+
+ NameNode namenode = new NameNode(conf);
namenode.join();
}
}
Modified: lucene/hadoop/trunk/src/java/overview.html
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/overview.html?rev=389566&r1=389565&r2=389566&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/overview.html (original)
+++ lucene/hadoop/trunk/src/java/overview.html Tue Mar 28 10:14:02 2006
@@ -134,6 +134,11 @@
<h3>Bootstrapping</h3>
+<p>A new distributed filesystem must formatted with the following
+command, run on the master node:</p>
+
+<p><tt>bin/hadoop namenode -format</tt></p>
+
<p>The Hadoop daemons are started with the following command:</p>
<p><tt>bin/start-all.sh</tt></p>
Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java?rev=389566&r1=389565&r2=389566&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java Tue Mar 28 10:14:02 2006
@@ -83,7 +83,7 @@
/**
* Create the config and start up the servers.
*/
- public MiniDFSCluster(int namenodePort, Configuration conf) {
+ public MiniDFSCluster(int namenodePort, Configuration conf) throws IOException {
this.conf = conf;
conf.set("fs.default.name",
"localhost:"+ Integer.toString(namenodePort));
@@ -95,6 +95,7 @@
// this timeout seems to control the minimum time for the test, so
// set it down at 5 seconds.
conf.setInt("ipc.client.timeout", 5000);
+ NameNode.format(conf);
nameNode = new NameNodeRunner();
nameNodeThread = new Thread(nameNode);
nameNodeThread.start();