You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2011/06/30 22:56:37 UTC
svn commit: r1141713 - in /hadoop/common/trunk/hdfs: ./
src/java/org/apache/hadoop/hdfs/server/datanode/
src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/
Author: eli
Date: Thu Jun 30 20:56:37 2011
New Revision: 1141713
URL: http://svn.apache.org/viewvc?rev=1141713&view=rev
Log:
HDFS-2118. Couple dfs data dir improvements. Contributed by Eli Collins
Modified:
hadoop/common/trunk/hdfs/CHANGES.txt
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
Modified: hadoop/common/trunk/hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/CHANGES.txt?rev=1141713&r1=1141712&r2=1141713&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hdfs/CHANGES.txt Thu Jun 30 20:56:37 2011
@@ -552,6 +552,8 @@ Trunk (unreleased changes)
HDFS-2056. Update fetchdt usage. (Tanping Wang via jitendra)
+ HDFS-2118. Couple dfs data dir improvements. (eli)
+
BUG FIXES
HDFS-1955. FSImage.doUpgrade() was made too fault-tolerant by HDFS-1826.
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1141713&r1=1141712&r2=1141713&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Thu Jun 30 20:56:37 2011
@@ -2186,20 +2186,21 @@ public class DataNode extends Configured
continue;
}
// drop any (illegal) authority in the URI for backwards compatibility
- File data = new File(dirURI.getPath());
+ File dir = new File(dirURI.getPath());
try {
- DiskChecker.checkDir(localFS, new Path(data.toURI()), permission);
- dirs.add(data);
- } catch (IOException e) {
- LOG.warn("Invalid directory in: "
- + DFS_DATANODE_DATA_DIR_KEY + ": ", e);
- invalidDirs.append("\"").append(data.getCanonicalPath()).append("\" ");
+ DiskChecker.checkDir(localFS, new Path(dir.toURI()), permission);
+ dirs.add(dir);
+ } catch (IOException ioe) {
+ LOG.warn("Invalid " + DFS_DATANODE_DATA_DIR_KEY + " "
+ + dir + " : ", ioe);
+ invalidDirs.append("\"").append(dir.getCanonicalPath()).append("\" ");
}
}
- if (dirs.size() == 0)
+ if (dirs.size() == 0) {
throw new IOException("All directories in "
+ DFS_DATANODE_DATA_DIR_KEY + " are invalid: "
+ invalidDirs);
+ }
return dirs;
}
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java?rev=1141713&r1=1141712&r2=1141713&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java Thu Jun 30 20:56:37 2011
@@ -125,7 +125,7 @@ class DataXceiver extends Receiver imple
DataNode getDataNode() {return datanode;}
/**
- * Read/write data from/to the DataXceiveServer.
+ * Read/write data from/to the DataXceiverServer.
*/
public void run() {
updateCurrentThreadName("Waiting for operation");
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java?rev=1141713&r1=1141712&r2=1141713&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java Thu Jun 30 20:56:37 2011
@@ -146,10 +146,10 @@ class DataXceiverServer implements Runna
} catch (SocketTimeoutException ignored) {
// wake up to see if should continue to run
} catch (IOException ie) {
- LOG.warn(datanode.getMachineName() + ":DataXceiveServer: ", ie);
+ LOG.warn(datanode.getMachineName() + ":DataXceiverServer: ", ie);
} catch (Throwable te) {
LOG.error(datanode.getMachineName()
- + ":DataXceiveServer: Exiting due to: ", te);
+ + ":DataXceiverServer: Exiting due to: ", te);
datanode.shouldRun = false;
}
}
@@ -157,7 +157,7 @@ class DataXceiverServer implements Runna
ss.close();
} catch (IOException ie) {
LOG.warn(datanode.getMachineName()
- + ":DataXceiveServer: Close exception due to: ", ie);
+ + ":DataXceiverServer: Close exception due to: ", ie);
}
}
@@ -167,7 +167,7 @@ class DataXceiverServer implements Runna
try {
this.ss.close();
} catch (IOException ie) {
- LOG.warn(datanode.getMachineName() + ":DataXceiveServer.kill(): "
+ LOG.warn(datanode.getMachineName() + ":DataXceiverServer.kill(): "
+ StringUtils.stringifyException(ie));
}
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java?rev=1141713&r1=1141712&r2=1141713&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java Thu Jun 30 20:56:37 2011
@@ -1150,7 +1150,7 @@ public class FSDataset implements FSCons
conf.getInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,
DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT);
- String[] dataDirs = conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
+ String[] dataDirs = conf.getTrimmedStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
int volsConfigured = (dataDirs == null) ? 0 : dataDirs.length;
Modified: hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java?rev=1141713&r1=1141712&r2=1141713&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java (original)
+++ hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java Thu Jun 30 20:56:37 2011
@@ -133,17 +133,17 @@ public class TestDataNodeVolumeFailureTo
}
/**
- * Restart the cluster with a new volume tolerated value.
- * @param volTolerated
- * @param manageCluster
+ * Restart the datanodes with a new volume tolerated value.
+ * @param volTolerated number of dfs data dir failures to tolerate
+ * @param manageDfsDirs whether the mini cluster should manage data dirs
* @throws IOException
*/
- private void restartCluster(int volTolerated, boolean manageCluster)
+ private void restartDatanodes(int volTolerated, boolean manageDfsDirs)
throws IOException {
//Make sure no datanode is running
cluster.shutdownDataNodes();
conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, volTolerated);
- cluster.startDataNodes(conf, 1, manageCluster, null, null);
+ cluster.startDataNodes(conf, 1, manageDfsDirs, null, null);
cluster.waitActive();
}
@@ -174,19 +174,14 @@ public class TestDataNodeVolumeFailureTo
/**
* Tests for a given volumes to be tolerated and volumes failed.
- *
- * @param volumesTolerated
- * @param volumesFailed
- * @param expectedBPServiceState
- * @param clusterManaged
- * @throws IOException
- * @throws InterruptedException
*/
private void testVolumeConfig(int volumesTolerated, int volumesFailed,
- boolean expectedBPServiceState, boolean clusterManaged)
+ boolean expectedBPServiceState, boolean manageDfsDirs)
throws IOException, InterruptedException {
assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
final int dnIndex = 0;
+ // Fail the current directory since invalid storage directory perms
+ // get fixed up automatically on datanode startup.
File[] dirs = {
new File(MiniDFSCluster.getStorageDir(dnIndex, 0), "current"),
new File(MiniDFSCluster.getStorageDir(dnIndex, 1), "current") };
@@ -195,11 +190,10 @@ public class TestDataNodeVolumeFailureTo
for (int i = 0; i < volumesFailed; i++) {
prepareDirToFail(dirs[i]);
}
- restartCluster(volumesTolerated, clusterManaged);
+ restartDatanodes(volumesTolerated, manageDfsDirs);
assertEquals(expectedBPServiceState, cluster.getDataNodes().get(0)
.isBPServiceAlive(cluster.getNamesystem().getBlockPoolId()));
} finally {
- // restore its old permission
for (File dir : dirs) {
FileUtil.chmod(dir.toString(), "755");
}
@@ -215,8 +209,7 @@ public class TestDataNodeVolumeFailureTo
private void prepareDirToFail(File dir) throws IOException,
InterruptedException {
dir.mkdirs();
- assertTrue("Couldn't chmod local vol", FileUtil
- .chmod(dir.toString(), "000") == 0);
+ assertEquals("Couldn't chmod local vol", 0,
+ FileUtil.chmod(dir.toString(), "000"));
}
-
}