You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sh...@apache.org on 2010/01/12 01:21:54 UTC
svn commit: r898131 - in /hadoop/hdfs/trunk: ./ src/java/
src/java/org/apache/hadoop/hdfs/server/common/
src/java/org/apache/hadoop/hdfs/server/datanode/
src/test/hdfs/org/apache/hadoop/hdfs/
src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Author: shv
Date: Tue Jan 12 00:21:53 2010
New Revision: 898131
URL: http://svn.apache.org/viewvc?rev=898131&view=rev
Log:
HDFS-873. Configuration specifies data-node storage directories as URIs. Contributed by Konstantin Shvachko.
Added:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java (with props)
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/hdfs-default.xml
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Util.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue Jan 12 00:21:53 2010
@@ -122,6 +122,12 @@
HDFS-512. Block.equals() and compareTo() compare blocks based
only on block Ids, ignoring generation stamps. (shv)
+ HDFS-512. Block.equals() and compareTo() compare blocks based
+ only on block Ids, ignoring generation stamps. (shv)
+
+ HDFS-873. Configuration specifies data-node storage directories as URIs.
+ (shv)
+
NEW FEATURES
HDFS-436. Introduce AspectJ framework for HDFS code and tests.
Modified: hadoop/hdfs/trunk/src/java/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/hdfs-default.xml?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/hdfs-default.xml (original)
+++ hadoop/hdfs/trunk/src/java/hdfs-default.xml Tue Jan 12 00:21:53 2010
@@ -236,7 +236,7 @@
<property>
<name>dfs.datanode.data.dir</name>
- <value>${hadoop.tmp.dir}/dfs/data</value>
+ <value>file://${hadoop.tmp.dir}/dfs/data</value>
<description>Determines where on the local filesystem an DFS data node
should store its blocks. If this is a comma-delimited
list of directories, then data will be stored in all named
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Util.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Util.java?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Util.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Util.java Tue Jan 12 00:21:53 2010
@@ -52,12 +52,14 @@
try {
u = new URI(s);
} catch (URISyntaxException e){
- LOG.warn("Path " + s + " should be specified as a URI " +
- "in configuration files. Please update hdfs configuration.", e);
+ LOG.error("Syntax error in URI " + s
+ + ". Please check hdfs configuration.", e);
}
// if URI is null or scheme is undefined, then assume it's file://
if(u == null || u.getScheme() == null){
+ LOG.warn("Path " + s + " should be specified as a URI "
+ + "in configuration files. Please update hdfs configuration.");
u = fileAsURI(new File(s));
}
return u;
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Tue Jan 12 00:21:53 2010
@@ -26,6 +26,7 @@
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketTimeoutException;
+import java.net.URI;
import java.net.UnknownHostException;
import java.nio.channels.ServerSocketChannel;
import java.nio.channels.SocketChannel;
@@ -63,6 +64,7 @@
import org.apache.hadoop.hdfs.server.common.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
@@ -401,7 +403,8 @@
//init ipc server
InetSocketAddress ipcAddr = NetUtils.createSocketAddr(
conf.get("dfs.datanode.ipc.address"));
- ipcServer = RPC.getServer(this, ipcAddr.getHostName(), ipcAddr.getPort(),
+ ipcServer = RPC.getServer(DataNode.class, this,
+ ipcAddr.getHostName(), ipcAddr.getPort(),
conf.getInt("dfs.datanode.handler.count", 3), false, conf);
ipcServer.start();
dnRegistration.setIpcPort(ipcServer.getListenerAddress().getPort());
@@ -1365,12 +1368,18 @@
" anymore. RackID resolution is handled by the NameNode.");
System.exit(-1);
}
- String[] dataDirs = conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
+ Collection<URI> dataDirs = getStorageDirs(conf);
dnThreadName = "DataNode: [" +
- StringUtils.arrayToString(dataDirs) + "]";
+ StringUtils.uriToString(dataDirs.toArray(new URI[0])) + "]";
return makeInstance(dataDirs, conf);
}
+ static Collection<URI> getStorageDirs(Configuration conf) {
+ Collection<String> dirNames =
+ conf.getStringCollection(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
+ return Util.stringCollectionAsURIs(dirNames);
+ }
+
/** Instantiate & Start a single datanode daemon and wait for it to finish.
* If this thread is specifically interrupted, it will stop waiting.
*/
@@ -1400,21 +1409,27 @@
* no directory from this directory list can be created.
* @throws IOException
*/
- public static DataNode makeInstance(String[] dataDirs, Configuration conf)
+ static DataNode makeInstance(Collection<URI> dataDirs, Configuration conf)
throws IOException {
ArrayList<File> dirs = new ArrayList<File>();
- for (int i = 0; i < dataDirs.length; i++) {
- File data = new File(dataDirs[i]);
+ for(URI dirURI : dataDirs) {
+ if(! "file".equalsIgnoreCase(dirURI.getScheme())) {
+ LOG.warn("Unsupported URI schema in " + dirURI + ". Ignoring ...");
+ continue;
+ }
+ File data = new File(dirURI.getPath());
try {
DiskChecker.checkDir(data);
dirs.add(data);
} catch(DiskErrorException e) {
- LOG.warn("Invalid directory in dfs.data.dir: " + e.getMessage());
+ LOG.warn("Invalid directory in "
+ + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY + ": " + e.getMessage());
}
}
if (dirs.size() > 0)
return new DataNode(conf, dirs);
- LOG.error("All directories in dfs.data.dir are invalid.");
+ LOG.error("All directories in "
+ + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY + " are invalid.");
return null;
}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java Tue Jan 12 00:21:53 2010
@@ -310,6 +310,14 @@
}
/**
+ * Get configuration.
+ * @return Configuration of this MiniDFSCluster
+ */
+ public Configuration getConfiguration() {
+ return conf;
+ }
+
+ /**
* wait for the cluster to get out of
* safemode.
*/
@@ -415,14 +423,16 @@
throw new IOException("Mkdirs failed to create directory for DataNode "
+ i + ": " + dir1 + " or " + dir2);
}
- dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dir1.getPath() + "," + dir2.getPath());
+ dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
+ fileAsURI(dir1) + "," + fileAsURI(dir2));
}
if (simulatedCapacities != null) {
dnConf.setBoolean("dfs.datanode.simulateddatastorage", true);
dnConf.setLong(SimulatedFSDataset.CONFIG_PROPERTY_CAPACITY,
simulatedCapacities[i-curDatanodesNum]);
}
- System.out.println("Starting DataNode " + i + " with dfs.data.dir: "
+ System.out.println("Starting DataNode " + i + " with "
+ + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY + ": "
+ dnConf.get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY));
if (hosts != null) {
dnConf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, hosts[i - curDatanodesNum]);
@@ -441,6 +451,9 @@
NetUtils.addStaticResolution(hosts[i - curDatanodesNum], "localhost");
}
DataNode dn = DataNode.instantiateDataNode(dnArgs, dnConf);
+ if(dn == null)
+ throw new IOException("Cannot start DataNode in "
+ + dnConf.get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY));
//since the HDFS does things based on IP:port, we need to add the mapping
//for IP:port to rackId
String ipAddr = dn.getSelfAddr().getAddress().getHostAddress();
Added: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java?rev=898131&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java (added)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java Tue Jan 12 00:21:53 2010
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests if a data-node can startup depending on configuration parameters.
+ */
+public class TestDatanodeConfig {
+ private static final File BASE_DIR =
+ new File(MiniDFSCluster.getBaseDirectory());
+
+ private static MiniDFSCluster cluster;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ clearBaseDir();
+ Configuration conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster(conf, 0, true, null);
+ cluster.waitActive();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ if(cluster != null)
+ cluster.shutdown();
+ clearBaseDir();
+ }
+
+ private static void clearBaseDir() throws IOException {
+ if(BASE_DIR.exists() && ! FileUtil.fullyDelete(BASE_DIR))
+ throw new IOException("Cannot clear BASE_DIR " + BASE_DIR);
+ }
+
+ /**
+ * Test that a data-node does not start if configuration specifies
+ * incorrect URI scheme in data directory.
+ * Test that a data-node starts if data directory is specified as
+ * URI = "file:///path" or as a non URI path.
+ */
+ @Test
+ public void testDataDirectories() throws IOException {
+ File dataDir = new File(BASE_DIR, "data").getCanonicalFile();
+ Configuration conf = cluster.getConfiguration();
+ // 1. Test unsupported schema. Only "file:" is supported.
+ String dnDir = makeURI("shv", null, fileAsURI(dataDir).getPath());
+ conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dnDir);
+ DataNode dn = DataNode.createDataNode(new String[]{}, conf);
+ assertNull("Data-node startup should have failed.", dn);
+
+ // 2. Test "file:" schema and no schema (path-only). Both should work.
+ String dnDir1 = fileAsURI(dataDir).toString() + "1";
+ String dnDir2 = makeURI("file", "localhost",
+ fileAsURI(dataDir).getPath() + "2");
+ String dnDir3 = dataDir.getAbsolutePath() + "3";
+ conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
+ dnDir1 + "," + dnDir2 + "," + dnDir3);
+ cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
+ assertTrue("Data-node should startup.", cluster.isDataNodeUp());
+ }
+
+ private static String makeURI(String scheme, String host, String path)
+ throws IOException {
+ try {
+ URI uDir = new URI(scheme, host, path, null);
+ return uDir.toString();
+ } catch(URISyntaxException e) {
+ throw new IOException("Bad URI", e);
+ }
+ }
+}
Propchange: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java Tue Jan 12 00:21:53 2010
@@ -82,9 +82,7 @@
protected void setUp() throws Exception {
config = new HdfsConfiguration();
- String baseDir = System.getProperty("test.build.data", "build/test/data");
-
- hdfsDir = new File(baseDir, "dfs");
+ hdfsDir = new File(MiniDFSCluster.getBaseDirectory()).getCanonicalFile();
if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'");
}
@@ -133,8 +131,7 @@
Iterator<StorageDirectory> it = fi.dirIterator();
while(it.hasNext()) {
StorageDirectory sd = it.next();
- if(sd.getRoot().getAbsolutePath().equals(path2.getAbsolutePath()) ||
- sd.getRoot().getAbsolutePath().equals(path3.getAbsolutePath())) {
+ if(sd.getRoot().equals(path2) || sd.getRoot().equals(path3)) {
al.add(sd);
}
}
@@ -260,8 +257,8 @@
// should be different
//assertTrue(fsImg1.length() != fsImg2.length());
//assertTrue(fsImg1.length() != fsImg3.length());
- assertTrue(fsEdits1.length() != fsEdits2.length());
- assertTrue(fsEdits1.length() != fsEdits3.length());
+ assertTrue("edits1 = edits2", fsEdits1.length() != fsEdits2.length());
+ assertTrue("edits1 = edits3", fsEdits1.length() != fsEdits3.length());
assertTrue(!md5_1.equals(md5_2));
assertTrue(!md5_1.equals(md5_3));