You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by cu...@apache.org on 2005/07/19 23:05:03 UTC
svn commit: r219773 - in /lucene/nutch/branches/mapred:
conf/nutch-default.xml src/java/org/apache/nutch/ndfs/DataNode.java
src/java/org/apache/nutch/ndfs/DatanodeProtocol.java
Author: cutting
Date: Tue Jul 19 14:05:00 2005
New Revision: 219773
URL: http://svn.apache.org/viewcvs?rev=219773&view=rev
Log:
Add support for multiple devices to DataNode.
Modified:
lucene/nutch/branches/mapred/conf/nutch-default.xml
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DataNode.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DatanodeProtocol.java
Modified: lucene/nutch/branches/mapred/conf/nutch-default.xml
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/conf/nutch-default.xml?rev=219773&r1=219772&r2=219773&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/conf/nutch-default.xml (original)
+++ lucene/nutch/branches/mapred/conf/nutch-default.xml Tue Jul 19 14:05:00 2005
@@ -358,7 +358,9 @@
<name>ndfs.data.dir</name>
<value>/tmp/nutch/ndfs/data</value>
<description>Determines where on the local filesystem an NDFS data node
- should store its blocks.</description>
+ should store its blocks. If this is a comma- or space-delimited
+ list of directories, then data will be stored in all named
+ directories, typically on different devices.</description>
</property>
<!-- map/reduce properties -->
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DataNode.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DataNode.java?rev=219773&r1=219772&r2=219773&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DataNode.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DataNode.java Tue Jul 19 14:05:00 2005
@@ -35,7 +35,7 @@
*
* @author Mike Cafarella
**********************************************************/
-public class DataNode implements FSConstants {
+public class DataNode implements FSConstants, Runnable {
public static final Logger LOG = LogFormatter.getLogger("org.apache.nutch.ndfs.DataNode");
//
// REMIND - mjc - I might bring "maxgigs" back so user can place
@@ -66,12 +66,12 @@
int xmitsInProgress = 0;
/**
- * Create using configured defaults and dataDir.
+ * Create given a configuration and a dataDir.
*/
- public DataNode(String datadir) throws IOException {
+ public DataNode(NutchConf conf, String datadir) throws IOException {
this(InetAddress.getLocalHost().getHostName(),
new File(datadir),
- createSocketAddr(NutchConf.get().get("fs.default.name", "local")));
+ createSocketAddr(conf.get("fs.default.name", "local")));
}
/**
@@ -548,28 +548,38 @@
}
}
- /**
- */
- public static void main(String argv[]) throws IOException {
- String dataDir = NutchConf.get().get("ndfs.data.dir",
- "/tmp/nutch/data/name");
- if (argv.length > 0){
- dataDir=argv[0];
- }
- LOG.info("Using ["+dataDir+"] directory for data storage.");
-
- DataNode datanode = new DataNode(dataDir);
- while (true) {
- try {
- datanode.offerService();
- } catch (Exception ex) {
- LOG.info("Exception: " + ex.toString());
- LOG.info("Lost connection to namenode [" + datanode.getNamenode() + "]. Retrying...");
- try {
- Thread.sleep(5000);
- } catch (InterruptedException ie) {
- }
- }
+ public void run() {
+ LOG.info("Starting DataNode in: "+data.data);
+ while (true) {
+ try {
+ offerService();
+ } catch (Exception ex) {
+ LOG.info("Exception: " + ex);
+ LOG.info("Lost connection to namenode. Retrying...");
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException ie) {
}
+ }
+ }
+ }
+
+ public static void run(NutchConf conf) throws IOException {
+ String[] dataDirs = conf.getStrings("ndfs.data.dir");
+ for (int i = 0; i < dataDirs.length; i++) {
+ String dataDir = dataDirs[i];
+ File data = new File(dataDir);
+ data.mkdirs();
+ if (!data.isDirectory()) {
+ LOG.warning("Can't start DataNode in non-directory: "+dataDir);
+ continue;
+ }
+ new Thread(new DataNode(conf, dataDir), "DataNode: "+dataDir).start();
}
+ }
+
+ public static void main(String args[]) throws IOException {
+ LogFormatter.setShowThreadIDs(true);
+ run(NutchConf.get());
+ }
}
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DatanodeProtocol.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DatanodeProtocol.java?rev=219773&r1=219772&r2=219773&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DatanodeProtocol.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DatanodeProtocol.java Tue Jul 19 14:05:00 2005
@@ -27,10 +27,10 @@
**********************************************************************/
public interface DatanodeProtocol {
- public void sendHeartbeat(String sender, long capacity, long remaining);
- public void blockReport(String sender, Block blocks[]);
- public void blockReceived(String sender, Block blocks[]);
- public void errorReport(String sender, String msg);
+ public void sendHeartbeat(String sender, long capacity, long remaining) throws IOException;
+ public void blockReport(String sender, Block blocks[]) throws IOException;
+ public void blockReceived(String sender, Block blocks[]) throws IOException;
+ public void errorReport(String sender, String msg) throws IOException;
- public BlockCommand getBlockwork(String sender, int xmitsInProgress);
+ public BlockCommand getBlockwork(String sender, int xmitsInProgress) throws IOException;
}