You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/02/03 21:53:03 UTC
svn commit: r374740 - in /lucene/hadoop/trunk: ./ conf/
src/java/org/apache/hadoop/conf/ src/java/org/apache/hadoop/dfs/
src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/mapred/
src/test/org/apache/hadoop/dfs/ src/test/org/apache/hadoop/mapred/
Author: cutting
Date: Fri Feb 3 12:52:59 2006
New Revision: 374740
URL: http://svn.apache.org/viewcvs?rev=374740&view=rev
Log:
First version that passes unit tests.
Added:
lucene/hadoop/trunk/conf/configuration.xsl
- copied unchanged from r374710, lucene/nutch/trunk/conf/nutch-conf.xsl
Modified:
lucene/hadoop/trunk/build.xml
lucene/hadoop/trunk/conf/hadoop-default.xml
lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/Block.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFS.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java
Modified: lucene/hadoop/trunk/build.xml
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/build.xml?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/build.xml (original)
+++ lucene/hadoop/trunk/build.xml Fri Feb 3 12:52:59 2006
@@ -147,9 +147,6 @@
<delete dir="${test.build.data}"/>
<mkdir dir="${test.build.data}"/>
- <copy file="${test.src.dir}/hadoop-site.xml"
- todir="${test.build.classes}"/>
-
<junit printsummary="yes" haltonfailure="no" fork="yes" dir="${basedir}"
errorProperty="tests.failed" failureProperty="tests.failed">
<sysproperty key="test.build.data" value="${test.build.data}"/>
@@ -195,7 +192,7 @@
<target name="default-doc">
<style basedir="${conf.dir}" destdir="${docs.dir}"
- includes="hadoop-default.xml" style="conf/hadoop-conf.xsl"/>
+ includes="hadoop-default.xml" style="conf/configuration.xsl"/>
</target>
<!-- ================================================================== -->
Modified: lucene/hadoop/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/conf/hadoop-default.xml?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/conf/hadoop-default.xml (original)
+++ lucene/hadoop/trunk/conf/hadoop-default.xml Fri Feb 3 12:52:59 2006
@@ -1,11 +1,11 @@
<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="nutch-conf.xsl"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Do not modify this file directly. Instead, copy entries that you -->
-<!-- wish to modify from this file into nutch-site.xml and change them -->
-<!-- there. If nutch-site.xml does not already exist, create it. -->
+<!-- wish to modify from this file into hadoop-site.xml and change them -->
+<!-- there. If hadoop-site.xml does not already exist, create it. -->
-<nutch-conf>
+<configuration>
<!-- file properties -->
@@ -97,14 +97,14 @@
<property>
<name>dfs.name.dir</name>
- <value>/tmp/nutch/dfs/name</value>
+ <value>/tmp/hadoop/dfs/name</value>
<description>Determines where on the local filesystem the DFS name node
should store the name table.</description>
</property>
<property>
<name>dfs.data.dir</name>
- <value>/tmp/nutch/dfs/data</value>
+ <value>/tmp/hadoop/dfs/data</value>
<description>Determines where on the local filesystem an DFS data node
should store its blocks. If this is a comma- or space-delimited
list of directories, then data will be stored in all named
@@ -155,7 +155,7 @@
<property>
<name>mapred.local.dir</name>
- <value>/tmp/nutch/mapred/local</value>
+ <value>/tmp/hadoop/mapred/local</value>
<description>The local directory where MapReduce stores intermediate
data files. May be a space- or comma- separated list of
directories on different devices in order to spread disk i/o.
@@ -164,14 +164,14 @@
<property>
<name>mapred.system.dir</name>
- <value>/tmp/nutch/mapred/system</value>
+ <value>/tmp/hadoop/mapred/system</value>
<description>The shared directory where MapReduce stores control files.
</description>
</property>
<property>
<name>mapred.temp.dir</name>
- <value>/tmp/nutch/mapred/temp</value>
+ <value>/tmp/hadoop/mapred/temp</value>
<description>A shared directory for temporary files.
</description>
</property>
@@ -234,4 +234,4 @@
<description>Defines the timeout for IPC calls in milliseconds.</description>
</property>
-</nutch-conf>
+</configuration>
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java Fri Feb 3 12:52:59 2006
@@ -35,10 +35,10 @@
* <p>An ordered list of configuration parameter files with
* default and always-overrides site parameters.
* <p>Default values for all parameters are specified in a file named
- * <tt>nutch-default.xml</tt> located on the classpath. Overrides for these
- * defaults should be in an optional file named <tt>nutch-site.xml</tt>, also
+ * <tt>hadoop-default.xml</tt> located on the classpath. Overrides for these
+ * defaults should be in an optional file named <tt>hadoop-site.xml</tt>, also
* located on the classpath. Typically these files reside in the
- * <tt>conf/</tt> subdirectory at the top-level of a Nutch installation.
+ * <tt>conf/</tt> subdirectory at the top-level of a Hadoop installation.
* <p>The resource files are read upon first access of values (set, get,
* or write) after {@link #addConfResource(String)} or
* {@link #addConfResource(File)}.
@@ -54,8 +54,8 @@
/** A new configuration. */
public Configuration() {
- resourceNames.add("nutch-default.xml");
- resourceNames.add("nutch-site.xml");
+ resourceNames.add("hadoop-default.xml");
+ resourceNames.add("hadoop-site.xml");
}
/** A new configuration with the same settings cloned from another. */
@@ -67,16 +67,16 @@
/** Adds a resource name to the chain of resources read. Such resources are
* located on the CLASSPATH. The first resource is always
- * <tt>nutch-default.xml</tt>, and the last is always
- * <tt>nutch-site.xml</tt>. New resources are inserted between these, so
+ * <tt>hadoop-default.xml</tt>, and the last is always
+ * <tt>hadoop-site.xml</tt>. New resources are inserted between these, so
* they can override defaults, but not site-specifics. */
public synchronized void addConfResource(String name) {
addConfResourceInternal(name);
}
/** Adds a file to the chain of resources read. The first resource is always
- * <tt>nutch-default.xml</tt>, and the last is always
- * <tt>nutch-site.xml</tt>. New resources are inserted between these, so
+ * <tt>hadoop-default.xml</tt>, and the last is always
+ * <tt>hadoop-site.xml</tt>. New resources are inserted between these, so
* they can override defaults, but not site-specifics. */
public synchronized void addConfResource(File file) {
addConfResourceInternal(file);
@@ -344,8 +344,8 @@
}
Element root = doc.getDocumentElement();
- if (!"nutch-conf".equals(root.getTagName()))
- LOG.severe("bad conf file: top-level element not <nutch-conf>");
+ if (!"configuration".equals(root.getTagName()))
+ LOG.severe("bad conf file: top-level element not <configuration>");
NodeList props = root.getChildNodes();
for (int i = 0; i < props.getLength(); i++) {
Node propNode = props.item(i);
@@ -384,7 +384,7 @@
try {
Document doc =
DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
- Element conf = doc.createElement("nutch-conf");
+ Element conf = doc.createElement("configuration");
doc.appendChild(conf);
conf.appendChild(doc.createTextNode("\n"));
for (Enumeration e = properties.keys(); e.hasMoreElements();) {
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/Block.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/Block.java?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/Block.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/Block.java Fri Feb 3 12:52:59 2006
@@ -21,7 +21,7 @@
import java.util.*;
/**************************************************
- * A Block is a Nutch FS primitive, identified by a
+ * A Block is a Hadoop FS primitive, identified by a
* long.
*
* @author Mike Cafarella
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Fri Feb 3 12:52:59 2006
@@ -27,7 +27,7 @@
import java.util.logging.*;
/********************************************************
- * DFSClient can connect to a Nutch Filesystem and perform basic file tasks.
+ * DFSClient can connect to a Hadoop Filesystem and perform basic file tasks.
* Connects to a namenode daemon.
* @author Mike Cafarella, Tessa MacDuff
********************************************************/
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Fri Feb 3 12:52:59 2006
@@ -191,7 +191,7 @@
/////////////////////////////////////////////////////////
//
- // These methods are called by NutchFS clients
+ // These methods are called by HadoopFS clients
//
/////////////////////////////////////////////////////////
/**
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java Fri Feb 3 12:52:59 2006
@@ -51,7 +51,7 @@
*/
public NameNode(Configuration conf) throws IOException {
this(new File(conf.get("dfs.name.dir",
- "/tmp/nutch/dfs/name")),
+ "/tmp/hadoop/dfs/name")),
DataNode.createSocketAddr
(conf.get("fs.default.name", "local")).getPort(), conf);
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Fri Feb 3 12:52:59 2006
@@ -27,15 +27,15 @@
/****************************************************************
* An abstract base class for a fairly simple
* distributed file system.
- * A Nutch installation might consist
+ * A Hadoop installation might consist
* of multiple machines, which should swap files transparently.
- * This interface allows other Nutch systems to find and place
- * files into the distributed Nutch-controlled file world.
+ * This interface allows other Hadoop systems to find and place
+ * files into the distributed Hadoop-controlled file world.
* <p>
- * A local implementation exists for testing and for small Nutch instances.
+ * A local implementation exists for testing and for small Hadoop instances.
* <p>
* The standard job of FileSystem is to take the location-
- * independent NutchFile objects, and resolve them using local
+ * independent HadoopFile objects, and resolve them using local
* knowledge and local instances of ShareGroup.
* <p>
* The local implementation is {@link LocalFileSystem} and distributed
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Fri Feb 3 12:52:59 2006
@@ -90,7 +90,7 @@
public File getSystemDir() {
return new File(get("mapred.system.dir",
- "/tmp/nutch/mapred/system"));
+ "/tmp/hadoop/mapred/system"));
}
public String[] getLocalDirs() throws IOException {
Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFS.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFS.java?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFS.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFS.java Fri Feb 3 12:52:59 2006
@@ -65,7 +65,7 @@
* <p>Bring the namenode down and restart it to verify that datanodes reconnect.
* <p>
* <p>For a another approach to filesystem testing, see the high level
- * (NutchFS level) test {@link org.apache.hadoop.fs.TestFileSystem}.
+ * (HadoopFS level) test {@link org.apache.hadoop.fs.TestFileSystem}.
* @author Paul Baclace
*/
public class TestDFS extends TestCase implements FSConstants {
@@ -199,8 +199,8 @@
// downsize for testing (just to save resources)
conf.setInt("dfs.namenode.handler.count", 3);
if (false) { // use MersenneTwister, if present
- conf.set("nutch.random.class",
- "org.apache.nutch.util.MersenneTwister");
+ conf.set("hadoop.random.class",
+ "org.apache.hadoop.util.MersenneTwister");
}
conf.setLong("dfs.blockreport.intervalMsec", 50*1000L);
conf.setLong("dfs.datanode.startupMsec", 15*1000L);
@@ -425,7 +425,7 @@
/**
* Make a data generator.
* Allows optional use of high quality PRNG by setting property
- * nutch.random.class to the full class path of a subclass of
+ * hadoop.random.class to the full class path of a subclass of
* java.util.Random such as "...util.MersenneTwister".
* The property test.dfs.random.seed can supply a seed for reproducible
* testing (a default is set here if property is not set.)
@@ -436,7 +436,7 @@
if (randomDataGeneratorCtor == null) {
// lazy init
String rndDataGenClassname =
- conf.get("nutch.random.class", "java.util.Random");
+ conf.get("hadoop.random.class", "java.util.Random");
Class clazz = Class.forName(rndDataGenClassname);
randomDataGeneratorCtor = clazz.getConstructor(new Class[]{Long.TYPE});
}
Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java?rev=374740&r1=374739&r2=374740&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java Fri Feb 3 12:52:59 2006
@@ -26,7 +26,7 @@
/**********************************************************
* MapredLoadTest generates a bunch of work that exercises
- * a Nutch Map-Reduce system (and DFS, too). It goes through
+ * a Hadoop Map-Reduce system (and DFS, too). It goes through
* the following steps:
*
* 1) Take inputs 'range' and 'counts'.