You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2008/09/30 00:36:08 UTC
svn commit: r700291 - in /hadoop/core/trunk: ./
src/core/org/apache/hadoop/conf/ src/mapred/org/apache/hadoop/mapred/
src/mapred/org/apache/hadoop/mapred/lib/ src/test/org/apache/hadoop/mapred/
src/test/org/apache/hadoop/mapred/pipes/
Author: omalley
Date: Mon Sep 29 15:36:07 2008
New Revision: 700291
URL: http://svn.apache.org/viewvc?rev=700291&view=rev
Log:
HADOOP-4293. Make Configuration Writable and remove unreleased
WritableJobConf. Configuration.write is renamed to writeXml. (omalley)
Removed:
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/WritableJobConf.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/core/org/apache/hadoop/conf/Configuration.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/Chain.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestWritableJobConf.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Sep 29 15:36:07 2008
@@ -108,6 +108,9 @@
HADOOP-3938. Disk space quotas for HDFS. This is similar to namespace
quotas in 0.18. (rangadi)
+ HADOOP-4293. Make Configuration Writable and remove unreleased
+ WritableJobConf. Configuration.write is renamed to writeXml. (omalley)
+
NEW FEATURES
HADOOP-3341. Allow streaming jobs to specify the field separator for map
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/conf/Configuration.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/conf/Configuration.java Mon Sep 29 15:36:07 2008
@@ -19,6 +19,8 @@
package org.apache.hadoop.conf;
import java.io.BufferedInputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -54,6 +56,8 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.util.StringUtils;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
@@ -128,7 +132,8 @@
* <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
* of the System property with that name.
*/
-public class Configuration implements Iterable<Map.Entry<String,String>> {
+public class Configuration implements Iterable<Map.Entry<String,String>>,
+ Writable {
private static final Log LOG =
LogFactory.getLog(Configuration.class);
@@ -1063,7 +1068,7 @@
*
* @param out the output stream to write to.
*/
- public void write(OutputStream out) throws IOException {
+ public void writeXml(OutputStream out) throws IOException {
Properties properties = getProps();
try {
Document doc =
@@ -1154,7 +1159,27 @@
/** For debugging. List non-default properties to the terminal and exit. */
public static void main(String[] args) throws Exception {
- new Configuration().write(System.out);
+ new Configuration().writeXml(System.out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ clear();
+ int size = WritableUtils.readVInt(in);
+ for(int i=0; i < size; ++i) {
+ set(org.apache.hadoop.io.Text.readString(in),
+ org.apache.hadoop.io.Text.readString(in));
+ }
+ }
+
+ //@Override
+ public void write(DataOutput out) throws IOException {
+ Properties props = getProps();
+ WritableUtils.writeVInt(out, props.size());
+ for(Map.Entry<Object, Object> item: props.entrySet()) {
+ org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
+ org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
+ }
}
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java Mon Sep 29 15:36:07 2008
@@ -808,7 +808,7 @@
new FsPermission(JOB_FILE_PERMISSION));
try {
- job.write(out);
+ job.writeXml(out);
} finally {
out.close();
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java Mon Sep 29 15:36:07 2008
@@ -860,7 +860,7 @@
FileOutputStream jobOut = null;
try {
jobOut = new FileOutputStream(localJobFile);
- jobConf.write(jobOut);
+ jobConf.writeXml(jobOut);
if (LOG.isDebugEnabled()) {
LOG.debug("Job conf for " + jobId + " stored at "
+ localJobFile.getAbsolutePath());
@@ -895,14 +895,14 @@
fs = new Path(LOG_DIR).getFileSystem(jobConf);
if (!fs.exists(jobFilePath)) {
jobFileOut = fs.create(jobFilePath);
- jobConf.write(jobFileOut);
+ jobConf.writeXml(jobFileOut);
jobFileOut.close();
}
}
if (userLogDir != null) {
fs = new Path(userLogDir).getFileSystem(jobConf);
jobFileOut = fs.create(userJobFilePath);
- jobConf.write(jobFileOut);
+ jobConf.writeXml(jobFileOut);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Job conf for " + jobId + " stored at "
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Mon Sep 29 15:36:07 2008
@@ -187,7 +187,7 @@
localFs.delete(localTaskFile, true);
OutputStream out = localFs.create(localTaskFile);
try {
- conf.write(out);
+ conf.writeXml(out);
} finally {
out.close();
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Mon Sep 29 15:36:07 2008
@@ -765,7 +765,7 @@
localJobConf.setJar(localJarFile.toString());
OutputStream out = localFs.create(localJobFile);
try {
- localJobConf.write(out);
+ localJobConf.writeXml(out);
} finally {
out.close();
}
@@ -1842,7 +1842,7 @@
}
OutputStream out = localFs.create(localTaskFile);
try {
- localJobConf.write(out);
+ localJobConf.writeXml(out);
} finally {
out.close();
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/Chain.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/Chain.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/Chain.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/Chain.java Mon Sep 29 15:36:07 2008
@@ -122,8 +122,8 @@
private static JobConf getChainElementConf(JobConf jobConf, String confKey) {
JobConf conf;
try {
- Stringifier<WritableJobConf> stringifier =
- new DefaultStringifier<WritableJobConf>(jobConf, WritableJobConf.class);
+ Stringifier<JobConf> stringifier =
+ new DefaultStringifier<JobConf>(jobConf, JobConf.class);
conf = stringifier.fromString(jobConf.get(confKey, null));
} catch (IOException ioex) {
throw new RuntimeException(ioex);
@@ -233,11 +233,11 @@
Object.class);
// serialize the private mapper jobconf in the chain jobconf.
- Stringifier<WritableJobConf> stringifier =
- new DefaultStringifier<WritableJobConf>(jobConf, WritableJobConf.class);
+ Stringifier<JobConf> stringifier =
+ new DefaultStringifier<JobConf>(jobConf, JobConf.class);
try {
jobConf.set(prefix + CHAIN_MAPPER_CONFIG + index,
- stringifier.toString(new WritableJobConf(mapperConf)));
+ stringifier.toString(new JobConf(mapperConf)));
}
catch (IOException ioEx) {
throw new RuntimeException(ioEx);
@@ -300,11 +300,11 @@
Object.class);
// serialize the private mapper jobconf in the chain jobconf.
- Stringifier<WritableJobConf> stringifier =
- new DefaultStringifier<WritableJobConf>(jobConf, WritableJobConf.class);
+ Stringifier<JobConf> stringifier =
+ new DefaultStringifier<JobConf>(jobConf, JobConf.class);
try {
jobConf.set(prefix + CHAIN_REDUCER_CONFIG,
- stringifier.toString(new WritableJobConf(reducerConf)));
+ stringifier.toString(new JobConf(reducerConf)));
}
catch (IOException ioEx) {
throw new RuntimeException(ioEx);
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestWritableJobConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestWritableJobConf.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestWritableJobConf.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestWritableJobConf.java Mon Sep 29 15:36:07 2008
@@ -77,13 +77,13 @@
}
public void testEmptyConfiguration() throws Exception {
- WritableJobConf conf = new WritableJobConf();
+ JobConf conf = new JobConf();
Configuration deser = serDeser(conf);
assertEquals(conf, deser);
}
public void testNonEmptyConfiguration() throws Exception {
- WritableJobConf conf = new WritableJobConf();
+ JobConf conf = new JobConf();
conf.set("a", "A");
conf.set("b", "B");
Configuration deser = serDeser(conf);
@@ -91,7 +91,7 @@
}
public void testConfigurationWithDefaults() throws Exception {
- WritableJobConf conf = new WritableJobConf(false);
+ JobConf conf = new JobConf(false);
conf.set("a", "A");
conf.set("b", "B");
Configuration deser = serDeser(conf);
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java Mon Sep 29 15:36:07 2008
@@ -231,7 +231,7 @@
local.delete(outDir, true);
local.mkdirs(outDir);
out = local.create(jobXml);
- job.write(out);
+ job.writeXml(out);
out.close();
System.err.println("About to run: Submitter -conf " + jobXml +
" -input " + inDir + " -output " + outDir +