You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2008/03/06 22:53:39 UTC
svn commit: r634432 - in /hadoop/core/branches/branch-0.16: ./ conf/
src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/
src/java/org/apache/hadoop/io/ src/java/org/apache/hadoop/mapred/
src/test/org/apache/hadoop/io/
Author: cdouglas
Date: Thu Mar 6 13:53:37 2008
New Revision: 634432
URL: http://svn.apache.org/viewvc?rev=634432&view=rev
Log:
HADOOP-2869. Deprecate SequenceFile.setCompressionType in favor of
SequenceFile.createWriter, SequenceFileOutputFormat.setCompressionType,
and JobConf.setMapOutputCompressionType. Contributed by Arun C Murthy
Configuration changes to hadoop-default.xml:
deprecated io.seqfile.compression.type
Modified:
hadoop/core/branches/branch-0.16/CHANGES.txt
hadoop/core/branches/branch-0.16/conf/hadoop-default.xml
hadoop/core/branches/branch-0.16/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/io/SequenceFile.java
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/JobConf.java
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/io/FileBench.java
Modified: hadoop/core/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/CHANGES.txt?rev=634432&r1=634431&r2=634432&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.16/CHANGES.txt Thu Mar 6 13:53:37 2008
@@ -7,6 +7,12 @@
HADOOP-2861. Improve the user interface for the HOD commands.
Command line structure has changed. (Hemanth Yamijala via nigel)
+ HADOOP-2869. Deprecate SequenceFile.setCompressionType in favor of
+ SequenceFile.createWriter, SequenceFileOutputFormat.setCompressionType,
+ and JobConf.setMapOutputCompressionType. (Arun C Murthy via cdouglas)
+ Configuration changes to hadoop-default.xml:
+ deprecated io.seqfile.compression.type
+
IMPROVEMENTS
HADOOP-2371. User guide for file permissions in HDFS.
Modified: hadoop/core/branches/branch-0.16/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/conf/hadoop-default.xml?rev=634432&r1=634431&r2=634432&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/conf/hadoop-default.xml (original)
+++ hadoop/core/branches/branch-0.16/conf/hadoop-default.xml Thu Mar 6 13:53:37 2008
@@ -858,13 +858,6 @@
</property>
<property>
- <name>io.seqfile.compression.type</name>
- <value>RECORD</value>
- <description>The default compression type for SequenceFile.Writer.
- </description>
-</property>
-
-<property>
<name>map.sort.class</name>
<value>org.apache.hadoop.mapred.MergeSorter</value>
<description>The default sort class for sorting keys.
Modified: hadoop/core/branches/branch-0.16/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java?rev=634432&r1=634431&r2=634432&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java Thu Mar 6 13:53:37 2008
@@ -102,8 +102,8 @@
job.setMapperClass(mapper);
job.setOutputPath(new Path(outputDir));
job.setOutputFormat(outputFormat);
- SequenceFile.setCompressionType(job,
- SequenceFile.CompressionType.BLOCK);
+ SequenceFileOutputFormat.setOutputCompressionType(job,
+ SequenceFile.CompressionType.BLOCK);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(mapoutputValueClass);
job.setOutputKeyClass(Text.class);
Modified: hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/io/SequenceFile.java?rev=634432&r1=634431&r2=634432&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/io/SequenceFile.java (original)
+++ hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/io/SequenceFile.java Thu Mar 6 13:53:37 2008
@@ -217,7 +217,12 @@
* Get the compression type for the reduce outputs
* @param job the job config to look in
* @return the kind of compression to use
+ * @deprecated Use {@link org.apache.hadoop.mapred.JobConf#getMapOutputCompressionType()}
+ * to get {@link CompressionType} for intermediate map-outputs or
+ * {@link org.apache.hadoop.mapred.SequenceFileOutputFormat#getOutputCompressionType(org.apache.hadoop.mapred.JobConf)}
+ * to get {@link CompressionType} for job-outputs.
*/
+ @Deprecated
static public CompressionType getCompressionType(Configuration job) {
String name = job.get("io.seqfile.compression.type");
return name == null ? CompressionType.RECORD :
@@ -228,7 +233,15 @@
* Set the compression type for sequence files.
* @param job the configuration to modify
* @param val the new compression type (none, block, record)
+ * @deprecated Use the one of the many SequenceFile.createWriter methods to specify
+ * the {@link CompressionType} while creating the {@link SequenceFile} or
+ * {@link org.apache.hadoop.mapred.JobConf#setMapOutputCompressionType(org.apache.hadoop.io.SequenceFile.CompressionType)}
+ * to specify the {@link CompressionType} for intermediate map-outputs or
+ * {@link org.apache.hadoop.mapred.SequenceFileOutputFormat#setOutputCompressionType(org.apache.hadoop.mapred.JobConf, org.apache.hadoop.io.SequenceFile.CompressionType)}
+ * to specify the {@link CompressionType} for job-outputs.
+ * or
*/
+ @Deprecated
static public void setCompressionType(Configuration job,
CompressionType val) {
job.set("io.seqfile.compression.type", val.toString());
Modified: hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/JobConf.java?rev=634432&r1=634431&r2=634432&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/JobConf.java Thu Mar 6 13:53:37 2008
@@ -505,6 +505,7 @@
* are compressed.
*/
public void setMapOutputCompressionType(CompressionType style) {
+ setCompressMapOutput(true);
set("mapred.map.output.compression.type", style.toString());
}
@@ -528,6 +529,7 @@
*/
public void
setMapOutputCompressorClass(Class<? extends CompressionCodec> codecClass) {
+ setCompressMapOutput(true);
setClass("mapred.map.output.compression.codec", codecClass,
CompressionCodec.class);
}
Modified: hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java?rev=634432&r1=634431&r2=634432&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java (original)
+++ hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java Thu Mar 6 13:53:37 2008
@@ -113,6 +113,7 @@
*/
public static void setOutputCompressionType(JobConf conf,
CompressionType style) {
+ setCompressOutput(conf, true);
conf.set("mapred.output.compression.type", style.toString());
}
Modified: hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/io/FileBench.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/io/FileBench.java?rev=634432&r1=634431&r2=634432&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/io/FileBench.java (original)
+++ hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/io/FileBench.java Thu Mar 6 13:53:37 2008
@@ -268,7 +268,6 @@
public void configure(JobConf job) {
job.set("mapred.map.output.compression.type", typ);
job.set("mapred.output.compression.type", typ);
- job.set("io.seqfile.compression.type", typ);
}
}
enum Format {