You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2010/10/06 08:16:31 UTC
svn commit: r1004900 - in /hadoop/common/trunk: ./
src/java/org/apache/hadoop/io/
Author: omalley
Date: Wed Oct 6 06:16:31 2010
New Revision: 1004900
URL: http://svn.apache.org/viewvc?rev=1004900&view=rev
Log:
HADOOP-6984. Combine the compress kind and the codec in the same option
for SequenceFiles. (cdouglas via omalley)
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/io/ArrayFile.java
hadoop/common/trunk/src/java/org/apache/hadoop/io/BloomMapFile.java
hadoop/common/trunk/src/java/org/apache/hadoop/io/MapFile.java
hadoop/common/trunk/src/java/org/apache/hadoop/io/SequenceFile.java
hadoop/common/trunk/src/java/org/apache/hadoop/io/SetFile.java
Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=1004900&r1=1004899&r2=1004900&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Wed Oct 6 06:16:31 2010
@@ -253,6 +253,9 @@ Trunk (unreleased changes)
HADOOP-6989. Correct the parameter for SetFile to set the value type
for SetFile to be NullWritable instead of the key. (cdouglas via omalley)
+ HADOOP-6984. Combine the compress kind and the codec in the same option
+ for SequenceFiles. (cdouglas via omalley)
+
Release 0.21.1 - Unreleased
IMPROVEMENTS
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/io/ArrayFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/io/ArrayFile.java?rev=1004900&r1=1004899&r2=1004900&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/io/ArrayFile.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/io/ArrayFile.java Wed Oct 6 06:16:31 2010
@@ -54,7 +54,7 @@ public class ArrayFile extends MapFile {
super(conf, new Path(file),
keyClass(LongWritable.class),
valueClass(valClass),
- compressionType(compress),
+ compression(compress),
progressable(progress));
}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/io/BloomMapFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/io/BloomMapFile.java?rev=1004900&r1=1004899&r2=1004900&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/io/BloomMapFile.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/io/BloomMapFile.java Wed Oct 6 06:16:31 2010
@@ -89,8 +89,7 @@ public class BloomMapFile {
Class<? extends Writable> valClass, CompressionType compress,
CompressionCodec codec, Progressable progress) throws IOException {
this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass),
- compressionType(compress), compressionCodec(codec),
- progressable(progress));
+ compression(compress, codec), progressable(progress));
}
@Deprecated
@@ -99,7 +98,7 @@ public class BloomMapFile {
Class valClass, CompressionType compress,
Progressable progress) throws IOException {
this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass),
- compressionType(compress), progressable(progress));
+ compression(compress), progressable(progress));
}
@Deprecated
@@ -108,7 +107,7 @@ public class BloomMapFile {
Class valClass, CompressionType compress)
throws IOException {
this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass),
- compressionType(compress));
+ compression(compress));
}
@Deprecated
@@ -117,8 +116,8 @@ public class BloomMapFile {
CompressionType compress, CompressionCodec codec, Progressable progress)
throws IOException {
this(conf, new Path(dirName), comparator(comparator),
- valueClass(valClass), compressionType(compress),
- compressionCodec(codec), progressable(progress));
+ valueClass(valClass), compression(compress, codec),
+ progressable(progress));
}
@Deprecated
@@ -126,7 +125,7 @@ public class BloomMapFile {
WritableComparator comparator, Class valClass,
CompressionType compress, Progressable progress) throws IOException {
this(conf, new Path(dirName), comparator(comparator),
- valueClass(valClass), compressionType(compress),
+ valueClass(valClass), compression(compress),
progressable(progress));
}
@@ -135,7 +134,7 @@ public class BloomMapFile {
WritableComparator comparator, Class valClass, CompressionType compress)
throws IOException {
this(conf, new Path(dirName), comparator(comparator),
- valueClass(valClass), compressionType(compress));
+ valueClass(valClass), compression(compress));
}
@Deprecated
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/io/MapFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/io/MapFile.java?rev=1004900&r1=1004899&r2=1004900&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/io/MapFile.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/io/MapFile.java Wed Oct 6 06:16:31 2010
@@ -113,7 +113,7 @@ public class MapFile {
CompressionType compress,
Progressable progress) throws IOException {
this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass),
- compressionType(compress), progressable(progress));
+ compression(compress), progressable(progress));
}
/** Create the named map for keys of the named class.
@@ -125,8 +125,7 @@ public class MapFile {
CompressionType compress, CompressionCodec codec,
Progressable progress) throws IOException {
this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass),
- compressionType(compress), compressionCodec(codec),
- progressable(progress));
+ compression(compress, codec), progressable(progress));
}
/** Create the named map for keys of the named class.
@@ -137,7 +136,7 @@ public class MapFile {
Class<? extends WritableComparable> keyClass, Class valClass,
CompressionType compress) throws IOException {
this(conf, new Path(dirName), keyClass(keyClass),
- valueClass(valClass), compressionType(compress));
+ valueClass(valClass), compression(compress));
}
/** Create the named map using the named key comparator.
@@ -159,7 +158,7 @@ public class MapFile {
WritableComparator comparator, Class valClass,
SequenceFile.CompressionType compress) throws IOException {
this(conf, new Path(dirName), comparator(comparator),
- valueClass(valClass), compressionType(compress));
+ valueClass(valClass), compression(compress));
}
/** Create the named map using the named key comparator.
@@ -171,7 +170,7 @@ public class MapFile {
SequenceFile.CompressionType compress,
Progressable progress) throws IOException {
this(conf, new Path(dirName), comparator(comparator),
- valueClass(valClass), compressionType(compress),
+ valueClass(valClass), compression(compress),
progressable(progress));
}
@@ -184,8 +183,8 @@ public class MapFile {
SequenceFile.CompressionType compress, CompressionCodec codec,
Progressable progress) throws IOException {
this(conf, new Path(dirName), comparator(comparator),
- valueClass(valClass), compressionType(compress),
- compressionCodec(codec), progressable(progress));
+ valueClass(valClass), compression(compress, codec),
+ progressable(progress));
}
// our options are a superset of sequence file writer options
@@ -221,13 +220,14 @@ public class MapFile {
}
public static
- SequenceFile.Writer.Option compressionType(CompressionType value) {
- return SequenceFile.Writer.compressionType(value);
+ SequenceFile.Writer.Option compression(CompressionType type) {
+ return SequenceFile.Writer.compression(type);
}
public static
- SequenceFile.Writer.Option compressionCodec(CompressionCodec value) {
- return SequenceFile.Writer.compressionCodec(value);
+ SequenceFile.Writer.Option compression(CompressionType type,
+ CompressionCodec codec) {
+ return SequenceFile.Writer.compression(type, codec);
}
public static SequenceFile.Writer.Option progressable(Progressable value) {
@@ -274,11 +274,10 @@ public class MapFile {
this.data = SequenceFile.createWriter(conf, dataOptions);
SequenceFile.Writer.Option[] indexOptions =
- Options.prependOptions(opts,
- SequenceFile.Writer.file(indexFile),
- SequenceFile.Writer.keyClass(keyClass),
- SequenceFile.Writer.valueClass(LongWritable.class),
- SequenceFile.Writer.compressionType(CompressionType.BLOCK));
+ Options.prependOptions(opts, SequenceFile.Writer.file(indexFile),
+ SequenceFile.Writer.keyClass(keyClass),
+ SequenceFile.Writer.valueClass(LongWritable.class),
+ SequenceFile.Writer.compression(CompressionType.BLOCK));
this.index = SequenceFile.createWriter(conf, indexOptions);
}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/io/SequenceFile.java?rev=1004900&r1=1004899&r2=1004900&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/io/SequenceFile.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/io/SequenceFile.java Wed Oct 6 06:16:31 2010
@@ -252,22 +252,23 @@ public class SequenceFile {
*/
public static Writer createWriter(Configuration conf, Writer.Option... opts
) throws IOException {
- Writer.CompressionTypeOption compressionOption =
- Options.getOption(Writer.CompressionTypeOption.class, opts);
+ Writer.CompressionOption compressionOption =
+ Options.getOption(Writer.CompressionOption.class, opts);
CompressionType kind;
if (compressionOption != null) {
kind = compressionOption.getValue();
} else {
kind = getDefaultCompressionType(conf);
+ opts = Options.prependOptions(opts, Writer.compression(kind));
}
switch (kind) {
- default:
- case NONE:
- return new Writer(conf, kind, opts);
- case RECORD:
- return new RecordCompressWriter(conf, kind, opts);
- case BLOCK:
- return new BlockCompressWriter(conf, kind, opts);
+ default:
+ case NONE:
+ return new Writer(conf, opts);
+ case RECORD:
+ return new RecordCompressWriter(conf, opts);
+ case BLOCK:
+ return new BlockCompressWriter(conf, opts);
}
}
@@ -311,7 +312,7 @@ public class SequenceFile {
CompressionType compressionType) throws IOException {
return createWriter(conf, Writer.file(name), Writer.keyClass(keyClass),
Writer.valueClass(valClass),
- Writer.compressionType(compressionType));
+ Writer.compression(compressionType));
}
/**
@@ -335,7 +336,7 @@ public class SequenceFile {
Progressable progress) throws IOException {
return createWriter(conf, Writer.file(name), Writer.keyClass(keyClass),
Writer.valueClass(valClass),
- Writer.compressionType(compressionType),
+ Writer.compression(compressionType),
Writer.progressable(progress));
}
@@ -360,8 +361,7 @@ public class SequenceFile {
CompressionCodec codec) throws IOException {
return createWriter(conf, Writer.file(name), Writer.keyClass(keyClass),
Writer.valueClass(valClass),
- Writer.compressionType(compressionType),
- Writer.compressionCodec(codec));
+ Writer.compression(compressionType, codec));
}
/**
@@ -388,8 +388,7 @@ public class SequenceFile {
Progressable progress, Metadata metadata) throws IOException {
return createWriter(conf, Writer.file(name), Writer.keyClass(keyClass),
Writer.valueClass(valClass),
- Writer.compressionType(compressionType),
- Writer.compressionCodec(codec),
+ Writer.compression(compressionType, codec),
Writer.progressable(progress),
Writer.metadata(metadata));
}
@@ -425,8 +424,7 @@ public class SequenceFile {
Writer.bufferSize(bufferSize),
Writer.replication(replication),
Writer.blockSize(blockSize),
- Writer.compressionType(compressionType),
- Writer.compressionCodec(codec),
+ Writer.compression(compressionType, codec),
Writer.progressable(progress),
Writer.metadata(metadata));
}
@@ -454,8 +452,7 @@ public class SequenceFile {
Progressable progress) throws IOException {
return createWriter(conf, Writer.file(name), Writer.keyClass(keyClass),
Writer.valueClass(valClass),
- Writer.compressionType(compressionType),
- Writer.compressionCodec(codec),
+ Writer.compression(compressionType, codec),
Writer.progressable(progress));
}
@@ -481,8 +478,7 @@ public class SequenceFile {
CompressionCodec codec, Metadata metadata) throws IOException {
return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass),
Writer.valueClass(valClass),
- Writer.compressionType(compressionType),
- Writer.compressionCodec(codec),
+ Writer.compression(compressionType, codec),
Writer.metadata(metadata));
}
@@ -506,8 +502,7 @@ public class SequenceFile {
CompressionCodec codec) throws IOException {
return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass),
Writer.valueClass(valClass),
- Writer.compressionType(compressionType),
- Writer.compressionCodec(codec));
+ Writer.compression(compressionType, codec));
}
@@ -839,23 +834,23 @@ public class SequenceFile {
}
}
- private static class CompressionTypeOption implements Option {
+ private static class CompressionOption implements Option {
private final CompressionType value;
- CompressionTypeOption(CompressionType value) {
+ private final CompressionCodec codec;
+ CompressionOption(CompressionType value) {
+ this(value, null);
+ }
+ CompressionOption(CompressionType value, CompressionCodec codec) {
this.value = value;
+ this.codec = (CompressionType.NONE != value && null == codec)
+ ? new DefaultCodec()
+ : codec;
}
CompressionType getValue() {
return value;
}
- }
-
- private static class CompressionCodecOption implements Option {
- private final CompressionCodec value;
- CompressionCodecOption(CompressionCodec value) {
- this.value = value;
- }
- CompressionCodec getValue() {
- return value;
+ CompressionCodec getCodec() {
+ return codec;
}
}
@@ -895,25 +890,23 @@ public class SequenceFile {
return new MetadataOption(value);
}
- public static Option compressionType(CompressionType value) {
- return new CompressionTypeOption(value);
- }
-
- public static Option compressionCodec(CompressionCodec value) {
- return new CompressionCodecOption(value);
+ public static Option compression(CompressionType value) {
+ return new CompressionOption(value);
}
+ public static Option compression(CompressionType value,
+ CompressionCodec codec) {
+ return new CompressionOption(value, codec);
+ }
+
/**
* Construct a uncompressed writer from a set of options.
* @param conf the configuration to use
- * @param compressionType the compression type being used
* @param options the options used when creating the writer
* @throws IOException if it fails
*/
Writer(Configuration conf,
- CompressionType compressionType,
Option... opts) throws IOException {
- this.compress = compressionType;
BlockSizeOption blockSizeOption =
Options.getOption(BlockSizeOption.class, opts);
BufferSizeOption bufferSizeOption =
@@ -928,10 +921,10 @@ public class SequenceFile {
Options.getOption(KeyClassOption.class, opts);
ValueClassOption valueClassOption =
Options.getOption(ValueClassOption.class, opts);
- CompressionCodecOption compressionCodecOption =
- Options.getOption(CompressionCodecOption.class, opts);
MetadataOption metadataOption =
Options.getOption(MetadataOption.class, opts);
+ CompressionOption compressionTypeOption =
+ Options.getOption(CompressionOption.class, opts);
// check consistency of options
if ((fileOption == null) == (streamOption == null)) {
throw new IllegalArgumentException("file or stream must be specified");
@@ -968,13 +961,8 @@ public class SequenceFile {
Object.class : valueClassOption.getValue();
Metadata metadata = metadataOption == null ?
new Metadata() : metadataOption.getValue();
- CompressionCodec codec;
- if (compressionType == CompressionType.NONE) {
- codec = null;
- } else {
- codec = compressionCodecOption == null ?
- new DefaultCodec() : compressionCodecOption.getValue();
- }
+ this.compress = compressionTypeOption.getValue();
+ final CompressionCodec codec = compressionTypeOption.getCodec();
if (codec != null &&
(codec instanceof GzipCodec) &&
!NativeCodeLoader.isNativeCodeLoaded() &&
@@ -1207,9 +1195,8 @@ public class SequenceFile {
static class RecordCompressWriter extends Writer {
RecordCompressWriter(Configuration conf,
- CompressionType compressionType,
Option... options) throws IOException {
- super(conf, compressionType, options);
+ super(conf, options);
}
/** Append a key/value pair. */
@@ -1276,9 +1263,8 @@ public class SequenceFile {
private final int compressionBlockSize;
BlockCompressWriter(Configuration conf,
- CompressionType compressionType,
Option... options) throws IOException {
- super(conf, compressionType, options);
+ super(conf, options);
compressionBlockSize =
conf.getInt("io.seqfile.compress.blocksize", 1000000);
keySerializer.close();
@@ -2756,14 +2742,10 @@ public class SequenceFile {
}
long segmentStart = out.getPos();
- Writer writer = createWriter(conf,
- Writer.stream(out),
- Writer.keyClass(keyClass),
- Writer.valueClass(valClass),
- Writer.compressionType(compressionType),
- Writer.compressionCodec(codec),
- Writer.metadata(done ? metadata :
- new Metadata()));
+ Writer writer = createWriter(conf, Writer.stream(out),
+ Writer.keyClass(keyClass), Writer.valueClass(valClass),
+ Writer.compression(compressionType, codec),
+ Writer.metadata(done ? metadata : new Metadata()));
if (!done) {
writer.sync = null; // disable sync on temp files
@@ -2943,8 +2925,7 @@ public class SequenceFile {
Writer.file(outputFile),
Writer.keyClass(keyClass),
Writer.valueClass(valClass),
- Writer.compressionType(compress),
- Writer.compressionCodec(codec),
+ Writer.compression(compress, codec),
Writer.progressable(prog));
return writer;
}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/io/SetFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/io/SetFile.java?rev=1004900&r1=1004899&r2=1004900&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/io/SetFile.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/io/SetFile.java Wed Oct 6 06:16:31 2010
@@ -60,7 +60,7 @@ public class SetFile extends MapFile {
super(conf, new Path(dirName),
comparator(comparator),
valueClass(NullWritable.class),
- compressionType(compress));
+ compression(compress));
}
/** Append a key to a set. The key must be strictly greater than the