You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2012/04/19 21:28:28 UTC
svn commit: r1328085 - in
/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/io/compress/ src/main/resources/
src/main/resources/META-INF/services/
src/test/java/org/apache/hadoop/io/compress/
Author: tomwhite
Date: Thu Apr 19 19:28:27 2012
New Revision: 1328085
URL: http://svn.apache.org/viewvc?rev=1328085&view=rev
Log:
Merge -r 1328082:1328083 from trunk to branch-2. Fixes: HADOOP-7350
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
- copied unchanged from r1328083, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1328085&r1=1328084&r2=1328085&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Apr 19 19:28:27 2012
@@ -242,6 +242,9 @@ Release 2.0.0 - UNRELEASED
HADOOP-8282. start-all.sh refers incorrectly start-dfs.sh
existence for starting start-yarn.sh. (Devaraj K via eli)
+ HADOOP-7350. Use ServiceLoader to discover compression codec classes.
+ (tomwhite)
+
BREAKDOWN OF HADOOP-7454 SUBTASKS
HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java?rev=1328085&r1=1328084&r2=1328085&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java Thu Apr 19 19:28:27 2012
@@ -36,6 +36,9 @@ public class CompressionCodecFactory {
public static final Log LOG =
LogFactory.getLog(CompressionCodecFactory.class.getName());
+
+ private static final ServiceLoader<CompressionCodec> CODEC_PROVIDERS =
+ ServiceLoader.load(CompressionCodec.class);
/**
* A map from the reversed filename suffixes to the codecs.
@@ -95,16 +98,23 @@ public class CompressionCodecFactory {
}
/**
- * Get the list of codecs listed in the configuration
+ * Get the list of codecs discovered via a Java ServiceLoader, or
+ * listed in the configuration. Codecs specified in configuration come
+ * later in the returned list, and are considered to override those
+ * from the ServiceLoader.
* @param conf the configuration to look in
- * @return a list of the Configuration classes or null if the attribute
- * was not set
+ * @return a list of the {@link CompressionCodec} classes
*/
public static List<Class<? extends CompressionCodec>> getCodecClasses(Configuration conf) {
+ List<Class<? extends CompressionCodec>> result
+ = new ArrayList<Class<? extends CompressionCodec>>();
+ // Add codec classes discovered via service loading
+ for (CompressionCodec codec : CODEC_PROVIDERS) {
+ result.add(codec.getClass());
+ }
+ // Add codec classes from configuration
String codecsString = conf.get("io.compression.codecs");
if (codecsString != null) {
- List<Class<? extends CompressionCodec>> result
- = new ArrayList<Class<? extends CompressionCodec>>();
StringTokenizer codecSplit = new StringTokenizer(codecsString, ",");
while (codecSplit.hasMoreElements()) {
String codecSubstring = codecSplit.nextToken();
@@ -123,14 +133,14 @@ public class CompressionCodecFactory {
}
}
}
- return result;
- } else {
- return null;
}
+ return result;
}
/**
- * Sets a list of codec classes in the configuration.
+ * Sets a list of codec classes in the configuration. In addition to any
+ * classes specified using this method, {@link CompressionCodec} classes on
+ * the classpath are discovered using a Java ServiceLoader.
* @param conf the configuration to modify
* @param classes the list of classes to set
*/
@@ -151,21 +161,19 @@ public class CompressionCodecFactory {
/**
* Find the codecs specified in the config value io.compression.codecs
- * and register them. Defaults to gzip and zip.
+ * and register them. Defaults to gzip and deflate.
*/
public CompressionCodecFactory(Configuration conf) {
codecs = new TreeMap<String, CompressionCodec>();
codecsByClassName = new HashMap<String, CompressionCodec>();
codecsByName = new HashMap<String, CompressionCodec>();
List<Class<? extends CompressionCodec>> codecClasses = getCodecClasses(conf);
- if (codecClasses == null) {
+ if (codecClasses == null || codecClasses.isEmpty()) {
addCodec(new GzipCodec());
addCodec(new DefaultCodec());
} else {
- Iterator<Class<? extends CompressionCodec>> itr = codecClasses.iterator();
- while (itr.hasNext()) {
- CompressionCodec codec = ReflectionUtils.newInstance(itr.next(), conf);
- addCodec(codec);
+ for (Class<? extends CompressionCodec> codecClass : codecClasses) {
+ addCodec(ReflectionUtils.newInstance(codecClass, conf));
}
}
}
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1328085&r1=1328084&r2=1328085&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Thu Apr 19 19:28:27 2012
@@ -268,9 +268,11 @@
<property>
<name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
- <description>A list of the compression codec classes that can be used
- for compression/decompression.</description>
+ <value></value>
+ <description>A comma-separated list of the compression codec classes that can
+ be used for compression/decompression. In addition to any classes specified
+ with this property (which take precedence), codec classes on the classpath
+ are discovered using a Java ServiceLoader.</description>
</property>
<property>
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java?rev=1328085&r1=1328084&r2=1328085&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java Thu Apr 19 19:28:27 2012
@@ -101,6 +101,12 @@ public class TestCodecFactory extends Te
}
}
+ private static class NewGzipCodec extends BaseCodec {
+ public String getDefaultExtension() {
+ return ".gz";
+ }
+ }
+
/**
* Returns a factory for a given set of codecs
* @param classes the codec classes to include
@@ -167,32 +173,43 @@ public class TestCodecFactory extends Te
checkCodec("default factory for deflate codec", DeflateCodec.class, codec);
factory = setClasses(new Class[0]);
+ // gz, bz2, snappy, lz4 are picked up by service loader, but bar isn't
codec = factory.getCodec(new Path("/tmp/foo.bar"));
- assertEquals("empty codec bar codec", null, codec);
+ assertEquals("empty factory bar codec", null, codec);
codec = factory.getCodecByClassName(BarCodec.class.getCanonicalName());
- assertEquals("empty codec bar codec", null, codec);
+ assertEquals("empty factory bar codec", null, codec);
codec = factory.getCodec(new Path("/tmp/foo.gz"));
- assertEquals("empty codec gz codec", null, codec);
+ checkCodec("empty factory gz codec", GzipCodec.class, codec);
codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName());
- assertEquals("empty codec gz codec", null, codec);
+ checkCodec("empty factory gz codec", GzipCodec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.bz2"));
- assertEquals("empty factory for .bz2", null, codec);
+ checkCodec("empty factory for .bz2", BZip2Codec.class, codec);
codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName());
- assertEquals("empty factory for bzip2 codec", null, codec);
+ checkCodec("empty factory for bzip2 codec", BZip2Codec.class, codec);
+
+ codec = factory.getCodec(new Path("/tmp/foo.snappy"));
+ checkCodec("empty factory snappy codec", SnappyCodec.class, codec);
+ codec = factory.getCodecByClassName(SnappyCodec.class.getCanonicalName());
+ checkCodec("empty factory snappy codec", SnappyCodec.class, codec);
+
+ codec = factory.getCodec(new Path("/tmp/foo.lz4"));
+ checkCodec("empty factory lz4 codec", Lz4Codec.class, codec);
+ codec = factory.getCodecByClassName(Lz4Codec.class.getCanonicalName());
+ checkCodec("empty factory lz4 codec", Lz4Codec.class, codec);
factory = setClasses(new Class[]{BarCodec.class, FooCodec.class,
FooBarCodec.class});
codec = factory.getCodec(new Path("/tmp/.foo.bar.gz"));
- assertEquals("full factory gz codec", null, codec);
+ checkCodec("full factory gz codec", GzipCodec.class, codec);
codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName());
- assertEquals("full codec gz codec", null, codec);
+ checkCodec("full codec gz codec", GzipCodec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.bz2"));
- assertEquals("full factory for .bz2", null, codec);
+ checkCodec("full factory for .bz2", BZip2Codec.class, codec);
codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName());
- assertEquals("full codec bzip2 codec", null, codec);
+ checkCodec("full codec bzip2 codec", BZip2Codec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.bar"));
checkCodec("full factory bar codec", BarCodec.class, codec);
@@ -220,5 +237,11 @@ public class TestCodecFactory extends Te
checkCodec("full factory foo codec", FooCodec.class, codec);
codec = factory.getCodecByName("FOO");
checkCodec("full factory foo codec", FooCodec.class, codec);
+
+ factory = setClasses(new Class[]{NewGzipCodec.class});
+ codec = factory.getCodec(new Path("/tmp/foo.gz"));
+ checkCodec("overridden factory for .gz", NewGzipCodec.class, codec);
+ codec = factory.getCodecByClassName(NewGzipCodec.class.getCanonicalName());
+ checkCodec("overridden factory for gzip codec", NewGzipCodec.class, codec);
}
}