You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2012/07/16 10:33:13 UTC
svn commit: r1361933 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt
src/main/java/org/apache/hadoop/io/SequenceFile.java
src/test/java/org/apache/hadoop/io/TestSequenceFile.java
Author: harsh
Date: Mon Jul 16 08:33:13 2012
New Revision: 1361933
URL: http://svn.apache.org/viewvc?rev=1361933&view=rev
Log:
HADOOP-8531. SequenceFile Writer can throw out a better error if a serializer or deserializer isn't available. Contributed by Madhukara Phatak. (harsh)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1361933&r1=1361932&r2=1361933&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Jul 16 08:33:13 2012
@@ -261,6 +261,10 @@ Branch-2 ( Unreleased changes )
HADOOP-7818. DiskChecker#checkDir should fail if the directory is
not executable. (Madhukara Phatak via harsh)
+ HADOOP-8531. SequenceFile Writer can throw out a better error if a
+ serializer or deserializer isn't available
+ (Madhukara Phatak via harsh)
+
BUG FIXES
HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java?rev=1361933&r1=1361932&r2=1361933&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java Mon Jul 16 08:33:13 2012
@@ -1161,8 +1161,26 @@ public class SequenceFile {
this.metadata = metadata;
SerializationFactory serializationFactory = new SerializationFactory(conf);
this.keySerializer = serializationFactory.getSerializer(keyClass);
+ if (this.keySerializer == null) {
+ throw new IOException(
+ "Could not find a serializer for the Key class: '"
+ + keyClass.getCanonicalName() + "'. "
+ + "Please ensure that the configuration '" +
+ CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is "
+ + "properly configured, if you're using"
+ + "custom serialization.");
+ }
this.keySerializer.open(buffer);
this.uncompressedValSerializer = serializationFactory.getSerializer(valClass);
+ if (this.uncompressedValSerializer == null) {
+ throw new IOException(
+ "Could not find a serializer for the Value class: '"
+ + valClass.getCanonicalName() + "'. "
+ + "Please ensure that the configuration '" +
+ CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is "
+ + "properly configured, if you're using"
+ + "custom serialization.");
+ }
this.uncompressedValSerializer.open(buffer);
if (this.codec != null) {
ReflectionUtils.setConf(this.codec, this.conf);
@@ -1171,6 +1189,15 @@ public class SequenceFile {
this.deflateOut =
new DataOutputStream(new BufferedOutputStream(deflateFilter));
this.compressedValSerializer = serializationFactory.getSerializer(valClass);
+ if (this.compressedValSerializer == null) {
+ throw new IOException(
+ "Could not find a serializer for the Value class: '"
+ + valClass.getCanonicalName() + "'. "
+ + "Please ensure that the configuration '" +
+ CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is "
+ + "properly configured, if you're using"
+ + "custom serialization.");
+ }
this.compressedValSerializer.open(deflateOut);
}
writeFileHeader();
@@ -1898,6 +1925,15 @@ public class SequenceFile {
new SerializationFactory(conf);
this.keyDeserializer =
getDeserializer(serializationFactory, getKeyClass());
+ if (this.keyDeserializer == null) {
+ throw new IOException(
+ "Could not find a deserializer for the Key class: '"
+ + getKeyClass().getCanonicalName() + "'. "
+ + "Please ensure that the configuration '" +
+ CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is "
+ + "properly configured, if you're using "
+ + "custom serialization.");
+ }
if (!blockCompressed) {
this.keyDeserializer.open(valBuffer);
} else {
@@ -1905,6 +1941,15 @@ public class SequenceFile {
}
this.valDeserializer =
getDeserializer(serializationFactory, getValueClass());
+ if (this.valDeserializer == null) {
+ throw new IOException(
+ "Could not find a deserializer for the Value class: '"
+ + getValueClass().getCanonicalName() + "'. "
+ + "Please ensure that the configuration '" +
+ CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is "
+ + "properly configured, if you're using "
+ + "custom serialization.");
+ }
this.valDeserializer.open(valIn);
}
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java?rev=1361933&r1=1361932&r2=1361933&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java Mon Jul 16 08:33:13 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.io.SequenceFile
import org.apache.hadoop.io.SequenceFile.Metadata;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.hadoop.io.serializer.avro.AvroReflectSerialization;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.conf.*;
import org.mockito.Mockito;
@@ -557,6 +558,60 @@ public class TestSequenceFile extends Te
// should succeed, fails if exception thrown
}
+ public void testSerializationAvailability() throws IOException {
+ Configuration conf = new Configuration();
+ Path path = new Path(System.getProperty("test.build.data", "."),
+ "serializationAvailability");
+ // Check if any serializers aren't found.
+ try {
+ SequenceFile.createWriter(
+ conf,
+ SequenceFile.Writer.file(path),
+ SequenceFile.Writer.keyClass(String.class),
+ SequenceFile.Writer.valueClass(NullWritable.class));
+ // Note: This may also fail someday if JavaSerialization
+ // is activated by default.
+ fail("Must throw IOException for missing serializer for the Key class");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().startsWith(
+ "Could not find a serializer for the Key class: '" +
+ String.class.getName() + "'."));
+ }
+ try {
+ SequenceFile.createWriter(
+ conf,
+ SequenceFile.Writer.file(path),
+ SequenceFile.Writer.keyClass(NullWritable.class),
+ SequenceFile.Writer.valueClass(String.class));
+ // Note: This may also fail someday if JavaSerialization
+ // is activated by default.
+ fail("Must throw IOException for missing serializer for the Value class");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().startsWith(
+ "Could not find a serializer for the Value class: '" +
+ String.class.getName() + "'."));
+ }
+
+ // Write a simple file to test deserialization failures with
+ writeTest(FileSystem.get(conf), 1, 1, path, CompressionType.NONE, null);
+
+ // Remove Writable serializations, to enforce error.
+ conf.setStrings(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY,
+ AvroReflectSerialization.class.getName());
+
+ // Now check if any deserializers aren't found.
+ try {
+ new SequenceFile.Reader(
+ conf,
+ SequenceFile.Reader.file(path));
+ fail("Must throw IOException for missing deserializer for the Key class");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().startsWith(
+ "Could not find a deserializer for the Key class: '" +
+ RandomDatum.class.getName() + "'."));
+ }
+ }
+
/** For debugging and testing. */
public static void main(String[] args) throws Exception {
int count = 1024 * 1024;