You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/08/22 23:28:50 UTC
git commit: TEZ-1474. detect missing native libraries for compression
at the beginning of a task rather than at the end. Contributed by Prakash
Ramachandran.
Repository: tez
Updated Branches:
refs/heads/master 4842c3a67 -> c5d3b1b32
TEZ-1474. detect missing native libraries for compression at the
beginning of a task rather than at the end. Contributed by Prakash
Ramachandran.
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c5d3b1b3
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c5d3b1b3
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c5d3b1b3
Branch: refs/heads/master
Commit: c5d3b1b327e9da04aa73a4f65377269c85fdaeb9
Parents: 4842c3a
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Aug 22 14:28:18 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Fri Aug 22 14:28:18 2014 -0700
----------------------------------------------------------------------
.../library/common/sort/impl/ExternalSorter.java | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/c5d3b1b3/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
index 00f985e..bb8c912 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.Compressor;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.io.serializer.SerializationFactory;
import org.apache.hadoop.io.serializer.Serializer;
@@ -182,6 +183,23 @@ public abstract class ExternalSorter {
Class<? extends CompressionCodec> codecClass =
ConfigUtils.getIntermediateOutputCompressorClass(this.conf, DefaultCodec.class);
codec = ReflectionUtils.newInstance(codecClass, this.conf);
+
+ if (codec != null) {
+ Class<? extends Compressor> compressorType = null;
+ Throwable cause = null;
+ try {
+ compressorType = codec.getCompressorType();
+ } catch (RuntimeException e) {
+ cause = e;
+ }
+ if (compressorType == null) {
+ String errMsg =
+ String.format("Unable to get CompressorType for codec (%s). This is most" +
+ " likely due to missing native libraries for the codec.",
+ conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC));
+ throw new IOException(errMsg, cause);
+ }
+ }
} else {
codec = null;
}