You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/07/27 11:33:01 UTC
[06/50] [abbrv] hadoop git commit: HADOOP-15612. Improve exception
when tfile fails to load LzoCodec. (gera)
HADOOP-15612. Improve exception when tfile fails to load LzoCodec. (gera)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bec03cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bec03cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bec03cf
Branch: refs/heads/YARN-3409
Commit: 6bec03cfc8bdcf6aa3df9c22231ab959ba31f2f5
Parents: ea2c6c8
Author: Gera Shegalov <ge...@apache.org>
Authored: Tue Jul 17 00:05:39 2018 -0700
Committer: Gera Shegalov <ge...@apache.org>
Committed: Tue Jul 24 14:32:30 2018 -0700
----------------------------------------------------------------------
.../hadoop/io/file/tfile/Compression.java | 31 +++++++++++-------
.../hadoop/io/file/tfile/TestCompression.java | 34 +++++++++++++++++++-
2 files changed, 53 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bec03cf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
index fa85ed7..c4347e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
@@ -5,9 +5,9 @@
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
@@ -24,6 +24,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -78,25 +79,33 @@ public final class Compression {
public enum Algorithm {
LZO(TFile.COMPRESSION_LZO) {
private transient boolean checked = false;
+ private transient ClassNotFoundException cnf;
+ private transient boolean reinitCodecInTests;
private static final String defaultClazz =
"org.apache.hadoop.io.compress.LzoCodec";
+ private transient String clazz;
private transient CompressionCodec codec = null;
+ private String getLzoCodecClass() {
+ String extClazzConf = conf.get(CONF_LZO_CLASS);
+ String extClazz = (extClazzConf != null) ?
+ extClazzConf : System.getProperty(CONF_LZO_CLASS);
+ return (extClazz != null) ? extClazz : defaultClazz;
+ }
+
@Override
public synchronized boolean isSupported() {
- if (!checked) {
+ if (!checked || reinitCodecInTests) {
checked = true;
- String extClazzConf = conf.get(CONF_LZO_CLASS);
- String extClazz = (extClazzConf != null) ?
- extClazzConf : System.getProperty(CONF_LZO_CLASS);
- String clazz = (extClazz != null) ? extClazz : defaultClazz;
+ reinitCodecInTests = conf.getBoolean("test.reload.lzo.codec", false);
+ clazz = getLzoCodecClass();
try {
LOG.info("Trying to load Lzo codec class: " + clazz);
codec =
(CompressionCodec) ReflectionUtils.newInstance(Class
.forName(clazz), conf);
} catch (ClassNotFoundException e) {
- // that is okay
+ cnf = e;
}
}
return codec != null;
@@ -105,9 +114,9 @@ public final class Compression {
@Override
CompressionCodec getCodec() throws IOException {
if (!isSupported()) {
- throw new IOException(
- "LZO codec class not specified. Did you forget to set property "
- + CONF_LZO_CLASS + "?");
+ throw new IOException(String.format(
+ "LZO codec %s=%s could not be loaded", CONF_LZO_CLASS, clazz),
+ cnf);
}
return codec;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bec03cf/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
index ff6c72a..b1bf077 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
@@ -17,14 +17,28 @@
*/
package org.apache.hadoop.io.file.tfile;
-import org.junit.Test;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.*;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class TestCompression {
+ @BeforeClass
+ public static void resetConfigBeforeAll() {
+ Compression.Algorithm.LZO.conf.setBoolean("test.reload.lzo.codec", true);
+ }
+
+ @AfterClass
+ public static void resetConfigAfterAll() {
+ Compression.Algorithm.LZO.conf.setBoolean("test.reload.lzo.codec", false);
+ }
+
/**
* Regression test for HADOOP-11418.
* Verify we can set a LZO codec different from default LZO codec.
@@ -38,4 +52,22 @@ public class TestCompression {
assertEquals(defaultCodec,
Compression.Algorithm.LZO.getCodec().getClass().getName());
}
+
+
+ @Test
+ public void testMisconfiguredLZOCodec() throws Exception {
+ // Dummy codec
+ String defaultCodec = "org.apache.hadoop.io.compress.InvalidLzoCodec";
+ Compression.Algorithm.conf.set(
+ Compression.Algorithm.CONF_LZO_CLASS, defaultCodec);
+ IOException ioEx = LambdaTestUtils.intercept(
+ IOException.class,
+ defaultCodec,
+ () -> Compression.Algorithm.LZO.getCodec());
+
+ if (!(ioEx.getCause() instanceof ClassNotFoundException)) {
+ throw ioEx;
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org