You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2017/04/12 22:52:58 UTC
[33/50] hadoop git commit: HADOOP-13665. Erasure Coding codec should
support fallback coder. Contributed by Kai Sasaki.
HADOOP-13665. Erasure Coding codec should support fallback coder. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f050afb5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f050afb5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f050afb5
Branch: refs/heads/HADOOP-13345
Commit: f050afb5785dc38875cf644fd4f80a219d4345e7
Parents: 90d9737
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Tue Apr 11 07:31:29 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Tue Apr 11 07:32:13 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/io/erasurecode/CodecUtil.java | 135 +++++++++++--------
.../src/main/resources/core-default.xml | 25 ++--
.../erasurecode/TestCodecRawCoderMapping.java | 71 +++++++++-
.../coder/TestHHXORErasureCoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 2 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +-
.../TestDFSStripedOutputStreamWithFailure.java | 2 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 2 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +-
10 files changed, 164 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f050afb5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 861451a..0c66df6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.io.erasurecode;
import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.erasurecode.codec.ErasureCodec;
@@ -26,6 +30,8 @@ import org.apache.hadoop.io.erasurecode.codec.RSErasureCodec;
import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec;
import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
@@ -35,6 +41,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
/**
* A codec & coder utility to help create coders conveniently.
@@ -49,41 +56,50 @@ import java.lang.reflect.InvocationTargetException;
@InterfaceAudience.Private
public final class CodecUtil {
+ private static final Log LOG = LogFactory.getLog(CodecUtil.class);
+
+ public static final String IO_ERASURECODE_CODEC = "io.erasurecode.codec.";
+
/** Erasure coder XOR codec. */
public static final String IO_ERASURECODE_CODEC_XOR_KEY =
- "io.erasurecode.codec.xor";
+ IO_ERASURECODE_CODEC + "xor";
public static final String IO_ERASURECODE_CODEC_XOR =
XORErasureCodec.class.getCanonicalName();
/** Erasure coder Reed-Solomon codec. */
public static final String IO_ERASURECODE_CODEC_RS_KEY =
- "io.erasurecode.codec.rs";
+ IO_ERASURECODE_CODEC + "rs";
public static final String IO_ERASURECODE_CODEC_RS =
RSErasureCodec.class.getCanonicalName();
/** Erasure coder hitch hiker XOR codec. */
public static final String IO_ERASURECODE_CODEC_HHXOR_KEY =
- "io.erasurecode.codec.hhxor";
+ IO_ERASURECODE_CODEC + "hhxor";
public static final String IO_ERASURECODE_CODEC_HHXOR =
HHXORErasureCodec.class.getCanonicalName();
- /** Supported erasure codec classes. */
-
- /** Raw coder factory for the RS codec. */
- public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
- "io.erasurecode.codec.rs.rawcoder";
- public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT =
- RSRawErasureCoderFactory.class.getCanonicalName();
-
- /** Raw coder factory for the RS legacy codec. */
- public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY =
- "io.erasurecode.codec.rs-legacy.rawcoder";
- public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT =
+ /** Comma separated raw codec name. The first coder is prior to the latter. */
+ public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY =
+ IO_ERASURECODE_CODEC + "rs-legacy.rawcoders";
+ public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT =
RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY =
+ IO_ERASURECODE_CODEC + "rs.rawcoders";
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT =
+ NativeRSRawErasureCoderFactory.class.getCanonicalName() +
+ "," + RSRawErasureCoderFactory.class.getCanonicalName();
/** Raw coder factory for the XOR codec. */
- public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY =
- "io.erasurecode.codec.xor.rawcoder";
- public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT =
- XORRawErasureCoderFactory.class.getCanonicalName();
+ public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY =
+ IO_ERASURECODE_CODEC + "xor.rawcoders";
+ public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT =
+ NativeXORRawErasureCoderFactory.class.getCanonicalName() +
+ "," + XORRawErasureCoderFactory.class.getCanonicalName();
+
+ // Default coders for each codec names.
+ public static final Map<String, String> DEFAULT_CODERS_MAP = ImmutableMap.of(
+ "rs", IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT,
+ "rs-legacy", IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT,
+ "xor", IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT
+ );
private CodecUtil() { }
@@ -133,12 +149,7 @@ public final class CodecUtil {
Preconditions.checkNotNull(conf);
Preconditions.checkNotNull(codec);
- String rawCoderFactoryKey = getRawCoderFactNameFromCodec(conf, codec);
-
- RawErasureCoderFactory fact = createRawCoderFactory(conf,
- rawCoderFactoryKey);
-
- return fact.createEncoder(coderOptions);
+ return createRawEncoderWithFallback(conf, codec, coderOptions);
}
/**
@@ -153,12 +164,7 @@ public final class CodecUtil {
Preconditions.checkNotNull(conf);
Preconditions.checkNotNull(codec);
- String rawCoderFactoryKey = getRawCoderFactNameFromCodec(conf, codec);
-
- RawErasureCoderFactory fact = createRawCoderFactory(conf,
- rawCoderFactoryKey);
-
- return fact.createDecoder(coderOptions);
+ return createRawDecoderWithFallback(conf, codec, coderOptions);
}
private static RawErasureCoderFactory createRawCoderFactory(
@@ -180,31 +186,52 @@ public final class CodecUtil {
return fact;
}
- private static String getRawCoderFactNameFromCodec(Configuration conf,
- String codec) {
- switch (codec) {
- case ErasureCodeConstants.RS_CODEC_NAME:
- return conf.get(
- IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
- IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT);
- case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
- return conf.get(
- IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
- IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT);
- case ErasureCodeConstants.XOR_CODEC_NAME:
- return conf.get(
- IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
- IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT);
- default:
- // For custom codec, we throw exception if the factory is not configured
- String rawCoderKey = "io.erasurecode.codec." + codec + ".rawcoder";
- String factName = conf.get(rawCoderKey);
- if (factName == null) {
- throw new IllegalArgumentException("Raw coder factory not configured " +
- "for custom codec " + codec);
+ // Return comma separated coder names
+ private static String getRawCoders(Configuration conf, String codec) {
+ return conf.get(
+ IO_ERASURECODE_CODEC + codec + ".rawcoders",
+ DEFAULT_CODERS_MAP.getOrDefault(codec, codec)
+ );
+ }
+
+ private static RawErasureEncoder createRawEncoderWithFallback(
+ Configuration conf, String codec, ErasureCoderOptions coderOptions) {
+ String coders = getRawCoders(conf, codec);
+ for (String factName : Splitter.on(",").split(coders)) {
+ try {
+ if (factName != null) {
+ RawErasureCoderFactory fact = createRawCoderFactory(conf,
+ factName);
+ return fact.createEncoder(coderOptions);
+ }
+ } catch (LinkageError | Exception e) {
+ // Fallback to next coder if possible
+ LOG.warn("Failed to create raw erasure encoder " + factName +
+ ", fallback to next codec if possible", e);
+ }
+ }
+ throw new IllegalArgumentException("Fail to create raw erasure " +
+ "encoder with given codec: " + codec);
+ }
+
+ private static RawErasureDecoder createRawDecoderWithFallback(
+ Configuration conf, String codec, ErasureCoderOptions coderOptions) {
+ String coders = getRawCoders(conf, codec);
+ for (String factName : Splitter.on(",").split(coders)) {
+ try {
+ if (factName != null) {
+ RawErasureCoderFactory fact = createRawCoderFactory(conf,
+ factName);
+ return fact.createDecoder(coderOptions);
+ }
+ } catch (LinkageError | Exception e) {
+ // Fallback to next coder if possible
+ LOG.warn("Failed to create raw erasure decoder " + factName +
+ ", fallback to next codec if possible", e);
}
- return factName;
}
+ throw new IllegalArgumentException("Fail to create raw erasure " +
+ "encoder with given codec: " + codec);
}
private static ErasureCodec createCodec(Configuration conf,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f050afb5/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 521b013..03e4996 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -657,34 +657,33 @@
</property>
<property>
- <name>io.erasurecode.codec.rs.rawcoder</name>
- <value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
+ <name>io.erasurecode.codec.rs.rawcoders</name>
+ <value>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
<description>
- Raw coder implementation for the rs codec. The default value is a
- pure Java implementation. There is also a native implementation. Its value
- is org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory.
+ Comma separated raw coder implementations for the rs codec. The earlier
+ factory is prior to followings in case of failure of creating raw coders.
</description>
</property>
<property>
- <name>io.erasurecode.codec.rs-legacy.rawcoder</name>
+ <name>io.erasurecode.codec.rs-legacy.rawcoders</name>
<value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy</value>
<description>
- Raw coder implementation for the rs-legacy codec.
+ Comma separated raw coder implementations for the rs-legacy codec. The earlier
+ factory is prior to followings in case of failure of creating raw coders.
</description>
</property>
<property>
- <name>io.erasurecode.codec.xor.rawcoder</name>
- <value>org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory</value>
+ <name>io.erasurecode.codec.xor.rawcoders</name>
+ <value>org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory</value>
<description>
- Raw coder implementation for the xor codec. The default value is a pure Java
- implementation. There is also a native implementation. Its value is
- org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory.
+ Comma separated raw coder implementations for the xor codec. The earlier
+ factory is prior to followings in case of failure of creating raw coders.
</description>
</property>
-<!-- file system properties -->
+ <!-- file system properties -->
<property>
<name>fs.defaultFS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f050afb5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
index 7f7fcf3..db31993 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
@@ -18,12 +18,16 @@
package org.apache.hadoop.io.erasurecode;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoderLegacy;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Before;
@@ -70,20 +74,73 @@ public class TestCodecRawCoderMapping {
numDataUnit, numParityUnit);
String dummyFactName = "DummyNoneExistingFactory";
- // set the dummy factory to rs-legacy and create a raw coder
- // with rs, which is OK as the raw coder key is not used
+ // set the dummy factory to raw coders then fail to create any rs raw coder.
conf.set(CodecUtil.
- IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName);
- RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf,
- ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
- Assert.assertTrue(encoder instanceof RSRawEncoder);
+ IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, dummyFactName);
+ try {
+ CodecUtil.createRawEncoder(conf,
+ ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
+ Assert.fail();
+ } catch (Exception e) {
+ GenericTestUtils.assertExceptionContains(
+ "Fail to create raw erasure encoder with given codec: rs", e);
+ }
+
// now create the raw coder with rs-legacy, which should throw exception
+ conf.set(CodecUtil.
+ IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY, dummyFactName);
try {
CodecUtil.createRawEncoder(conf,
ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
Assert.fail();
} catch (Exception e) {
- GenericTestUtils.assertExceptionContains("Failed to create raw coder", e);
+ GenericTestUtils.assertExceptionContains(
+ "Fail to create raw erasure encoder with given codec: rs", e);
}
}
+
+ @Test
+ public void testFallbackCoders() {
+ ErasureCoderOptions coderOptions = new ErasureCoderOptions(
+ numDataUnit, numParityUnit);
+ conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
+ RSRawErasureCoderFactory.class.getCanonicalName() +
+ "," + NativeRSRawErasureCoderFactory.class.getCanonicalName());
+ // should return default raw coder of rs codec
+ RawErasureEncoder encoder = CodecUtil.createRawEncoder(
+ conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
+ Assert.assertTrue(encoder instanceof RSRawEncoder);
+ RawErasureDecoder decoder = CodecUtil.createRawDecoder(
+ conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
+ Assert.assertTrue(decoder instanceof RSRawDecoder);
+ }
+
+ @Test
+ public void testLegacyCodecFallback() {
+ ErasureCoderOptions coderOptions = new ErasureCoderOptions(
+ numDataUnit, numParityUnit);
+ // should return default raw coder of rs-legacy codec
+ RawErasureEncoder encoder = CodecUtil.createRawEncoder(
+ conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
+ Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
+ RawErasureDecoder decoder = CodecUtil.createRawDecoder(
+ conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
+ Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
+ }
+
+ @Test
+ public void testIgnoreInvalidCodec() {
+ ErasureCoderOptions coderOptions = new ErasureCoderOptions(
+ numDataUnit, numParityUnit);
+ conf.set(CodecUtil.IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY,
+ "invalid-codec," +
+ "org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory");
+ // should return second coder specified by IO_ERASURECODE_CODEC_CODERS
+ RawErasureEncoder encoder = CodecUtil.createRawEncoder(
+ conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+ Assert.assertTrue(encoder instanceof XORRawEncoder);
+ RawErasureDecoder decoder = CodecUtil.createRawDecoder(
+ conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+ Assert.assertTrue(decoder instanceof XORRawDecoder);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f050afb5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
index 91c4ba8..a475a39 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
@@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f050afb5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
index 3b18347..11abac4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
@@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f050afb5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 68fde95..e3332fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -97,7 +97,7 @@ public class TestDFSStripedInputStream {
getEcPolicy().getName());
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
SimulatedFSDataset.setFactory(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f050afb5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index ebdecfd..a2d5c8d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -84,7 +84,7 @@ public class TestDFSStripedOutputStream {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
DFSTestUtil.enableAllECPolicies(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f050afb5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index c66c7f2..66a805c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -213,7 +213,7 @@ public class TestDFSStripedOutputStreamWithFailure {
final int numDNs = dataBlocks + parityBlocks;
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
DFSTestUtil.enableAllECPolicies(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f050afb5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index affb541..2bd4a90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -99,7 +99,7 @@ public class TestReconstructStripedFile {
false);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f050afb5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
index cc2453b..6379db5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -67,7 +67,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
DFSTestUtil.enableAllECPolicies(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org