You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2016/04/22 20:58:41 UTC
[18/50] [abbrv] hadoop git commit: HADOOP-12924. Configure raw
erasure coders for supported codecs. Contributed by Rui Li.
HADOOP-12924. Configure raw erasure coders for supported codecs. Contributed by Rui Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5d4c7dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5d4c7dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5d4c7dc
Branch: refs/heads/yarn-2877
Commit: b5d4c7dc76ddb3e0af95d792c2cbc0f99353a42a
Parents: be0bce1
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Apr 20 15:47:01 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Wed Apr 20 15:47:01 2016 +0800
----------------------------------------------------------------------
.../hadoop/fs/CommonConfigurationKeys.java | 19 +++-
.../apache/hadoop/io/erasurecode/CodecUtil.java | 114 +++++++++++--------
.../io/erasurecode/ErasureCodeConstants.java | 40 +++++++
.../erasurecode/coder/HHXORErasureEncoder.java | 2 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 1 +
.../io/erasurecode/coder/RSErasureEncoder.java | 3 +-
.../src/main/resources/core-default.xml | 8 ++
.../erasurecode/TestCodecRawCoderMapping.java | 85 ++++++++++++++
.../hadoop/io/erasurecode/TestCoderBase.java | 2 +-
.../coder/TestHHXORErasureCoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 2 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 2 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 2 +-
.../hdfs/protocol/ErasureCodingPolicy.java | 4 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 5 +-
.../erasurecode/StripedReconstructor.java | 2 +-
.../namenode/ErasureCodingPolicyManager.java | 14 ++-
.../src/site/markdown/HDFSErasureCoding.md | 6 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 3 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +-
.../test/resources/testErasureCodingConf.xml | 12 +-
21 files changed, 255 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index a708900..9f52447 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -21,6 +21,9 @@ package org.apache.hadoop.fs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.http.lib.StaticUserWebFilter;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
+import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
/**
* This class contains constants for configuration keys used
@@ -156,13 +159,23 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
/** Supported erasure codec classes */
public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs";
- /** Raw coder factory for the RS codec. */
- public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
- "io.erasurecode.codec.rs.rawcoder";
+ /** Raw coder factory for the RS default codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY =
+ "io.erasurecode.codec.rs-default.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT =
+ RSRawErasureCoderFactory.class.getCanonicalName();
+
+ /** Raw coder factory for the RS legacy codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY =
+ "io.erasurecode.codec.rs-legacy.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT =
+ RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
/** Raw coder factory for the XOR codec. */
public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY =
"io.erasurecode.codec.xor.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT =
+ XORRawErasureCoderFactory.class.getCanonicalName();
/**
* Service Authorization
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index a2354b6..fcce071 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -17,17 +17,14 @@
*/
package org.apache.hadoop.io.erasurecode;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
-import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
-import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
-import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
/**
* A codec & coder utility to help create raw coders conveniently.
@@ -42,40 +39,56 @@ public final class CodecUtil {
* @param conf configuration possibly with some items to configure the coder
* @param numDataUnits number of data units in a coding group
* @param numParityUnits number of parity units in a coding group
+ * @param codec the codec to use. If null, will use the default codec
* @return raw encoder
*/
public static RawErasureEncoder createRSRawEncoder(
- Configuration conf, int numDataUnits, int numParityUnits) {
- RawErasureCoder rawCoder = createRawCoder(conf,
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
- true, numDataUnits, numParityUnits);
- if (rawCoder == null) {
- rawCoder = new RSRawEncoder(numDataUnits, numParityUnits);
+ Configuration conf, int numDataUnits, int numParityUnits, String codec) {
+ Preconditions.checkNotNull(conf);
+ if (codec == null) {
+ codec = ErasureCodeConstants.RS_DEFAULT_CODEC_NAME;
}
-
+ RawErasureCoder rawCoder = createRawCoder(conf,
+ getFactNameFromCodec(conf, codec), true, numDataUnits, numParityUnits);
return (RawErasureEncoder) rawCoder;
}
/**
+ * Create RS raw encoder using the default codec.
+ */
+ public static RawErasureEncoder createRSRawEncoder(
+ Configuration conf, int numDataUnits, int numParityUnits) {
+ return createRSRawEncoder(conf, numDataUnits, numParityUnits, null);
+ }
+
+ /**
* Create RS raw decoder according to configuration.
* @param conf configuration possibly with some items to configure the coder
* @param numDataUnits number of data units in a coding group
* @param numParityUnits number of parity units in a coding group
+ * @param codec the codec to use. If null, will use the default codec
* @return raw decoder
*/
public static RawErasureDecoder createRSRawDecoder(
- Configuration conf, int numDataUnits, int numParityUnits) {
- RawErasureCoder rawCoder = createRawCoder(conf,
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
- false, numDataUnits, numParityUnits);
- if (rawCoder == null) {
- rawCoder = new RSRawDecoder(numDataUnits, numParityUnits);
+ Configuration conf, int numDataUnits, int numParityUnits, String codec) {
+ Preconditions.checkNotNull(conf);
+ if (codec == null) {
+ codec = ErasureCodeConstants.RS_DEFAULT_CODEC_NAME;
}
-
+ RawErasureCoder rawCoder = createRawCoder(conf,
+ getFactNameFromCodec(conf, codec), false, numDataUnits, numParityUnits);
return (RawErasureDecoder) rawCoder;
}
/**
+ * Create RS raw decoder using the default codec.
+ */
+ public static RawErasureDecoder createRSRawDecoder(
+ Configuration conf, int numDataUnits, int numParityUnits) {
+ return createRSRawDecoder(conf, numDataUnits, numParityUnits, null);
+ }
+
+ /**
* Create XOR raw encoder according to configuration.
* @param conf configuration possibly with some items to configure the coder
* @param numDataUnits number of data units in a coding group
@@ -84,13 +97,10 @@ public final class CodecUtil {
*/
public static RawErasureEncoder createXORRawEncoder(
Configuration conf, int numDataUnits, int numParityUnits) {
+ Preconditions.checkNotNull(conf);
RawErasureCoder rawCoder = createRawCoder(conf,
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
+ getFactNameFromCodec(conf, ErasureCodeConstants.XOR_CODEC_NAME),
true, numDataUnits, numParityUnits);
- if (rawCoder == null) {
- rawCoder = new XORRawEncoder(numDataUnits, numParityUnits);
- }
-
return (RawErasureEncoder) rawCoder;
}
@@ -103,51 +113,65 @@ public final class CodecUtil {
*/
public static RawErasureDecoder createXORRawDecoder(
Configuration conf, int numDataUnits, int numParityUnits) {
+ Preconditions.checkNotNull(conf);
RawErasureCoder rawCoder = createRawCoder(conf,
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
+ getFactNameFromCodec(conf, ErasureCodeConstants.XOR_CODEC_NAME),
false, numDataUnits, numParityUnits);
- if (rawCoder == null) {
- rawCoder = new XORRawDecoder(numDataUnits, numParityUnits);
- }
-
return (RawErasureDecoder) rawCoder;
}
/**
* Create raw coder using specified conf and raw coder factory key.
* @param conf configuration possibly with some items to configure the coder
- * @param rawCoderFactoryKey configuration key to find the raw coder factory
+ * @param rawCoderFactory name of the raw coder factory
* @param isEncoder is encoder or not we're going to create
* @param numDataUnits number of data units in a coding group
* @param numParityUnits number of parity units in a coding group
* @return raw coder
*/
public static RawErasureCoder createRawCoder(Configuration conf,
- String rawCoderFactoryKey, boolean isEncoder, int numDataUnits,
+ String rawCoderFactory, boolean isEncoder, int numDataUnits,
int numParityUnits) {
- if (conf == null) {
- return null;
- }
-
- Class<? extends RawErasureCoderFactory> factClass = null;
- factClass = conf.getClass(rawCoderFactoryKey,
- factClass, RawErasureCoderFactory.class);
-
- if (factClass == null) {
- return null;
- }
-
RawErasureCoderFactory fact;
try {
+ Class<? extends RawErasureCoderFactory> factClass = conf.getClassByName(
+ rawCoderFactory).asSubclass(RawErasureCoderFactory.class);
fact = factClass.newInstance();
- } catch (InstantiationException e) {
- throw new RuntimeException("Failed to create raw coder", e);
- } catch (IllegalAccessException e) {
+ } catch (ClassNotFoundException | InstantiationException |
+ IllegalAccessException e) {
throw new RuntimeException("Failed to create raw coder", e);
}
return isEncoder ? fact.createEncoder(numDataUnits, numParityUnits) :
fact.createDecoder(numDataUnits, numParityUnits);
}
+
+ private static String getFactNameFromCodec(Configuration conf, String codec) {
+ switch (codec) {
+ case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
+ return conf.get(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CommonConfigurationKeys.
+ IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT);
+ case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
+ return conf.get(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
+ CommonConfigurationKeys.
+ IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT);
+ case ErasureCodeConstants.XOR_CODEC_NAME:
+ return conf.get(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT);
+ default:
+ // For custom codec, we throw exception if the factory is not configured
+ String rawCoderKey = "io.erasurecode.codec." + codec + ".rawcoder";
+ String factName = conf.get(rawCoderKey);
+ if (factName == null) {
+ throw new IllegalArgumentException("Raw coder factory not configured " +
+ "for custom codec " + codec);
+ }
+ return factName;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
new file mode 100644
index 0000000..1fb9488
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+/**
+ * Constants related to the erasure code feature.
+ */
+public final class ErasureCodeConstants {
+
+ private ErasureCodeConstants(){
+ }
+
+ public static final String RS_DEFAULT_CODEC_NAME = "rs-default";
+ public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
+ public static final String XOR_CODEC_NAME = "xor";
+
+ public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
+ RS_DEFAULT_CODEC_NAME, 6, 3);
+
+ public static final ECSchema RS_3_2_SCHEMA = new ECSchema(
+ RS_DEFAULT_CODEC_NAME, 3, 2);
+
+ public static final ECSchema RS_6_3_LEGACY_SCHEMA = new ECSchema(
+ RS_LEGACY_CODEC_NAME, 6, 3);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
index f30572f..a402469 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
@@ -65,7 +65,7 @@ public class HHXORErasureEncoder extends AbstractErasureEncoder {
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rsRawEncoder == null) {
rsRawEncoder = CodecUtil.createRSRawEncoder(getConf(),
- getNumDataUnits(), getNumParityUnits());
+ getNumDataUnits(), getNumParityUnits());
}
return rsRawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
index 6728945..47efd29 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
@@ -54,6 +54,7 @@ public class RSErasureDecoder extends AbstractErasureDecoder {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
+ // TODO: we should create the raw coder according to codec.
rsRawDecoder = CodecUtil.createRSRawDecoder(getConf(),
getNumDataUnits(), getNumParityUnits());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
index 7784573..4806d9e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
@@ -54,8 +54,9 @@ public class RSErasureEncoder extends AbstractErasureEncoder {
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rawEncoder == null) {
+ // TODO: we should create the raw coder according to codec.
rawEncoder = CodecUtil.createRSRawEncoder(getConf(),
- getNumDataUnits(), getNumParityUnits());
+ getNumDataUnits(), getNumParityUnits());
}
return rawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 96b108f..ac8bc5a 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -506,6 +506,14 @@
</description>
</property>
+<property>
+ <name>io.erasurecode.codec.rs-default.rawcoder</name>
+ <value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
+ <description>
+ Raw coder implementation for the rs-default codec.
+ </description>
+</property>
+
<!-- file system properties -->
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
new file mode 100644
index 0000000..5075966
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoderLegacy;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test the codec to raw coder mapping.
+ */
+public class TestCodecRawCoderMapping {
+
+ private static Configuration conf;
+ private static final int numDataUnit = 6;
+ private static final int numParityUnit = 3;
+
+ @Before
+ public void setup() {
+ conf = new Configuration();
+ }
+
+ @Test
+ public void testRSDefaultRawCoder() {
+ // should return default raw coder of rs-default codec
+ RawErasureEncoder encoder = CodecUtil.createRSRawEncoder(
+ conf, numDataUnit, numParityUnit);
+ Assert.assertTrue(encoder instanceof RSRawEncoder);
+ RawErasureDecoder decoder = CodecUtil.createRSRawDecoder(
+ conf, numDataUnit, numParityUnit);
+ Assert.assertTrue(decoder instanceof RSRawDecoder);
+
+ // should return default raw coder of rs-legacy codec
+ encoder = CodecUtil.createRSRawEncoder(conf, numDataUnit, numParityUnit,
+ ErasureCodeConstants.RS_LEGACY_CODEC_NAME);
+ Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
+ decoder = CodecUtil.createRSRawDecoder(conf, numDataUnit, numParityUnit,
+ ErasureCodeConstants.RS_LEGACY_CODEC_NAME);
+ Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
+ }
+
+ @Test
+ public void testDedicatedRawCoderKey() {
+ String dummyFactName = "DummyNoneExistingFactory";
+ // set the dummy factory to rs-legacy and create a raw coder
+ // with rs-default, which is OK as the raw coder key is not used
+ conf.set(CommonConfigurationKeys.
+ IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName);
+ RawErasureEncoder encoder = CodecUtil.createRSRawEncoder(conf, numDataUnit,
+ numParityUnit, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME);
+ Assert.assertTrue(encoder instanceof RSRawEncoder);
+ // now create the raw coder with rs-legacy, which should throw exception
+ try {
+ CodecUtil.createRSRawEncoder(conf, numDataUnit, numParityUnit,
+ ErasureCodeConstants.RS_LEGACY_CODEC_NAME);
+ Assert.fail();
+ } catch (Exception e) {
+ GenericTestUtils.assertExceptionContains("Failed to create raw coder", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
index 5721336..633e064 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
@@ -112,7 +112,7 @@ public abstract class TestCoderBase {
protected void prepare(Configuration conf, int numDataUnits,
int numParityUnits, int[] erasedDataIndexes,
int[] erasedParityIndexes, boolean usingFixedData) {
- this.conf = conf;
+ this.conf = conf != null ? conf : new Configuration();
this.numDataUnits = numDataUnits;
this.numParityUnits = numParityUnits;
this.erasedDataIndexes = erasedDataIndexes != null ?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
index ad346e0..09d1ec2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
@@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
index ee2348e..b0fe8f8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
@@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 38236ad..1944782 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -185,7 +185,7 @@ public class DFSStripedInputStream extends DFSInputStream {
readingService =
new ExecutorCompletionService<>(dfsClient.getStripedReadsThreadPool());
decoder = CodecUtil.createRSRawDecoder(dfsClient.getConfiguration(),
- dataBlkNum, parityBlkNum);
+ dataBlkNum, parityBlkNum, ecPolicy.getCodecName());
if (DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug("Creating an striped input stream for file " + src);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 3dfdbee..b049286 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -287,7 +287,7 @@ public class DFSStripedOutputStream extends DFSOutputStream {
ExecutorCompletionService<>(flushAllExecutor);
encoder = CodecUtil.createRSRawEncoder(dfsClient.getConfiguration(),
- numDataBlocks, numParityBlocks);
+ numDataBlocks, numParityBlocks, ecPolicy.getCodecName());
coordinator = new Coordinator(numAllBlocks);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
index 2c748d2..c320fdf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
@@ -72,6 +72,10 @@ public final class ErasureCodingPolicy {
return schema.getNumParityUnits();
}
+ public String getCodecName() {
+ return schema.getCodecName();
+ }
+
public byte getId() {
return id;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 3c5c441..8df2d54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -22,7 +22,6 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.util.StringUtils;
@InterfaceAudience.Private
@@ -145,11 +144,9 @@ public final class HdfsConstants {
ALL, LIVE, DEAD, DECOMMISSIONING
}
- public static final ECSchema RS_6_3_SCHEMA = new ECSchema("rs", 6, 3);
public static final byte RS_6_3_POLICY_ID = 0;
-
- public static final ECSchema RS_3_2_SCHEMA = new ECSchema("rs", 3, 2);
public static final byte RS_3_2_POLICY_ID = 1;
+ public static final byte RS_6_3_LEGACY_POLICY_ID = 2;
/* Hidden constructor */
protected HdfsConstants() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
index a0a5f83..1b59b22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
@@ -216,7 +216,7 @@ class StripedReconstructor implements Runnable {
private void initDecoderIfNecessary() {
if (decoder == null) {
decoder = CodecUtil.createRSRawDecoder(conf, ecPolicy.getNumDataUnits(),
- ecPolicy.getNumParityUnits());
+ ecPolicy.getNumParityUnits(), ecPolicy.getCodecName());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index eee80d3..eaf63f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import java.util.Map;
import java.util.TreeMap;
@@ -39,15 +40,18 @@ public final class ErasureCodingPolicyManager {
*/
private static final int DEFAULT_CELLSIZE = 64 * 1024;
private static final ErasureCodingPolicy SYS_POLICY1 =
- new ErasureCodingPolicy(HdfsConstants.RS_6_3_SCHEMA, DEFAULT_CELLSIZE,
- HdfsConstants.RS_6_3_POLICY_ID);
+ new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_SCHEMA,
+ DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_POLICY_ID);
private static final ErasureCodingPolicy SYS_POLICY2 =
- new ErasureCodingPolicy(HdfsConstants.RS_3_2_SCHEMA, DEFAULT_CELLSIZE,
- HdfsConstants.RS_3_2_POLICY_ID);
+ new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA,
+ DEFAULT_CELLSIZE, HdfsConstants.RS_3_2_POLICY_ID);
+ private static final ErasureCodingPolicy SYS_POLICY3 =
+ new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_LEGACY_SCHEMA,
+ DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_LEGACY_POLICY_ID);
//We may add more later.
private static final ErasureCodingPolicy[] SYS_POLICIES =
- new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2};
+ new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2, SYS_POLICY3};
/**
* All active policies maintained in NN memory for fast querying,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 5b3aa34..9066a15 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -96,8 +96,10 @@ Deployment
### Configuration keys
The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys:
- `io.erasurecode.codec.rs.rawcoder` and `io.erasurecode.codec.xor.rawcoder`.
- The default implementations for both of these codecs are pure Java.
+ `io.erasurecode.codec.rs-default.rawcoder` for the default RS codec,
+ `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec,
+ `io.erasurecode.codec.xor.rawcoder` for the XOR codec.
+ The default implementations for all of these codecs are pure Java.
Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
index 6d0dfa8..ccc94a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
@@ -491,7 +491,8 @@ public class StripedFileTestUtil {
}
}
final RawErasureEncoder encoder =
- CodecUtil.createRSRawEncoder(conf, dataBytes.length, parityBytes.length);
+ CodecUtil.createRSRawEncoder(conf, dataBytes.length, parityBytes.length,
+ TEST_EC_POLICY.getCodecName());
encoder.encode(dataBytes, expectedParityBytes);
for (int i = 0; i < parityBytes.length; i++) {
if (checkSet.contains(i + dataBytes.length)){
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 4dcb654..e4f7ac0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -218,7 +218,7 @@ public class TestDFSStripedInputStream {
}
RawErasureDecoder rawDecoder = CodecUtil.createRSRawDecoder(conf,
- DATA_BLK_NUM, PARITY_BLK_NUM);
+ DATA_BLK_NUM, PARITY_BLK_NUM, ecPolicy.getCodecName());
// Update the expected content for decoded data
int[] missingBlkIdx = new int[PARITY_BLK_NUM];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5d4c7dc/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index fd7cf9d..1c782c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -109,7 +109,7 @@
<description>setPolicy : set erasure coding policy on a directory to encode files</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -179,7 +179,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
@@ -188,7 +188,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-6-3-64k</expected-output>
+ <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -197,7 +197,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy /ecdir/ecfile</ec-admin-command>
</test-commands>
@@ -208,7 +208,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-6-3-64k</expected-output>
+ <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -223,7 +223,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-6-3</expected-output>
+ <expected-output>RS-DEFAULT-6-3</expected-output>
</comparator>
</comparators>
</test>