You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/04/02 20:10:16 UTC
[38/50] [abbrv] hadoop git commit: HADOOP-11647. Reed-Solomon
ErasureCoder. Contributed by Kai Zheng
HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe450f92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe450f92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe450f92
Branch: refs/heads/HDFS-7285
Commit: fe450f927479b90f2c7737013bb21f4fd01701d0
Parents: 93d42dc
Author: Kai Zheng <ka...@intel.com>
Authored: Fri Mar 20 19:15:52 2015 +0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Thu Apr 2 11:06:09 2015 -0700
----------------------------------------------------------------------
.../hadoop-common/CHANGES-HDFS-EC-7285.txt | 3 +
.../hadoop/fs/CommonConfigurationKeys.java | 15 ++++
.../erasurecode/coder/AbstractErasureCoder.java | 65 ++++++++++++++
.../coder/AbstractErasureDecoder.java | 6 +-
.../coder/AbstractErasureEncoder.java | 6 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 83 ++++++++++++++++++
.../io/erasurecode/coder/RSErasureEncoder.java | 47 ++++++++++
.../io/erasurecode/coder/XorErasureDecoder.java | 2 +-
.../io/erasurecode/coder/XorErasureEncoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 92 ++++++++++++++++++++
10 files changed, 315 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe450f92/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
index f566f0e..b69e69a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -26,3 +26,6 @@
HADOOP-11707. Add factory to create raw erasure coder. Contributed by Kai Zheng
( Kai Zheng )
+
+ HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng
+ ( Kai Zheng )
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe450f92/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 7575496..70fea01 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -135,6 +135,21 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
false;
/**
+ * Erasure Coding configuration family
+ */
+
+ /** Supported erasure codec classes */
+ public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs";
+
+ /** Use XOR raw coder when possible for the RS codec */
+ public static final String IO_ERASURECODE_CODEC_RS_USEXOR_KEY =
+ "io.erasurecode.codec.rs.usexor";
+
+ /** Raw coder factory for the RS codec */
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
+ "io.erasurecode.codec.rs.rawcoder";
+
+ /**
* Service Authorization
*/
public static final String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe450f92/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
index 8d3bc34..0e4de89 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
@@ -17,7 +17,12 @@
*/
package org.apache.hadoop.io.erasurecode.coder;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
/**
* A common class of basic facilities to be shared by encoder and decoder
@@ -31,6 +36,66 @@ public abstract class AbstractErasureCoder
private int numParityUnits;
private int chunkSize;
+ /**
+ * Create raw decoder using the factory specified by rawCoderFactoryKey
+ * @param rawCoderFactoryKey
+ * @return raw decoder
+ */
+ protected RawErasureDecoder createRawDecoder(String rawCoderFactoryKey) {
+ RawErasureCoder rawCoder = createRawCoder(getConf(),
+ rawCoderFactoryKey, false);
+ return (RawErasureDecoder) rawCoder;
+ }
+
+ /**
+ * Create raw encoder using the factory specified by rawCoderFactoryKey
+ * @param rawCoderFactoryKey
+ * @return raw encoder
+ */
+ protected RawErasureEncoder createRawEncoder(String rawCoderFactoryKey) {
+ RawErasureCoder rawCoder = createRawCoder(getConf(),
+ rawCoderFactoryKey, true);
+ return (RawErasureEncoder) rawCoder;
+ }
+
+ /**
+ * Create raw coder using specified conf and raw coder factory key.
+ * @param conf
+ * @param rawCoderFactoryKey
+ * @param isEncoder
+ * @return raw coder
+ */
+ protected static RawErasureCoder createRawCoder(Configuration conf,
+ String rawCoderFactoryKey, boolean isEncoder) {
+
+ if (conf == null) {
+ return null;
+ }
+
+ Class<? extends RawErasureCoderFactory> factClass = null;
+ factClass = conf.getClass(rawCoderFactoryKey,
+ factClass, RawErasureCoderFactory.class);
+
+ if (factClass == null) {
+ return null;
+ }
+
+ RawErasureCoderFactory fact;
+ try {
+ fact = factClass.newInstance();
+ } catch (InstantiationException e) {
+ throw new RuntimeException("Failed to create raw coder", e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException("Failed to create raw coder", e);
+ }
+
+ if (fact != null) {
+ return isEncoder ? fact.createEncoder() : fact.createDecoder();
+ }
+
+ return null;
+ }
+
@Override
public void initialize(int numDataUnits, int numParityUnits,
int chunkSize) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe450f92/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
index df71528..54a6d1e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
@@ -30,7 +30,8 @@ public abstract class AbstractErasureDecoder extends AbstractErasureCoder
@Override
public ErasureCodingStep decode(ECBlockGroup blockGroup) {
- return performDecoding(blockGroup);
+ // We may have more than this when considering complicate cases. HADOOP-11550
+ return prepareDecodingStep(blockGroup);
}
/**
@@ -38,7 +39,8 @@ public abstract class AbstractErasureDecoder extends AbstractErasureCoder
* @param blockGroup
* @return decoding step for caller to do the real work
*/
- protected abstract ErasureCodingStep performDecoding(ECBlockGroup blockGroup);
+ protected abstract ErasureCodingStep prepareDecodingStep(
+ ECBlockGroup blockGroup);
/**
* We have all the data blocks and parity blocks as input blocks for
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe450f92/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
index 80b9875..09b31e5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
@@ -30,7 +30,8 @@ public abstract class AbstractErasureEncoder extends AbstractErasureCoder
@Override
public ErasureCodingStep encode(ECBlockGroup blockGroup) {
- return performEncoding(blockGroup);
+ // We may have more than this when considering complicate cases. HADOOP-11550
+ return prepareEncodingStep(blockGroup);
}
/**
@@ -38,7 +39,8 @@ public abstract class AbstractErasureEncoder extends AbstractErasureCoder
* @param blockGroup
* @return encoding step for caller to do the real work
*/
- protected abstract ErasureCodingStep performEncoding(ECBlockGroup blockGroup);
+ protected abstract ErasureCodingStep prepareEncodingStep(
+ ECBlockGroup blockGroup);
protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
return blockGroup.getDataBlocks();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe450f92/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
new file mode 100644
index 0000000..ba32f04
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
@@ -0,0 +1,83 @@
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.rawcoder.JRSRawDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.XorRawDecoder;
+
+/**
+ * Reed-Solomon erasure decoder that decodes a block group.
+ *
+ * It implements {@link ErasureDecoder}.
+ */
+public class RSErasureDecoder extends AbstractErasureDecoder {
+ private RawErasureDecoder rsRawDecoder;
+ private RawErasureDecoder xorRawDecoder;
+ private boolean useXorWhenPossible = true;
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+
+ if (conf != null) {
+ this.useXorWhenPossible = conf.getBoolean(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_USEXOR_KEY, true);
+ }
+ }
+
+ @Override
+ protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) {
+
+ RawErasureDecoder rawDecoder;
+
+ ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+ ECBlock[] outputBlocks = getOutputBlocks(blockGroup);
+
+ /**
+ * Optimization: according to some benchmark, when only one block is erased
+ * and to be recovering, the most simple XOR scheme can be much efficient.
+ * We will have benchmark tests to verify this opt is effect or not.
+ */
+ if (outputBlocks.length == 1 && useXorWhenPossible) {
+ rawDecoder = checkCreateXorRawDecoder();
+ } else {
+ rawDecoder = checkCreateRSRawDecoder();
+ }
+
+ return new ErasureDecodingStep(inputBlocks,
+ getErasedIndexes(inputBlocks), outputBlocks, rawDecoder);
+ }
+
+ private RawErasureDecoder checkCreateRSRawDecoder() {
+ if (rsRawDecoder == null) {
+ rsRawDecoder = createRawDecoder(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY);
+ if (rsRawDecoder == null) {
+ rsRawDecoder = new JRSRawDecoder();
+ }
+ rsRawDecoder.initialize(getNumDataUnits(),
+ getNumParityUnits(), getChunkSize());
+ }
+ return rsRawDecoder;
+ }
+
+ private RawErasureDecoder checkCreateXorRawDecoder() {
+ if (xorRawDecoder == null) {
+ xorRawDecoder = new XorRawDecoder();
+ xorRawDecoder.initialize(getNumDataUnits(), 1, getChunkSize());
+ }
+ return xorRawDecoder;
+ }
+
+ @Override
+ public void release() {
+ if (xorRawDecoder != null) {
+ xorRawDecoder.release();
+ } else if (rsRawDecoder != null) {
+ rsRawDecoder.release();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe450f92/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
new file mode 100644
index 0000000..430749d
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
@@ -0,0 +1,47 @@
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.rawcoder.JRSRawEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+
+/**
+ * Reed-Solomon erasure encoder that encodes a block group.
+ *
+ * It implements {@link ErasureEncoder}.
+ */
+public class RSErasureEncoder extends AbstractErasureEncoder {
+ private RawErasureEncoder rawEncoder;
+
+ @Override
+ protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) {
+
+ RawErasureEncoder rawEncoder = checkCreateRSRawEncoder();
+
+ ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+
+ return new ErasureEncodingStep(inputBlocks,
+ getOutputBlocks(blockGroup), rawEncoder);
+ }
+
+ private RawErasureEncoder checkCreateRSRawEncoder() {
+ if (rawEncoder == null) {
+ rawEncoder = createRawEncoder(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY);
+ if (rawEncoder == null) {
+ rawEncoder = new JRSRawEncoder();
+ }
+ rawEncoder.initialize(getNumDataUnits(),
+ getNumParityUnits(), getChunkSize());
+ }
+ return rawEncoder;
+ }
+
+ @Override
+ public void release() {
+ if (rawEncoder != null) {
+ rawEncoder.release();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe450f92/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XorErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XorErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XorErasureDecoder.java
index 9a6f6e2..33f5386 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XorErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XorErasureDecoder.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.XorRawDecoder;
public class XorErasureDecoder extends AbstractErasureDecoder {
@Override
- protected ErasureCodingStep performDecoding(final ECBlockGroup blockGroup) {
+ protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) {
// May be configured
RawErasureDecoder rawDecoder = new XorRawDecoder();
rawDecoder.initialize(getNumDataUnits(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe450f92/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XorErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XorErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XorErasureEncoder.java
index db6e3db..f8d67c3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XorErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XorErasureEncoder.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.XorRawEncoder;
public class XorErasureEncoder extends AbstractErasureEncoder {
@Override
- protected ErasureCodingStep performEncoding(final ECBlockGroup blockGroup) {
+ protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) {
// May be configured
RawErasureEncoder rawEncoder = new XorRawEncoder();
rawEncoder.initialize(getNumDataUnits(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe450f92/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
new file mode 100644
index 0000000..8a7561c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.rawcoder.JRSRawErasureCoderFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test Reed-Solomon encoding and decoding.
+ */
+public class TestRSErasureCoder extends TestErasureCoderBase {
+
+ @Before
+ public void setup() {
+ this.encoderClass = RSErasureEncoder.class;
+ this.decoderClass = RSErasureDecoder.class;
+
+ this.numDataUnits = 10;
+ this.numParityUnits = 1;
+
+ this.numChunksInBlock = 10;
+ }
+
+ @Test
+ public void testCodingNoDirectBuffer_10x4() {
+ prepare(null, 10, 4, null);
+ testCoding(false);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4() {
+ prepare(null, 10, 4, null);
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBufferWithConf_10x4() {
+ /**
+ * This tests if the two configuration items work or not.
+ */
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ JRSRawErasureCoderFactory.class.getCanonicalName());
+ conf.setBoolean(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_USEXOR_KEY, false);
+ prepare(conf, 10, 4, null);
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasure_of_2_4() {
+ prepare(null, 10, 4, new int[] {2, 4});
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasing_all() {
+ prepare(null, 10, 4, new int[] {0, 1, 2, 3});
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingNoDirectBuffer_3x3() {
+ prepare(null, 3, 3, null);
+ testCoding(false);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_3x3() {
+ prepare(null, 3, 3, null);
+ testCoding(true);
+ }
+
+}