You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2016/10/18 23:45:27 UTC
[41/50] [abbrv] hadoop git commit: HADOOP-13061. Refactor erasure
coders. Contributed by Kai Sasaki
HADOOP-13061. Refactor erasure coders. Contributed by Kai Sasaki
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c023c748
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c023c748
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c023c748
Branch: refs/heads/HADOOP-13070
Commit: c023c748869063fb67d14ea996569c42578d1cea
Parents: bedfec0
Author: Kai Zheng <ka...@intel.com>
Authored: Tue Oct 18 12:02:53 2016 +0600
Committer: Kai Zheng <ka...@intel.com>
Committed: Tue Oct 18 12:02:53 2016 +0600
----------------------------------------------------------------------
.../hadoop/fs/CommonConfigurationKeys.java | 26 ---
.../apache/hadoop/io/erasurecode/CodecUtil.java | 168 ++++++++++++++--
.../io/erasurecode/ErasureCodeConstants.java | 3 +-
.../io/erasurecode/ErasureCodecOptions.java | 37 ++++
.../erasurecode/codec/AbstractErasureCodec.java | 53 -----
.../io/erasurecode/codec/DummyErasureCodec.java | 45 +++++
.../io/erasurecode/codec/ErasureCodec.java | 76 +++++--
.../io/erasurecode/codec/HHXORErasureCodec.java | 20 +-
.../io/erasurecode/codec/RSErasureCodec.java | 20 +-
.../io/erasurecode/codec/XORErasureCodec.java | 22 ++-
.../io/erasurecode/codec/package-info.java | 28 +++
.../erasurecode/coder/AbstractErasureCoder.java | 64 ------
.../coder/AbstractErasureCodingStep.java | 61 ------
.../coder/AbstractErasureDecoder.java | 170 ----------------
.../coder/AbstractErasureEncoder.java | 62 ------
.../coder/AbstractHHErasureCodingStep.java | 49 -----
.../erasurecode/coder/DummyErasureDecoder.java | 46 +++++
.../erasurecode/coder/DummyErasureEncoder.java | 45 +++++
.../io/erasurecode/coder/ErasureCoder.java | 25 ++-
.../io/erasurecode/coder/ErasureCodingStep.java | 8 +-
.../io/erasurecode/coder/ErasureDecoder.java | 198 +++++++++++++++++++
.../erasurecode/coder/ErasureDecodingStep.java | 21 +-
.../io/erasurecode/coder/ErasureEncoder.java | 91 +++++++++
.../erasurecode/coder/ErasureEncodingStep.java | 22 ++-
.../erasurecode/coder/HHErasureCodingStep.java | 68 +++++++
.../erasurecode/coder/HHXORErasureDecoder.java | 24 +--
.../coder/HHXORErasureDecodingStep.java | 2 +-
.../erasurecode/coder/HHXORErasureEncoder.java | 19 +-
.../coder/HHXORErasureEncodingStep.java | 2 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 16 +-
.../io/erasurecode/coder/RSErasureEncoder.java | 20 +-
.../io/erasurecode/coder/XORErasureDecoder.java | 15 +-
.../io/erasurecode/coder/XORErasureEncoder.java | 16 +-
.../io/erasurecode/coder/package-info.java | 28 +++
.../io/erasurecode/rawcoder/CoderUtil.java | 2 +-
.../conf/TestCommonConfigurationFields.java | 5 +-
.../erasurecode/TestCodecRawCoderMapping.java | 3 +-
.../codec/TestHHXORErasureCodec.java | 6 +-
.../erasurecode/coder/TestErasureCoderBase.java | 13 +-
.../coder/TestHHXORErasureCoder.java | 4 +-
.../erasurecode/coder/TestRSErasureCoder.java | 4 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 3 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 6 +-
.../TestDFSStripedOutputStreamWithFailure.java | 4 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 4 +-
45 files changed, 964 insertions(+), 660 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 2b530f0..fe522b3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -21,9 +21,6 @@ package org.apache.hadoop.fs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.http.lib.StaticUserWebFilter;
-import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
-import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
-import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
/**
* This class contains constants for configuration keys used
@@ -160,30 +157,7 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
public static final boolean IO_COMPRESSION_CODEC_LZ4_USELZ4HC_DEFAULT =
false;
- /**
- * Erasure Coding configuration family
- */
- /** Supported erasure codec classes */
- public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs";
-
- /** Raw coder factory for the RS default codec. */
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY =
- "io.erasurecode.codec.rs-default.rawcoder";
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT =
- RSRawErasureCoderFactory.class.getCanonicalName();
-
- /** Raw coder factory for the RS legacy codec. */
- public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY =
- "io.erasurecode.codec.rs-legacy.rawcoder";
- public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT =
- RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
-
- /** Raw coder factory for the XOR codec. */
- public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY =
- "io.erasurecode.codec.xor.rawcoder";
- public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT =
- XORRawErasureCoderFactory.class.getCanonicalName();
/**
* Service Authorization
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 9cd9561..977bacb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -20,20 +20,108 @@ package org.apache.hadoop.io.erasurecode;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.codec.ErasureCodec;
+import org.apache.hadoop.io.erasurecode.codec.HHXORErasureCodec;
+import org.apache.hadoop.io.erasurecode.codec.RSErasureCodec;
+import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
/**
- * A codec & coder utility to help create raw coders conveniently.
+ * A codec & coder utility to help create coders conveniently.
+ *
+ * {@link CodecUtil} includes erasure coder configurations key and default
+ * values such as coder class name and erasure codec option values included
+ * by {@link ErasureCodecOptions}. {@link ErasureEncoder} and
+ * {@link ErasureDecoder} are created by createEncoder and createDecoder
+ * respectively.{@link RawErasureEncoder} and {@link RawErasureDecoder} are
+ * are created by createRawEncoder and createRawDecoder.
*/
@InterfaceAudience.Private
public final class CodecUtil {
+ /** Erasure coder XOR codec. */
+ public static final String IO_ERASURECODE_CODEC_XOR_KEY =
+ "io.erasurecode.codec.xor";
+ public static final String IO_ERASURECODE_CODEC_XOR =
+ XORErasureCodec.class.getCanonicalName();
+ /** Erasure coder Reed-Solomon codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_KEY =
+ "io.erasurecode.codec.rs";
+ public static final String IO_ERASURECODE_CODEC_RS_DEFAULT =
+ RSErasureCodec.class.getCanonicalName();
+ /** Erasure coder hitch hiker XOR codec. */
+ public static final String IO_ERASURECODE_CODEC_HHXOR_KEY =
+ "io.erasurecode.codec.hhxor";
+ public static final String IO_ERASURECODE_CODEC_HHXOR =
+ HHXORErasureCodec.class.getCanonicalName();
+
+ /** Supported erasure codec classes. */
+
+ /** Raw coder factory for the RS default codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY =
+ "io.erasurecode.codec.rs-default.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT =
+ RSRawErasureCoderFactory.class.getCanonicalName();
+
+ /** Raw coder factory for the RS legacy codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY =
+ "io.erasurecode.codec.rs-legacy.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT =
+ RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
+
+ /** Raw coder factory for the XOR codec. */
+ public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY =
+ "io.erasurecode.codec.xor.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT =
+ XORRawErasureCoderFactory.class.getCanonicalName();
+
private CodecUtil() { }
/**
+ * Create encoder corresponding to given codec.
+ * @param options Erasure codec options
+ * @return erasure encoder
+ */
+ public static ErasureEncoder createEncoder(Configuration conf,
+ ErasureCodecOptions options) {
+ Preconditions.checkNotNull(conf);
+ Preconditions.checkNotNull(options);
+
+ String codecKey = getCodecClassName(conf,
+ options.getSchema().getCodecName());
+
+ ErasureCodec codec = createCodec(conf, codecKey, options);
+ return codec.createEncoder();
+ }
+
+ /**
+ * Create decoder corresponding to given codec.
+ * @param options Erasure codec options
+ * @return erasure decoder
+ */
+ public static ErasureDecoder createDecoder(Configuration conf,
+ ErasureCodecOptions options) {
+ Preconditions.checkNotNull(conf);
+ Preconditions.checkNotNull(options);
+
+ String codecKey = getCodecClassName(conf,
+ options.getSchema().getCodecName());
+
+ ErasureCodec codec = createCodec(conf, codecKey, options);
+ return codec.createDecoder();
+ }
+
+ /**
* Create RS raw encoder according to configuration.
* @param conf configuration
* @param coderOptions coder options that's used to create the coder
@@ -45,7 +133,7 @@ public final class CodecUtil {
Preconditions.checkNotNull(conf);
Preconditions.checkNotNull(codec);
- String rawCoderFactoryKey = getFactNameFromCodec(conf, codec);
+ String rawCoderFactoryKey = getRawCoderFactNameFromCodec(conf, codec);
RawErasureCoderFactory fact = createRawCoderFactory(conf,
rawCoderFactoryKey);
@@ -65,7 +153,7 @@ public final class CodecUtil {
Preconditions.checkNotNull(conf);
Preconditions.checkNotNull(codec);
- String rawCoderFactoryKey = getFactNameFromCodec(conf, codec);
+ String rawCoderFactoryKey = getRawCoderFactNameFromCodec(conf, codec);
RawErasureCoderFactory fact = createRawCoderFactory(conf,
rawCoderFactoryKey);
@@ -92,22 +180,21 @@ public final class CodecUtil {
return fact;
}
- private static String getFactNameFromCodec(Configuration conf, String codec) {
+ private static String getRawCoderFactNameFromCodec(Configuration conf,
+ String codec) {
switch (codec) {
case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
return conf.get(
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
- CommonConfigurationKeys.
- IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT);
+ IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT);
case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
return conf.get(
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
- CommonConfigurationKeys.
- IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT);
+ IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
+ IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT);
case ErasureCodeConstants.XOR_CODEC_NAME:
return conf.get(
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT);
+ IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
+ IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT);
default:
// For custom codec, we throw exception if the factory is not configured
String rawCoderKey = "io.erasurecode.codec." + codec + ".rawcoder";
@@ -119,4 +206,59 @@ public final class CodecUtil {
return factName;
}
}
+
+ private static ErasureCodec createCodec(Configuration conf,
+ String codecClassName, ErasureCodecOptions options) {
+ ErasureCodec codec = null;
+ try {
+ Class<? extends ErasureCodec> codecClass =
+ conf.getClassByName(codecClassName)
+ .asSubclass(ErasureCodec.class);
+ Constructor<? extends ErasureCodec> constructor
+ = codecClass.getConstructor(Configuration.class,
+ ErasureCodecOptions.class);
+ codec = constructor.newInstance(conf, options);
+ } catch (ClassNotFoundException | InstantiationException |
+ IllegalAccessException | NoSuchMethodException |
+ InvocationTargetException e) {
+ throw new RuntimeException("Failed to create erasure codec", e);
+ }
+
+ if (codec == null) {
+ throw new RuntimeException("Failed to create erasure codec");
+ }
+
+ return codec;
+ }
+
+ private static String getCodecClassName(Configuration conf, String codec) {
+ switch (codec) {
+ case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
+ return conf.get(
+ CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT);
+ case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
+ //TODO:rs-legacy should be handled differently.
+ return conf.get(
+ CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT);
+ case ErasureCodeConstants.XOR_CODEC_NAME:
+ return conf.get(
+ CodecUtil.IO_ERASURECODE_CODEC_XOR_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_XOR);
+ case ErasureCodeConstants.HHXOR_CODEC_NAME:
+ return conf.get(
+ CodecUtil.IO_ERASURECODE_CODEC_HHXOR_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_HHXOR);
+ default:
+ // For custom codec, we throw exception if the factory is not configured
+ String codecKey = "io.erasurecode.codec." + codec + ".coder";
+ String codecClass = conf.get(codecKey);
+ if (codecClass == null) {
+ throw new IllegalArgumentException("Codec not configured " +
+ "for custom codec " + codec);
+ }
+ return codecClass;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index 1fb9488..8d6ff85 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -22,12 +22,13 @@ package org.apache.hadoop.io.erasurecode;
*/
public final class ErasureCodeConstants {
- private ErasureCodeConstants(){
+ private ErasureCodeConstants() {
}
public static final String RS_DEFAULT_CODEC_NAME = "rs-default";
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
public static final String XOR_CODEC_NAME = "xor";
+ public static final String HHXOR_CODEC_NAME = "hhxor";
public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
RS_DEFAULT_CODEC_NAME, 6, 3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodecOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodecOptions.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodecOptions.java
new file mode 100644
index 0000000..7ae0fc2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodecOptions.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.erasurecode;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Erasure codec options.
+ */
+@InterfaceAudience.Private
+public class ErasureCodecOptions {
+ private ECSchema schema;
+
+ public ErasureCodecOptions(ECSchema schema) {
+ this.schema = schema;
+ }
+
+ public ECSchema getSchema() {
+ return schema;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
deleted file mode 100644
index e2fb2cb..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.codec;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper;
-
-/**
- * Abstract Erasure Codec that implements {@link ErasureCodec}.
- */
-@InterfaceAudience.Private
-public abstract class AbstractErasureCodec extends Configured
- implements ErasureCodec {
-
- private final ECSchema schema;
-
- public AbstractErasureCodec(ECSchema schema) {
- this.schema = schema;
- }
-
- public String getName() {
- return schema.getCodecName();
- }
-
- public ECSchema getSchema() {
- return schema;
- }
-
- @Override
- public BlockGrouper createBlockGrouper() {
- BlockGrouper blockGrouper = new BlockGrouper();
- blockGrouper.setSchema(getSchema());
-
- return blockGrouper;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/DummyErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/DummyErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/DummyErasureCodec.java
new file mode 100644
index 0000000..3646f72
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/DummyErasureCodec.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.codec;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
+import org.apache.hadoop.io.erasurecode.coder.DummyErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.DummyErasureEncoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
+
+/**
+ * Dummy erasure coder does not real coding computing. This is used for only
+ * test or performance comparison with other erasure coders.
+ */
+public class DummyErasureCodec extends ErasureCodec {
+ public DummyErasureCodec(Configuration conf, ErasureCodecOptions options) {
+ super(conf, options);
+ }
+
+ @Override
+ public ErasureEncoder createEncoder() {
+ return new DummyErasureEncoder(getCoderOptions());
+ }
+
+ @Override
+ public ErasureDecoder createDecoder() {
+ return new DummyErasureDecoder(getCoderOptions());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
index a2ea865..c75eaea 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
@@ -18,34 +18,76 @@
package org.apache.hadoop.io.erasurecode.codec;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper;
/**
- * Erasure Codec API that's to cover the essential specific aspects of a code.
- * Currently it cares only block grouper and erasure coder. In future we may
- * add more aspects here to make the behaviors customizable.
+ * Abstract Erasure Codec is defines the interface of each actual erasure
+ * codec classes.
*/
@InterfaceAudience.Private
-public interface ErasureCodec extends Configurable {
+public abstract class ErasureCodec {
- /**
- * Create block grouper
- * @return block grouper
- */
- public BlockGrouper createBlockGrouper();
+ private ECSchema schema;
+ private ErasureCodecOptions codecOptions;
+ private ErasureCoderOptions coderOptions;
+
+ public ErasureCodec(Configuration conf,
+ ErasureCodecOptions options) {
+ this.schema = options.getSchema();
+ this.codecOptions = options;
+ boolean allowChangeInputs = false;
+ this.coderOptions = new ErasureCoderOptions(schema.getNumDataUnits(),
+ schema.getNumParityUnits(), allowChangeInputs, false);
+ }
+
+ public String getName() {
+ return schema.getCodecName();
+ }
+
+ public ECSchema getSchema() {
+ return schema;
+ }
/**
- * Create Erasure Encoder
- * @return erasure encoder
+ * Get a {@link ErasureCodecOptions}.
+ * @return erasure codec options
*/
- public ErasureCoder createEncoder();
+ public ErasureCodecOptions getCodecOptions() {
+ return codecOptions;
+ }
+
+ protected void setCodecOptions(ErasureCodecOptions options) {
+ this.codecOptions = options;
+ this.schema = options.getSchema();
+ }
/**
- * Create Erasure Decoder
- * @return erasure decoder
+ * Get a {@link ErasureCoderOptions}.
+ * @return erasure coder options
*/
- public ErasureCoder createDecoder();
+ public ErasureCoderOptions getCoderOptions() {
+ return coderOptions;
+ }
+
+ protected void setCoderOptions(ErasureCoderOptions options) {
+ this.coderOptions = options;
+ }
+
+ public abstract ErasureEncoder createEncoder();
+
+ public abstract ErasureDecoder createDecoder();
+
+ public BlockGrouper createBlockGrouper() {
+ BlockGrouper blockGrouper = new BlockGrouper();
+ blockGrouper.setSchema(getSchema());
+ return blockGrouper;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java
index 3c8061d..42380f3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.io.erasurecode.codec;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
import org.apache.hadoop.io.erasurecode.coder.HHXORErasureDecoder;
import org.apache.hadoop.io.erasurecode.coder.HHXORErasureEncoder;
@@ -27,19 +29,19 @@ import org.apache.hadoop.io.erasurecode.coder.HHXORErasureEncoder;
* A Hitchhiker-XOR erasure codec.
*/
@InterfaceAudience.Private
-public class HHXORErasureCodec extends AbstractErasureCodec {
+public class HHXORErasureCodec extends ErasureCodec {
- public HHXORErasureCodec(ECSchema schema) {
- super(schema);
+ public HHXORErasureCodec(Configuration conf, ErasureCodecOptions options) {
+ super(conf, options);
}
@Override
- public ErasureCoder createEncoder() {
- return new HHXORErasureEncoder(getSchema());
+ public ErasureEncoder createEncoder() {
+ return new HHXORErasureEncoder(getCoderOptions());
}
@Override
- public ErasureCoder createDecoder() {
- return new HHXORErasureDecoder(getSchema());
+ public ErasureDecoder createDecoder() {
+ return new HHXORErasureDecoder(getCoderOptions());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
index 7a5786e..e57161c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.io.erasurecode.codec;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
import org.apache.hadoop.io.erasurecode.coder.RSErasureDecoder;
import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder;
@@ -27,19 +29,19 @@ import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder;
* A Reed-Solomon erasure codec.
*/
@InterfaceAudience.Private
-public class RSErasureCodec extends AbstractErasureCodec {
+public class RSErasureCodec extends ErasureCodec {
- public RSErasureCodec(ECSchema schema) {
- super(schema);
+ public RSErasureCodec(Configuration conf, ErasureCodecOptions options) {
+ super(conf, options);
}
@Override
- public ErasureCoder createEncoder() {
- return new RSErasureEncoder(getSchema());
+ public ErasureEncoder createEncoder() {
+ return new RSErasureEncoder(getCoderOptions());
}
@Override
- public ErasureCoder createDecoder() {
- return new RSErasureDecoder(getSchema());
+ public ErasureDecoder createDecoder() {
+ return new RSErasureDecoder(getCoderOptions());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
index 9c38fc2..f9a0608 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.io.erasurecode.codec;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
import org.apache.hadoop.io.erasurecode.coder.XORErasureDecoder;
import org.apache.hadoop.io.erasurecode.coder.XORErasureEncoder;
@@ -27,20 +29,20 @@ import org.apache.hadoop.io.erasurecode.coder.XORErasureEncoder;
* A XOR erasure codec.
*/
@InterfaceAudience.Private
-public class XORErasureCodec extends AbstractErasureCodec {
+public class XORErasureCodec extends ErasureCodec {
- public XORErasureCodec(ECSchema schema) {
- super(schema);
- assert(schema.getNumParityUnits() == 1);
+ public XORErasureCodec(Configuration conf, ErasureCodecOptions options) {
+ super(conf, options);
+ assert(options.getSchema().getNumParityUnits() == 1);
}
@Override
- public ErasureCoder createEncoder() {
- return new XORErasureEncoder(getSchema());
+ public ErasureEncoder createEncoder() {
+ return new XORErasureEncoder(getCoderOptions());
}
@Override
- public ErasureCoder createDecoder() {
- return new XORErasureDecoder(getSchema());
+ public ErasureDecoder createDecoder() {
+ return new XORErasureDecoder(getCoderOptions());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/package-info.java
new file mode 100644
index 0000000..ddfdd4b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/package-info.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Erasure codec framework.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.io.erasurecode.codec;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
deleted file mode 100644
index 1d1641f..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.coder;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-
-/**
- * A common class of basic facilities to be shared by encoder and decoder
- *
- * It implements the {@link ErasureCoder} interface.
- */
-@InterfaceAudience.Private
-public abstract class AbstractErasureCoder
- extends Configured implements ErasureCoder {
-
- private final int numDataUnits;
- private final int numParityUnits;
-
- public AbstractErasureCoder(int numDataUnits, int numParityUnits) {
- this.numDataUnits = numDataUnits;
- this.numParityUnits = numParityUnits;
- }
-
- public AbstractErasureCoder(ECSchema schema) {
- this(schema.getNumDataUnits(), schema.getNumParityUnits());
- }
-
- @Override
- public int getNumDataUnits() {
- return numDataUnits;
- }
-
- @Override
- public int getNumParityUnits() {
- return numParityUnits;
- }
-
- @Override
- public boolean preferDirectBuffer() {
- return false;
- }
-
- @Override
- public void release() {
- // Nothing to do by default
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java
deleted file mode 100644
index ccff92d..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.coder;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECBlock;
-
-/**
- * Abstract class for common facilities shared by {@link ErasureEncodingStep}
- * and {@link ErasureDecodingStep}.
- *
- * It implements {@link ErasureEncodingStep}.
- */
-@InterfaceAudience.Private
-public abstract class AbstractErasureCodingStep implements ErasureCodingStep {
-
- private ECBlock[] inputBlocks;
- private ECBlock[] outputBlocks;
-
- /**
- * Constructor given input blocks and output blocks.
- * @param inputBlocks
- * @param outputBlocks
- */
- public AbstractErasureCodingStep(ECBlock[] inputBlocks,
- ECBlock[] outputBlocks) {
- this.inputBlocks = inputBlocks;
- this.outputBlocks = outputBlocks;
- }
-
- @Override
- public ECBlock[] getInputBlocks() {
- return inputBlocks;
- }
-
- @Override
- public ECBlock[] getOutputBlocks() {
- return outputBlocks;
- }
-
- @Override
- public void finish() {
- // NOOP by default
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
deleted file mode 100644
index d976dd1..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.coder;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECBlock;
-import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-
-/**
- * An abstract erasure decoder that's to be inherited by new decoders.
- *
- * It implements the {@link ErasureCoder} interface.
- */
-@InterfaceAudience.Private
-public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
-
- public AbstractErasureDecoder(int numDataUnits, int numParityUnits) {
- super(numDataUnits, numParityUnits);
- }
-
- public AbstractErasureDecoder(ECSchema schema) {
- super(schema);
- }
-
- @Override
- public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
- // We may have more than this when considering complicate cases. HADOOP-11550
- return prepareDecodingStep(blockGroup);
- }
-
- /**
- * Perform decoding against a block blockGroup.
- * @param blockGroup
- * @return decoding step for caller to do the real work
- */
- protected abstract ErasureCodingStep prepareDecodingStep(
- ECBlockGroup blockGroup);
-
- /**
- * We have all the data blocks and parity blocks as input blocks for
- * recovering by default. It's codec specific
- * @param blockGroup
- * @return input blocks
- */
- protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
- ECBlock[] inputBlocks = new ECBlock[getNumDataUnits() +
- getNumParityUnits()];
-
- System.arraycopy(blockGroup.getDataBlocks(), 0, inputBlocks,
- 0, getNumDataUnits());
-
- System.arraycopy(blockGroup.getParityBlocks(), 0, inputBlocks,
- getNumDataUnits(), getNumParityUnits());
-
- return inputBlocks;
- }
-
- /**
- * Which blocks were erased ?
- * @param blockGroup
- * @return output blocks to recover
- */
- protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
- ECBlock[] outputBlocks = new ECBlock[getNumErasedBlocks(blockGroup)];
-
- int idx = 0;
-
- for (int i = 0; i < getNumDataUnits(); i++) {
- if (blockGroup.getDataBlocks()[i].isErased()) {
- outputBlocks[idx++] = blockGroup.getDataBlocks()[i];
- }
- }
-
- for (int i = 0; i < getNumParityUnits(); i++) {
- if (blockGroup.getParityBlocks()[i].isErased()) {
- outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
- }
- }
-
- return outputBlocks;
- }
-
- /**
- * Get the number of erased blocks in the block group.
- * @param blockGroup
- * @return number of erased blocks
- */
- protected int getNumErasedBlocks(ECBlockGroup blockGroup) {
- int num = getNumErasedBlocks(blockGroup.getParityBlocks());
- num += getNumErasedBlocks(blockGroup.getDataBlocks());
- return num;
- }
-
- /**
- * Find out how many blocks are erased.
- * @param inputBlocks all the input blocks
- * @return number of erased blocks
- */
- protected static int getNumErasedBlocks(ECBlock[] inputBlocks) {
- int numErased = 0;
- for (int i = 0; i < inputBlocks.length; i++) {
- if (inputBlocks[i].isErased()) {
- numErased ++;
- }
- }
-
- return numErased;
- }
-
- /**
- * Get indexes of erased blocks from inputBlocks
- * @param inputBlocks
- * @return indexes of erased blocks from inputBlocks
- */
- protected int[] getErasedIndexes(ECBlock[] inputBlocks) {
- int numErased = getNumErasedBlocks(inputBlocks);
- if (numErased == 0) {
- return new int[0];
- }
-
- int[] erasedIndexes = new int[numErased];
- int i = 0, j = 0;
- for (; i < inputBlocks.length && j < erasedIndexes.length; i++) {
- if (inputBlocks[i].isErased()) {
- erasedIndexes[j++] = i;
- }
- }
-
- return erasedIndexes;
- }
-
- /**
- * Get erased input blocks from inputBlocks
- * @param inputBlocks
- * @return an array of erased blocks from inputBlocks
- */
- protected ECBlock[] getErasedBlocks(ECBlock[] inputBlocks) {
- int numErased = getNumErasedBlocks(inputBlocks);
- if (numErased == 0) {
- return new ECBlock[0];
- }
-
- ECBlock[] erasedBlocks = new ECBlock[numErased];
- int i = 0, j = 0;
- for (; i < inputBlocks.length && j < erasedBlocks.length; i++) {
- if (inputBlocks[i].isErased()) {
- erasedBlocks[j++] = inputBlocks[i];
- }
- }
-
- return erasedBlocks;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
deleted file mode 100644
index 5ec6b4e..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.coder;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECBlock;
-import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-
-/**
- * An abstract erasure encoder that's to be inherited by new encoders.
- *
- * It implements the {@link ErasureCoder} interface.
- */
-@InterfaceAudience.Private
-public abstract class AbstractErasureEncoder extends AbstractErasureCoder {
-
- public AbstractErasureEncoder(int numDataUnits, int numParityUnits) {
- super(numDataUnits, numParityUnits);
- }
-
- public AbstractErasureEncoder(ECSchema schema) {
- super(schema);
- }
-
- @Override
- public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
- // We may have more than this when considering complicate cases. HADOOP-11550
- return prepareEncodingStep(blockGroup);
- }
-
- /**
- * Perform encoding against a block group.
- * @param blockGroup
- * @return encoding step for caller to do the real work
- */
- protected abstract ErasureCodingStep prepareEncodingStep(
- ECBlockGroup blockGroup);
-
- protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
- return blockGroup.getDataBlocks();
- }
-
- protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
- return blockGroup.getParityBlocks();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java
deleted file mode 100644
index e577c5d..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.coder;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECBlock;
-
-/**
- * Abstract class for Hitchhiker common facilities shared by
- * {@link HHXORErasureEncodingStep}and {@link HHXORErasureDecodingStep}.
- *
- * It implements {@link AbstractErasureCodingStep}.
- */
-@InterfaceAudience.Private
-public abstract class AbstractHHErasureCodingStep
- extends AbstractErasureCodingStep {
-
- private static final int SUB_PACKET_SIZE = 2;
-
- /**
- * Constructor given input blocks and output blocks.
- *
- * @param inputBlocks
- * @param outputBlocks
- */
- public AbstractHHErasureCodingStep(ECBlock[] inputBlocks,
- ECBlock[] outputBlocks) {
- super(inputBlocks, outputBlocks);
- }
-
- protected int getSubPacketSize() {
- return SUB_PACKET_SIZE;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureDecoder.java
new file mode 100644
index 0000000..8a9434e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureDecoder.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.rawcoder.DummyRawDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+
+/**
+ * Dummy erasure decoder does no real computation. Instead, it just returns
+ * zero bytes. This decoder can be used to isolate the performance issue to
+ * HDFS side logic instead of codec, and is intended for test only.
+ */
+public class DummyErasureDecoder extends ErasureDecoder {
+ public DummyErasureDecoder(ErasureCoderOptions options) {
+ super(options);
+ }
+
+ @Override
+ protected ErasureCodingStep prepareDecodingStep(ECBlockGroup blockGroup) {
+ RawErasureDecoder rawDecoder = new DummyRawDecoder(getOptions());
+
+ ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+
+ return new ErasureDecodingStep(inputBlocks,
+ getErasedIndexes(inputBlocks),
+ getOutputBlocks(blockGroup), rawDecoder);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureEncoder.java
new file mode 100644
index 0000000..fc04f5e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureEncoder.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.rawcoder.DummyRawEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+
+/**
+ * Dummy erasure encoder does no real computation. Instead, it just returns
+ * zero bytes. This decoder can be used to isolate the performance issue to
+ * HDFS side logic instead of codec, and is intended for test only.
+ */
+public class DummyErasureEncoder extends ErasureEncoder {
+ public DummyErasureEncoder(ErasureCoderOptions options) {
+ super(options);
+ }
+
+ @Override
+ protected ErasureCodingStep prepareEncodingStep(ECBlockGroup blockGroup) {
+ RawErasureEncoder rawEncoder = new DummyRawEncoder(getOptions());
+
+ ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+
+ return new ErasureEncodingStep(inputBlocks,
+ getOutputBlocks(blockGroup), rawEncoder);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
index 49cc8dd..b5ae1f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
/**
* An erasure coder to perform encoding or decoding given a group. Generally it
@@ -39,18 +40,25 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
public interface ErasureCoder extends Configurable {
/**
- * The number of data input units for the coding. A unit can be a byte,
- * chunk or buffer or even a block.
+ * The number of data input units for the coding. A unit can be a byte, chunk
+ * or buffer or even a block.
* @return count of data input units
*/
- public int getNumDataUnits();
+ int getNumDataUnits();
/**
* The number of parity output units for the coding. A unit can be a byte,
* chunk, buffer or even a block.
* @return count of parity output units
*/
- public int getNumParityUnits();
+ int getNumParityUnits();
+
+ /**
+ * The options of erasure coder. This option is passed to
+ * raw erasure coder as it is.
+ * @return erasure coder options
+ */
+ ErasureCoderOptions getOptions();
/**
* Calculate the encoding or decoding steps given a block blockGroup.
@@ -61,7 +69,7 @@ public interface ErasureCoder extends Configurable {
* @param blockGroup the erasure coding block group containing all necessary
* information for codec calculation
*/
- public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
+ ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
/**
* Tell if direct or off-heap buffer is preferred or not. It's for callers to
@@ -70,10 +78,11 @@ public interface ErasureCoder extends Configurable {
* @return true if direct buffer is preferred for performance consideration,
* otherwise false.
*/
- public boolean preferDirectBuffer();
+ boolean preferDirectBuffer();
/**
- * Release the resources if any. Good chance to invoke RawErasureCoder#release.
+ * Release the resources if any. Good chance to invoke
+ * RawErasureCoder#release.
*/
- public void release();
+ void release();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
index 991eb56..9dd0aed 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
@@ -32,14 +32,14 @@ public interface ErasureCodingStep {
* or parity blocks.
* @return input blocks
*/
- public ECBlock[] getInputBlocks();
+ ECBlock[] getInputBlocks();
/**
* Output blocks of writable buffers involved in this step, may be data
* blocks or parity blocks.
* @return output blocks
*/
- public ECBlock[] getOutputBlocks();
+ ECBlock[] getOutputBlocks();
/**
* Perform encoding or decoding given the input chunks, and generated results
@@ -47,11 +47,11 @@ public interface ErasureCodingStep {
* @param inputChunks
* @param outputChunks
*/
- public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks);
+ void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks);
/**
* Notify erasure coder that all the chunks of input blocks are processed so
* the coder can be able to update internal states, considering next step.
*/
- public void finish();
+ void finish();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
new file mode 100644
index 0000000..faf44d9
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+
+/**
+ * An abstract erasure decoder that's to be inherited by new decoders.
+ *
+ * It implements the {@link ErasureCoder} interface.
+ */
+@InterfaceAudience.Private
+public abstract class ErasureDecoder extends Configured
+ implements ErasureCoder {
+ private final int numDataUnits;
+ private final int numParityUnits;
+ private final ErasureCoderOptions options;
+
+ public ErasureDecoder(ErasureCoderOptions options) {
+ this.options = options;
+ this.numDataUnits = options.getNumDataUnits();
+ this.numParityUnits = options.getNumParityUnits();
+ }
+
+ @Override
+ public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
+ // We may have more than this when considering complicate cases. HADOOP-11550
+ return prepareDecodingStep(blockGroup);
+ }
+
+ @Override
+ public int getNumDataUnits() {
+ return this.numDataUnits;
+ }
+
+ @Override
+ public int getNumParityUnits() {
+ return this.numParityUnits;
+ }
+
+ @Override
+ public ErasureCoderOptions getOptions() {
+ return options;
+ }
+
+ /**
+ * We have all the data blocks and parity blocks as input blocks for
+ * recovering by default. It's codec specific
+ * @param blockGroup
+ * @return input blocks
+ */
+ protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
+ ECBlock[] inputBlocks = new ECBlock[getNumDataUnits() +
+ getNumParityUnits()];
+
+ System.arraycopy(blockGroup.getDataBlocks(), 0, inputBlocks,
+ 0, getNumDataUnits());
+
+ System.arraycopy(blockGroup.getParityBlocks(), 0, inputBlocks,
+ getNumDataUnits(), getNumParityUnits());
+
+ return inputBlocks;
+ }
+
+ /**
+ * Which blocks were erased ?
+ * @param blockGroup
+ * @return output blocks to recover
+ */
+ protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
+ ECBlock[] outputBlocks = new ECBlock[getNumErasedBlocks(blockGroup)];
+
+ int idx = 0;
+
+ for (int i = 0; i < getNumDataUnits(); i++) {
+ if (blockGroup.getDataBlocks()[i].isErased()) {
+ outputBlocks[idx++] = blockGroup.getDataBlocks()[i];
+ }
+ }
+
+ for (int i = 0; i < getNumParityUnits(); i++) {
+ if (blockGroup.getParityBlocks()[i].isErased()) {
+ outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
+ }
+ }
+
+ return outputBlocks;
+ }
+
+ @Override
+ public boolean preferDirectBuffer() {
+ return false;
+ }
+
+ @Override
+ public void release() {
+ // Nothing to do by default
+ }
+
+ /**
+ * Perform decoding against a block blockGroup.
+ * @param blockGroup
+ * @return decoding step for caller to do the real work
+ */
+ protected abstract ErasureCodingStep prepareDecodingStep(
+ ECBlockGroup blockGroup);
+
+ /**
+ * Get the number of erased blocks in the block group.
+ * @param blockGroup
+ * @return number of erased blocks
+ */
+ protected int getNumErasedBlocks(ECBlockGroup blockGroup) {
+ int num = getNumErasedBlocks(blockGroup.getParityBlocks());
+ num += getNumErasedBlocks(blockGroup.getDataBlocks());
+ return num;
+ }
+
+ /**
+ * Find out how many blocks are erased.
+ * @param inputBlocks all the input blocks
+ * @return number of erased blocks
+ */
+ protected static int getNumErasedBlocks(ECBlock[] inputBlocks) {
+ int numErased = 0;
+ for (int i = 0; i < inputBlocks.length; i++) {
+ if (inputBlocks[i].isErased()) {
+ numErased ++;
+ }
+ }
+
+ return numErased;
+ }
+
+ /**
+ * Get indexes of erased blocks from inputBlocks
+ * @param inputBlocks
+ * @return indexes of erased blocks from inputBlocks
+ */
+ protected int[] getErasedIndexes(ECBlock[] inputBlocks) {
+ int numErased = getNumErasedBlocks(inputBlocks);
+ if (numErased == 0) {
+ return new int[0];
+ }
+
+ int[] erasedIndexes = new int[numErased];
+ int i = 0, j = 0;
+ for (; i < inputBlocks.length && j < erasedIndexes.length; i++) {
+ if (inputBlocks[i].isErased()) {
+ erasedIndexes[j++] = i;
+ }
+ }
+
+ return erasedIndexes;
+ }
+
+ /**
+ * Get erased input blocks from inputBlocks
+ * @param inputBlocks
+ * @return an array of erased blocks from inputBlocks
+ */
+ protected ECBlock[] getErasedBlocks(ECBlock[] inputBlocks) {
+ int numErased = getNumErasedBlocks(inputBlocks);
+ if (numErased == 0) {
+ return new ECBlock[0];
+ }
+
+ ECBlock[] erasedBlocks = new ECBlock[numErased];
+ int i = 0, j = 0;
+ for (; i < inputBlocks.length && j < erasedBlocks.length; i++) {
+ if (inputBlocks[i].isErased()) {
+ erasedBlocks[j++] = inputBlocks[i];
+ }
+ }
+
+ return erasedBlocks;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
index b319cfa..ae396a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
@@ -27,7 +27,9 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
* a decoding step involved in the whole process of decoding a block group.
*/
@InterfaceAudience.Private
-public class ErasureDecodingStep extends AbstractErasureCodingStep {
+public class ErasureDecodingStep implements ErasureCodingStep {
+ private ECBlock[] inputBlocks;
+ private ECBlock[] outputBlocks;
private int[] erasedIndexes;
private RawErasureDecoder rawDecoder;
@@ -41,7 +43,8 @@ public class ErasureDecodingStep extends AbstractErasureCodingStep {
public ErasureDecodingStep(ECBlock[] inputBlocks, int[] erasedIndexes,
ECBlock[] outputBlocks,
RawErasureDecoder rawDecoder) {
- super(inputBlocks, outputBlocks);
+ this.inputBlocks = inputBlocks;
+ this.outputBlocks = outputBlocks;
this.erasedIndexes = erasedIndexes;
this.rawDecoder = rawDecoder;
}
@@ -51,4 +54,18 @@ public class ErasureDecodingStep extends AbstractErasureCodingStep {
rawDecoder.decode(inputChunks, erasedIndexes, outputChunks);
}
+ @Override
+ public ECBlock[] getInputBlocks() {
+ return inputBlocks;
+ }
+
+ @Override
+ public ECBlock[] getOutputBlocks() {
+ return outputBlocks;
+ }
+
+ @Override
+ public void finish() {
+ // TODO: Finalize decoder if necessary
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
new file mode 100644
index 0000000..81666e9
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+
+/**
+ * An abstract erasure encoder that's to be inherited by new encoders.
+ *
+ * It implements the {@link ErasureCoder} interface.
+ */
+@InterfaceAudience.Private
+public abstract class ErasureEncoder extends Configured
+ implements ErasureCoder {
+
+ private final int numDataUnits;
+ private final int numParityUnits;
+ private final ErasureCoderOptions options;
+
+ public ErasureEncoder(ErasureCoderOptions options) {
+ this.options = options;
+ this.numDataUnits = options.getNumDataUnits();
+ this.numParityUnits = options.getNumParityUnits();
+ }
+
+ @Override
+ public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
+ // We may have more than this when considering complicate cases. HADOOP-11550
+ return prepareEncodingStep(blockGroup);
+ }
+
+ @Override
+ public int getNumDataUnits() {
+ return numDataUnits;
+ }
+
+ @Override
+ public int getNumParityUnits() {
+ return numParityUnits;
+ }
+
+ @Override
+ public ErasureCoderOptions getOptions() {
+ return options;
+ }
+
+ protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
+ return blockGroup.getDataBlocks();
+ }
+
+ protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
+ return blockGroup.getParityBlocks();
+ }
+
+ @Override
+ public boolean preferDirectBuffer() {
+ return false;
+ }
+
+ @Override
+ public void release() {
+ // Nothing to do by default
+ }
+
+ /**
+ * Perform encoding against a block group.
+ * @param blockGroup
+ * @return encoding step for caller to do the real work
+ */
+ protected abstract ErasureCodingStep prepareEncodingStep(
+ ECBlockGroup blockGroup);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
index 311de21..f0b2be8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
@@ -27,8 +27,9 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
* an encoding step involved in the whole process of encoding a block group.
*/
@InterfaceAudience.Private
-public class ErasureEncodingStep extends AbstractErasureCodingStep {
-
+public class ErasureEncodingStep implements ErasureCodingStep {
+ private ECBlock[] inputBlocks;
+ private ECBlock[] outputBlocks;
private RawErasureEncoder rawEncoder;
/**
@@ -39,7 +40,8 @@ public class ErasureEncodingStep extends AbstractErasureCodingStep {
*/
public ErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks,
RawErasureEncoder rawEncoder) {
- super(inputBlocks, outputBlocks);
+ this.inputBlocks = inputBlocks;
+ this.outputBlocks = outputBlocks;
this.rawEncoder = rawEncoder;
}
@@ -48,4 +50,18 @@ public class ErasureEncodingStep extends AbstractErasureCodingStep {
rawEncoder.encode(inputChunks, outputChunks);
}
+ @Override
+ public ECBlock[] getInputBlocks() {
+ return inputBlocks;
+ }
+
+ @Override
+ public ECBlock[] getOutputBlocks() {
+ return outputBlocks;
+ }
+
+ @Override
+ public void finish() {
+ rawEncoder.release();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java
new file mode 100644
index 0000000..a0f5b72
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+
+/**
+ * Abstract class for Hitchhiker common facilities shared by
+ * {@link HHXORErasureEncodingStep}and {@link HHXORErasureDecodingStep}.
+ *
+ * It implements {@link ErasureCodingStep}.
+ */
+@InterfaceAudience.Private
+public abstract class HHErasureCodingStep
+ implements ErasureCodingStep {
+
+ private ECBlock[] inputBlocks;
+ private ECBlock[] outputBlocks;
+
+ private static final int SUB_PACKET_SIZE = 2;
+
+ /**
+ * Constructor given input blocks and output blocks.
+ *
+ * @param inputBlocks
+ * @param outputBlocks
+ */
+ public HHErasureCodingStep(ECBlock[] inputBlocks,
+ ECBlock[] outputBlocks) {
+ this.inputBlocks = inputBlocks;
+ this.outputBlocks = outputBlocks;
+ }
+
+ protected int getSubPacketSize() {
+ return SUB_PACKET_SIZE;
+ }
+
+ @Override
+ public ECBlock[] getInputBlocks() {
+ return inputBlocks;
+ }
+
+ @Override
+ public ECBlock[] getOutputBlocks() {
+ return outputBlocks;
+ }
+
+ @Override
+ public void finish() {
+ // TODO: Finalize encoder/decoder if necessary
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
index 94487d8..05e9384 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
@@ -39,16 +38,12 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
* This is Hitchhiker-XOR erasure decoder that decodes a block group.
*/
@InterfaceAudience.Private
-public class HHXORErasureDecoder extends AbstractErasureDecoder {
+public class HHXORErasureDecoder extends ErasureDecoder {
private RawErasureDecoder rsRawDecoder;
private RawErasureEncoder xorRawEncoder;
- public HHXORErasureDecoder(int numDataUnits, int numParityUnits) {
- super(numDataUnits, numParityUnits);
- }
-
- public HHXORErasureDecoder(ECSchema schema) {
- super(schema);
+ public HHXORErasureDecoder(ErasureCoderOptions options) {
+ super(options);
}
@Override
@@ -71,25 +66,26 @@ public class HHXORErasureDecoder extends AbstractErasureDecoder {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
- ErasureCoderOptions coderOptions = new ErasureCoderOptions(
- getNumDataUnits(), getNumParityUnits());
rsRawDecoder = CodecUtil.createRawDecoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
}
return rsRawDecoder;
}
private RawErasureEncoder checkCreateXorRawEncoder() {
if (xorRawEncoder == null) {
- ErasureCoderOptions coderOptions = new ErasureCoderOptions(
- getNumDataUnits(), getNumParityUnits());
xorRawEncoder = CodecUtil.createRawEncoder(getConf(),
- ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+ ErasureCodeConstants.XOR_CODEC_NAME, getOptions());
}
return xorRawEncoder;
}
@Override
+ public boolean preferDirectBuffer() {
+ return false;
+ }
+
+ @Override
public void release() {
if (rsRawDecoder != null) {
rsRawDecoder.release();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
index 6c81836..98b8503 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
* decoding a block group.
*/
@InterfaceAudience.Private
-public class HHXORErasureDecodingStep extends AbstractHHErasureCodingStep {
+public class HHXORErasureDecodingStep extends HHErasureCodingStep {
private int pbIndex;
private int[] piggyBackIndex;
private int[] piggyBackFullIndex;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
index 219f25c..7a15a05 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
@@ -38,16 +37,12 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
* This is Hitchhiker-XOR erasure encoder that encodes a block group.
*/
@InterfaceAudience.Private
-public class HHXORErasureEncoder extends AbstractErasureEncoder {
+public class HHXORErasureEncoder extends ErasureEncoder {
private RawErasureEncoder rsRawEncoder;
private RawErasureEncoder xorRawEncoder;
- public HHXORErasureEncoder(int numDataUnits, int numParityUnits) {
- super(numDataUnits, numParityUnits);
- }
-
- public HHXORErasureEncoder(ECSchema schema) {
- super(schema);
+ public HHXORErasureEncoder(ErasureCoderOptions options) {
+ super(options);
}
@Override
@@ -65,21 +60,17 @@ public class HHXORErasureEncoder extends AbstractErasureEncoder {
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rsRawEncoder == null) {
- ErasureCoderOptions coderOptions = new ErasureCoderOptions(
- getNumDataUnits(), getNumParityUnits());
rsRawEncoder = CodecUtil.createRawEncoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
}
return rsRawEncoder;
}
private RawErasureEncoder checkCreateXorRawEncoder() {
if (xorRawEncoder == null) {
- ErasureCoderOptions erasureCoderOptions = new ErasureCoderOptions(
- getNumDataUnits(), getNumParityUnits());
xorRawEncoder = CodecUtil.createRawEncoder(getConf(),
ErasureCodeConstants.XOR_CODEC_NAME,
- erasureCoderOptions);
+ getOptions());
}
return xorRawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
index f83ee26..11b1bf1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
* encoding a block group.
*/
@InterfaceAudience.Private
-public class HHXORErasureEncodingStep extends AbstractHHErasureCodingStep {
+public class HHXORErasureEncodingStep extends HHErasureCodingStep {
private int[] piggyBackIndex;
private RawErasureEncoder rsRawEncoder;
private RawErasureEncoder xorRawEncoder;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org