You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dr...@apache.org on 2015/05/25 02:16:05 UTC
hadoop git commit: HDFS-8382 Remove chunkSize and initialize from
erasure coder. Contributed by Kai Zheng
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 61172fba8 -> 6a0afa6b1
HDFS-8382 Remove chunkSize and initialize from erasure coder. Contributed by Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a0afa6b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a0afa6b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a0afa6b
Branch: refs/heads/HDFS-7285
Commit: 6a0afa6b1fcdfcdb5e4ddd4f170112619056a385
Parents: 61172fb
Author: Kai Zheng <ka...@intel.com>
Authored: Mon May 25 16:13:29 2015 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Mon May 25 16:13:29 2015 +0800
----------------------------------------------------------------------
.../erasurecode/codec/AbstractErasureCodec.java | 43 ++------------------
.../io/erasurecode/codec/ErasureCodec.java | 7 ----
.../io/erasurecode/codec/RSErasureCodec.java | 13 ++++--
.../io/erasurecode/codec/XORErasureCodec.java | 13 +++---
.../erasurecode/coder/AbstractErasureCoder.java | 39 ++++++++----------
.../coder/AbstractErasureDecoder.java | 9 ++++
.../coder/AbstractErasureEncoder.java | 9 ++++
.../io/erasurecode/coder/ErasureCoder.java | 27 ++----------
.../io/erasurecode/coder/RSErasureDecoder.java | 16 ++++++--
.../io/erasurecode/coder/RSErasureEncoder.java | 16 ++++++--
.../io/erasurecode/coder/XORErasureDecoder.java | 14 +++++--
.../io/erasurecode/coder/XORErasureEncoder.java | 16 ++++++--
.../rawcoder/AbstractRawErasureCoder.java | 15 ++-----
.../rawcoder/AbstractRawErasureDecoder.java | 4 ++
.../rawcoder/AbstractRawErasureEncoder.java | 4 ++
.../io/erasurecode/rawcoder/RSRawDecoder.java | 5 +--
.../io/erasurecode/rawcoder/RSRawEncoder.java | 6 +--
.../rawcoder/RSRawErasureCoderFactory.java | 8 ++--
.../erasurecode/rawcoder/RawErasureCoder.java | 14 -------
.../rawcoder/RawErasureCoderFactory.java | 8 +++-
.../io/erasurecode/rawcoder/XORRawDecoder.java | 4 ++
.../io/erasurecode/rawcoder/XORRawEncoder.java | 4 ++
.../rawcoder/XORRawErasureCoderFactory.java | 8 ++--
.../erasurecode/coder/TestErasureCoderBase.java | 42 +++++++++++--------
.../erasurecode/rawcoder/TestRawCoderBase.java | 15 ++++---
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 2 +
.../hadoop/hdfs/DFSStripedOutputStream.java | 3 +-
.../erasurecode/ErasureCodingWorker.java | 14 +++----
.../hadoop/hdfs/util/StripedBlockUtil.java | 3 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 4 +-
.../hadoop/hdfs/TestWriteReadStripedFile.java | 3 +-
31 files changed, 187 insertions(+), 201 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
index 9993786..0cacfbc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.io.erasurecode.codec;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.erasurecode.ECSchema;
-import org.apache.hadoop.io.erasurecode.coder.*;
import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper;
/**
@@ -28,10 +27,9 @@ import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper;
public abstract class AbstractErasureCodec extends Configured
implements ErasureCodec {
- private ECSchema schema;
+ private final ECSchema schema;
- @Override
- public void setSchema(ECSchema schema) {
+ public AbstractErasureCodec(ECSchema schema) {
this.schema = schema;
}
@@ -39,7 +37,7 @@ public abstract class AbstractErasureCodec extends Configured
return schema.getCodecName();
}
- protected ECSchema getSchema() {
+ public ECSchema getSchema() {
return schema;
}
@@ -50,39 +48,4 @@ public abstract class AbstractErasureCodec extends Configured
return blockGrouper;
}
-
- @Override
- public ErasureCoder createEncoder() {
- ErasureCoder encoder = doCreateEncoder();
- prepareErasureCoder(encoder);
- return encoder;
- }
-
- /**
- * Create a new encoder instance to be initialized afterwards.
- * @return encoder
- */
- protected abstract ErasureCoder doCreateEncoder();
-
- @Override
- public ErasureCoder createDecoder() {
- ErasureCoder decoder = doCreateDecoder();
- prepareErasureCoder(decoder);
- return decoder;
- }
-
- /**
- * Create a new decoder instance to be initialized afterwards.
- * @return decoder
- */
- protected abstract ErasureCoder doCreateDecoder();
-
- private void prepareErasureCoder(ErasureCoder erasureCoder) {
- if (getSchema() == null) {
- throw new RuntimeException("No schema been set yet");
- }
-
- erasureCoder.setConf(getConf());
- erasureCoder.initialize(getSchema());
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
index e639484..9aa3db2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.io.erasurecode.codec;
import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper;
@@ -30,12 +29,6 @@ import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper;
public interface ErasureCodec extends Configurable {
/**
- * Set EC schema to be used by this codec.
- * @param schema
- */
- public void setSchema(ECSchema schema);
-
- /**
* Create block grouper
* @return block grouper
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
index 9e91b60..6edd638 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.io.erasurecode.codec;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
import org.apache.hadoop.io.erasurecode.coder.RSErasureDecoder;
import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder;
@@ -26,13 +27,17 @@ import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder;
*/
public class RSErasureCodec extends AbstractErasureCodec {
+ public RSErasureCodec(ECSchema schema) {
+ super(schema);
+ }
+
@Override
- protected ErasureCoder doCreateEncoder() {
- return new RSErasureEncoder();
+ public ErasureCoder createEncoder() {
+ return new RSErasureEncoder(getSchema());
}
@Override
- protected ErasureCoder doCreateDecoder() {
- return new RSErasureDecoder();
+ public ErasureCoder createDecoder() {
+ return new RSErasureDecoder(getSchema());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
index 0f726d7..e2dcfa7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
@@ -27,19 +27,18 @@ import org.apache.hadoop.io.erasurecode.coder.XORErasureEncoder;
*/
public class XORErasureCodec extends AbstractErasureCodec {
- @Override
- public void setSchema(ECSchema schema) {
- super.setSchema(schema);
+ public XORErasureCodec(ECSchema schema) {
+ super(schema);
assert(schema.getNumParityUnits() == 1);
}
@Override
- protected ErasureCoder doCreateEncoder() {
- return new XORErasureEncoder();
+ public ErasureCoder createEncoder() {
+ return new XORErasureEncoder(getSchema());
}
@Override
- protected ErasureCoder doCreateDecoder() {
- return new XORErasureDecoder();
+ public ErasureCoder createDecoder() {
+ return new XORErasureDecoder(getSchema());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
index d491570..c572bad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
@@ -33,18 +33,18 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
public abstract class AbstractErasureCoder
extends Configured implements ErasureCoder {
- private int numDataUnits;
- private int numParityUnits;
- private int chunkSize;
+ private final int numDataUnits;
+ private final int numParityUnits;
/**
* Create raw decoder using the factory specified by rawCoderFactoryKey
* @param rawCoderFactoryKey
* @return raw decoder
*/
- protected RawErasureDecoder createRawDecoder(String rawCoderFactoryKey) {
+ protected RawErasureDecoder createRawDecoder(
+ String rawCoderFactoryKey, int dataUnitsCount, int parityUnitsCount) {
RawErasureCoder rawCoder = createRawCoder(getConf(),
- rawCoderFactoryKey, false);
+ rawCoderFactoryKey, false, dataUnitsCount, parityUnitsCount);
return (RawErasureDecoder) rawCoder;
}
@@ -53,9 +53,10 @@ public abstract class AbstractErasureCoder
* @param rawCoderFactoryKey
* @return raw encoder
*/
- protected RawErasureEncoder createRawEncoder(String rawCoderFactoryKey) {
+ protected RawErasureEncoder createRawEncoder(
+ String rawCoderFactoryKey, int dataUnitsCount, int parityUnitsCount) {
RawErasureCoder rawCoder = createRawCoder(getConf(),
- rawCoderFactoryKey, true);
+ rawCoderFactoryKey, true, dataUnitsCount, parityUnitsCount);
return (RawErasureEncoder) rawCoder;
}
@@ -67,7 +68,8 @@ public abstract class AbstractErasureCoder
* @return raw coder
*/
public static RawErasureCoder createRawCoder(Configuration conf,
- String rawCoderFactoryKey, boolean isEncoder) {
+ String rawCoderFactoryKey, boolean isEncoder, int numDataUnits,
+ int numParityUnits) {
if (conf == null) {
return null;
@@ -90,21 +92,17 @@ public abstract class AbstractErasureCoder
throw new RuntimeException("Failed to create raw coder", e);
}
- return isEncoder ? fact.createEncoder() : fact.createDecoder();
+ return isEncoder ? fact.createEncoder(numDataUnits, numParityUnits) :
+ fact.createDecoder(numDataUnits, numParityUnits);
}
- @Override
- public void initialize(int numDataUnits, int numParityUnits,
- int chunkSize) {
+ public AbstractErasureCoder(int numDataUnits, int numParityUnits) {
this.numDataUnits = numDataUnits;
this.numParityUnits = numParityUnits;
- this.chunkSize = chunkSize;
}
- @Override
- public void initialize(ECSchema schema) {
- initialize(schema.getNumDataUnits(), schema.getNumParityUnits(),
- schema.getChunkSize());
+ public AbstractErasureCoder(ECSchema schema) {
+ this(schema.getNumDataUnits(), schema.getNumParityUnits());
}
@Override
@@ -118,12 +116,7 @@ public abstract class AbstractErasureCoder
}
@Override
- public int getChunkSize() {
- return chunkSize;
- }
-
- @Override
- public boolean preferNativeBuffer() {
+ public boolean preferDirectBuffer() {
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
index 6437236..3ea9311 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
/**
* An abstract erasure decoder that's to be inherited by new decoders.
@@ -27,6 +28,14 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
*/
public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
+ public AbstractErasureDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public AbstractErasureDecoder(ECSchema schema) {
+ super(schema);
+ }
+
@Override
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
// We may have more than this when considering complicate cases. HADOOP-11550
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
index a836b75..7c887e8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
/**
* An abstract erasure encoder that's to be inherited by new encoders.
@@ -27,6 +28,14 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
*/
public abstract class AbstractErasureEncoder extends AbstractErasureCoder {
+ public AbstractErasureEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public AbstractErasureEncoder(ECSchema schema) {
+ super(schema);
+ }
+
@Override
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
// We may have more than this when considering complicate cases. HADOOP-11550
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
index 64a82ea..f05ea41 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
/**
* An erasure coder to perform encoding or decoding given a group. Generally it
@@ -38,20 +37,6 @@ import org.apache.hadoop.io.erasurecode.ECSchema;
public interface ErasureCoder extends Configurable {
/**
- * Initialize with the important parameters for the code.
- * @param numDataUnits how many data inputs for the coding
- * @param numParityUnits how many parity outputs the coding generates
- * @param chunkSize the size of the input/output buffer
- */
- public void initialize(int numDataUnits, int numParityUnits, int chunkSize);
-
- /**
- * Initialize with an EC schema.
- * @param schema
- */
- public void initialize(ECSchema schema);
-
- /**
* The number of data input units for the coding. A unit can be a byte,
* chunk or buffer or even a block.
* @return count of data input units
@@ -66,12 +51,6 @@ public interface ErasureCoder extends Configurable {
public int getNumParityUnits();
/**
- * Chunk buffer size for the input/output
- * @return chunk buffer size
- */
- public int getChunkSize();
-
- /**
* Calculate the encoding or decoding steps given a block blockGroup.
*
* Note, currently only one coding step is supported. Will support complex
@@ -83,13 +62,13 @@ public interface ErasureCoder extends Configurable {
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
/**
- * Tell if native or off-heap buffer is preferred or not. It's for callers to
+ * Tell if direct or off-heap buffer is preferred or not. It's for callers to
* decide how to allocate coding chunk buffers, either on heap or off heap.
* It will return false by default.
- * @return true if native buffer is preferred for performance consideration,
+ * @return true if direct buffer is preferred for performance consideration,
* otherwise false.
*/
- public boolean preferNativeBuffer();
+ public boolean preferDirectBuffer();
/**
* Release the resources if any. Good chance to invoke RawErasureCoder#release.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
index ec7cbb5..57f4373 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
@@ -31,6 +32,14 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
public class RSErasureDecoder extends AbstractErasureDecoder {
private RawErasureDecoder rsRawDecoder;
+ public RSErasureDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public RSErasureDecoder(ECSchema schema) {
+ super(schema);
+ }
+
@Override
protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) {
@@ -45,12 +54,11 @@ public class RSErasureDecoder extends AbstractErasureDecoder {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
rsRawDecoder = createRawDecoder(
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY);
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ getNumDataUnits(), getNumParityUnits());
if (rsRawDecoder == null) {
- rsRawDecoder = new RSRawDecoder();
+ rsRawDecoder = new RSRawDecoder(getNumDataUnits(), getNumParityUnits());
}
- rsRawDecoder.initialize(getNumDataUnits(),
- getNumParityUnits(), getChunkSize());
}
return rsRawDecoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
index 18ca5ac..ab23474 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
@@ -31,6 +32,14 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
public class RSErasureEncoder extends AbstractErasureEncoder {
private RawErasureEncoder rawEncoder;
+ public RSErasureEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public RSErasureEncoder(ECSchema schema) {
+ super(schema);
+ }
+
@Override
protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) {
@@ -45,12 +54,11 @@ public class RSErasureEncoder extends AbstractErasureEncoder {
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rawEncoder == null) {
rawEncoder = createRawEncoder(
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY);
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ getNumDataUnits(), getNumParityUnits());
if (rawEncoder == null) {
- rawEncoder = new RSRawEncoder();
+ rawEncoder = new RSRawEncoder(getNumDataUnits(), getNumParityUnits());
}
- rawEncoder.initialize(getNumDataUnits(),
- getNumParityUnits(), getChunkSize());
}
return rawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java
index 0672549..3fe8d1b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
@@ -29,12 +30,19 @@ import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
*/
public class XORErasureDecoder extends AbstractErasureDecoder {
+ public XORErasureDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public XORErasureDecoder(ECSchema schema) {
+ super(schema);
+ }
+
@Override
protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) {
// May be configured
- RawErasureDecoder rawDecoder = new XORRawDecoder();
- rawDecoder.initialize(getNumDataUnits(),
- getNumParityUnits(), getChunkSize());
+ RawErasureDecoder rawDecoder = new XORRawDecoder(
+ getNumDataUnits(), getNumParityUnits());
ECBlock[] inputBlocks = getInputBlocks(blockGroup);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java
index 9011857..5020896 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java
@@ -19,22 +19,30 @@ package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
/**
* Xor erasure encoder that encodes a block group.
*
- * It implements {@link ErasureEncoder}.
+ * It implements {@link ErasureCoder}.
*/
public class XORErasureEncoder extends AbstractErasureEncoder {
+ public XORErasureEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public XORErasureEncoder(ECSchema schema) {
+ super(schema);
+ }
+
@Override
protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) {
// May be configured
- RawErasureEncoder rawEncoder = new XORRawEncoder();
- rawEncoder.initialize(getNumDataUnits(),
- getNumParityUnits(), getChunkSize());
+ RawErasureEncoder rawEncoder = new XORRawEncoder(
+ getNumDataUnits(), getNumParityUnits());
ECBlock[] inputBlocks = getInputBlocks(blockGroup);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
index 5268962..06ae660 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
@@ -31,16 +31,12 @@ import java.util.Arrays;
public abstract class AbstractRawErasureCoder
extends Configured implements RawErasureCoder {
- private int numDataUnits;
- private int numParityUnits;
- private int chunkSize;
+ private final int numDataUnits;
+ private final int numParityUnits;
- @Override
- public void initialize(int numDataUnits, int numParityUnits,
- int chunkSize) {
+ public AbstractRawErasureCoder(int numDataUnits, int numParityUnits) {
this.numDataUnits = numDataUnits;
this.numParityUnits = numParityUnits;
- this.chunkSize = chunkSize;
}
@Override
@@ -54,11 +50,6 @@ public abstract class AbstractRawErasureCoder
}
@Override
- public int getChunkSize() {
- return chunkSize;
- }
-
- @Override
public boolean preferDirectBuffer() {
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
index 31f4fb8..0c1f80f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
@@ -30,6 +30,10 @@ import java.nio.ByteBuffer;
public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder
implements RawErasureDecoder {
+ public AbstractRawErasureDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
@Override
public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
ByteBuffer[] outputs) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
index 0ae54c5..c7a136b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
@@ -30,6 +30,10 @@ import java.nio.ByteBuffer;
public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder
implements RawErasureEncoder {
+ public AbstractRawErasureEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
@Override
public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
checkParameters(inputs, outputs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
index ff1162f..e265dce 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
@@ -31,9 +31,8 @@ public class RSRawDecoder extends AbstractRawErasureDecoder {
private int[] errSignature;
private int[] primitivePower;
- @Override
- public void initialize(int numDataUnits, int numParityUnits, int chunkSize) {
- super.initialize(numDataUnits, numParityUnits, chunkSize);
+ public RSRawDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
assert (getNumDataUnits() + getNumParityUnits() < RSUtil.GF.getFieldSize());
this.errSignature = new int[numParityUnits];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
index 9136331..efeee90 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
@@ -29,9 +29,9 @@ import java.nio.ByteBuffer;
public class RSRawEncoder extends AbstractRawErasureEncoder {
private int[] generatingPolynomial;
- @Override
- public void initialize(int numDataUnits, int numParityUnits, int chunkSize) {
- super.initialize(numDataUnits, numParityUnits, chunkSize);
+ public RSRawEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+
assert (getNumDataUnits() + getNumParityUnits() < RSUtil.GF.getFieldSize());
int[] primitivePower = RSUtil.getPrimitivePower(numDataUnits,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
index 19a95af..5db49e3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
@@ -23,12 +23,12 @@ package org.apache.hadoop.io.erasurecode.rawcoder;
public class RSRawErasureCoderFactory implements RawErasureCoderFactory {
@Override
- public RawErasureEncoder createEncoder() {
- return new RSRawEncoder();
+ public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits) {
+ return new RSRawEncoder(numDataUnits, numParityUnits);
}
@Override
- public RawErasureDecoder createDecoder() {
- return new RSRawDecoder();
+ public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits) {
+ return new RSRawDecoder(numDataUnits, numParityUnits);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
index 3fb211f..0ddb460 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
@@ -36,14 +36,6 @@ import org.apache.hadoop.conf.Configurable;
public interface RawErasureCoder extends Configurable {
/**
- * Initialize with the important parameters for the code.
- * @param numDataUnits how many data inputs for the coding
- * @param numParityUnits how many parity outputs the coding generates
- * @param chunkSize the size of the input/output buffer
- */
- public void initialize(int numDataUnits, int numParityUnits, int chunkSize);
-
- /**
* The number of data input units for the coding. A unit can be a byte,
* chunk or buffer or even a block.
* @return count of data input units
@@ -58,12 +50,6 @@ public interface RawErasureCoder extends Configurable {
public int getNumParityUnits();
/**
- * Chunk buffer size for the input/output
- * @return chunk buffer size
- */
- public int getChunkSize();
-
- /**
* Tell if direct buffer is preferred or not. It's for callers to
* decide how to allocate coding chunk buffers, using DirectByteBuffer or
* bytes array. It will return false by default.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
index 95a1cfe..26eddfc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
@@ -26,13 +26,17 @@ public interface RawErasureCoderFactory {
/**
* Create raw erasure encoder.
+ * @param numDataUnits
+ * @param numParityUnits
* @return raw erasure encoder
*/
- public RawErasureEncoder createEncoder();
+ public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits);
/**
* Create raw erasure decoder.
+ * @param numDataUnits
+ * @param numParityUnits
* @return raw erasure decoder
*/
- public RawErasureDecoder createDecoder();
+ public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
index bf6e894..a09105c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
@@ -28,6 +28,10 @@ import java.nio.ByteBuffer;
*/
public class XORRawDecoder extends AbstractRawErasureDecoder {
+ public XORRawDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
@Override
protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes,
ByteBuffer[] outputs) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
index feffbbf..894f20c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
@@ -28,6 +28,10 @@ import java.nio.ByteBuffer;
*/
public class XORRawEncoder extends AbstractRawErasureEncoder {
+ public XORRawEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
ByteBuffer output = outputs[0];
resetOutputBuffer(output);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
index 67f45c1..de20c95 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
@@ -23,12 +23,12 @@ package org.apache.hadoop.io.erasurecode.rawcoder;
public class XORRawErasureCoderFactory implements RawErasureCoderFactory {
@Override
- public RawErasureEncoder createEncoder() {
- return new XORRawEncoder();
+ public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits) {
+ return new XORRawEncoder(numDataUnits, numParityUnits);
}
@Override
- public RawErasureDecoder createDecoder() {
- return new XORRawDecoder();
+ public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits) {
+ return new XORRawDecoder(numDataUnits, numParityUnits);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
index 154ec18..f9666b6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.TestCoderBase;
+import java.lang.reflect.Constructor;
+
/**
* Erasure coder test base with utilities.
*/
@@ -139,46 +141,50 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
}
}
+ private void prepareCoders() {
+ if (encoder == null) {
+ encoder = createEncoder();
+ }
+
+ if (decoder == null) {
+ decoder = createDecoder();
+ }
+ }
+
/**
- * Create erasure encoder for test.
+ * Create the raw erasure encoder to test
* @return
*/
- private ErasureCoder createEncoder() {
+ protected ErasureCoder createEncoder() {
ErasureCoder encoder;
try {
- encoder = encoderClass.newInstance();
+ Constructor<? extends ErasureCoder> constructor =
+ (Constructor<? extends ErasureCoder>)
+ encoderClass.getConstructor(int.class, int.class);
+ encoder = constructor.newInstance(numDataUnits, numParityUnits);
} catch (Exception e) {
throw new RuntimeException("Failed to create encoder", e);
}
- encoder.initialize(numDataUnits, numParityUnits, getChunkSize());
encoder.setConf(getConf());
return encoder;
}
- private void prepareCoders() {
- if (encoder == null) {
- encoder = createEncoder();
- }
-
- if (decoder == null) {
- decoder = createDecoder();
- }
- }
-
/**
- * Create the erasure decoder for the test.
+ * create the raw erasure decoder to test
* @return
*/
- private ErasureCoder createDecoder() {
+ protected ErasureCoder createDecoder() {
ErasureCoder decoder;
try {
- decoder = decoderClass.newInstance();
+ Constructor<? extends ErasureCoder> constructor =
+ (Constructor<? extends ErasureCoder>)
+ decoderClass.getConstructor(int.class, int.class);
+ decoder = constructor.newInstance(numDataUnits, numParityUnits);
} catch (Exception e) {
throw new RuntimeException("Failed to create decoder", e);
}
- decoder.initialize(numDataUnits, numParityUnits, getChunkSize());
decoder.setConf(getConf());
return decoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
index 4582317..cfaa2c5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
@@ -21,6 +21,8 @@ import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.TestCoderBase;
import org.junit.Assert;
+import java.lang.reflect.Constructor;
+
/**
* Raw coder test base with utilities.
*/
@@ -136,12 +138,14 @@ public abstract class TestRawCoderBase extends TestCoderBase {
protected RawErasureEncoder createEncoder() {
RawErasureEncoder encoder;
try {
- encoder = encoderClass.newInstance();
+ Constructor<? extends RawErasureEncoder> constructor =
+ (Constructor<? extends RawErasureEncoder>)
+ encoderClass.getConstructor(int.class, int.class);
+ encoder = constructor.newInstance(numDataUnits, numParityUnits);
} catch (Exception e) {
throw new RuntimeException("Failed to create encoder", e);
}
- encoder.initialize(numDataUnits, numParityUnits, getChunkSize());
encoder.setConf(getConf());
return encoder;
}
@@ -153,14 +157,15 @@ public abstract class TestRawCoderBase extends TestCoderBase {
protected RawErasureDecoder createDecoder() {
RawErasureDecoder decoder;
try {
- decoder = decoderClass.newInstance();
+ Constructor<? extends RawErasureDecoder> constructor =
+ (Constructor<? extends RawErasureDecoder>)
+ decoderClass.getConstructor(int.class, int.class);
+ decoder = constructor.newInstance(numDataUnits, numParityUnits);
} catch (Exception e) {
throw new RuntimeException("Failed to create decoder", e);
}
- decoder.initialize(numDataUnits, numParityUnits, getChunkSize());
decoder.setConf(getConf());
return decoder;
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index efbadaf..4609fb6 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -255,3 +255,5 @@
(waltersu4549)
HDFS-7768. Change fsck to support EC files. (Takanobu Asanuma via szetszwo)
+
+ HDFS-8382. Remove chunkSize and initialize from erasure coder. (Kai Zheng)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 515ce0c..0935d5c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -247,8 +247,7 @@ public class DFSStripedOutputStream extends DFSOutputStream {
numDataBlocks = schema.getNumDataUnits();
numAllBlocks = numDataBlocks + numParityBlocks;
- encoder = new RSRawEncoder();
- encoder.initialize(numDataBlocks, numParityBlocks, cellSize);
+ encoder = new RSRawEncoder(numDataBlocks, numParityBlocks);
coordinator = new Coordinator(dfsClient.getConf(), numDataBlocks, numAllBlocks);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index ded51eb..00cf0fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -111,12 +111,12 @@ public final class ErasureCodingWorker {
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_DEFAULT);
}
- private RawErasureEncoder newEncoder() {
- return new RSRawEncoder();
+ private RawErasureEncoder newEncoder(int numDataUnits, int numParityUnits) {
+ return new RSRawEncoder(numDataUnits, numParityUnits);
}
- private RawErasureDecoder newDecoder() {
- return new RSRawDecoder();
+ private RawErasureDecoder newDecoder(int numDataUnits, int numParityUnits) {
+ return new RSRawDecoder(numDataUnits, numParityUnits);
}
private void initializeStripedReadThreadPool(int num) {
@@ -517,16 +517,14 @@ public final class ErasureCodingWorker {
// Initialize encoder
private void initEncoderIfNecessary() {
if (encoder == null) {
- encoder = newEncoder();
- encoder.initialize(dataBlkNum, parityBlkNum, bufferSize);
+ encoder = newEncoder(dataBlkNum, parityBlkNum);
}
}
// Initialize decoder
private void initDecoderIfNecessary() {
if (decoder == null) {
- decoder = newDecoder();
- decoder.initialize(dataBlkNum, parityBlkNum, bufferSize);
+ decoder = newDecoder(dataBlkNum, parityBlkNum);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 38dc61a..8f63236 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -287,8 +287,7 @@ public class StripedBlockUtil {
}
byte[][] outputs = new byte[parityBlkNum][(int) alignedStripe.getSpanInBlock()];
- RSRawDecoder rsRawDecoder = new RSRawDecoder();
- rsRawDecoder.initialize(dataBlkNum, parityBlkNum, (int) alignedStripe.getSpanInBlock());
+ RSRawDecoder rsRawDecoder = new RSRawDecoder(dataBlkNum, parityBlkNum);
rsRawDecoder.decode(decodeInputs, decodeIndices, outputs);
for (int i = 0; i < dataBlkNum + parityBlkNum; i++) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index e795ea7..e041dbe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -274,8 +274,8 @@ public class TestDFSStripedOutputStream {
System.arraycopy(tmp, 0, dataBytes[i], 0, tmp.length);
}
}
- final RawErasureEncoder encoder = new RSRawEncoder();
- encoder.initialize(dataBytes.length, parityBytes.length, cellSize);
+ final RawErasureEncoder encoder =
+ new RSRawEncoder(dataBytes.length, parityBytes.length);
encoder.encode(dataBytes, expectedParityBytes);
for (int i = 0; i < parityBytes.length; i++) {
if (i != killedDnIndex) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0afa6b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
index 1976dca..70802fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
@@ -382,8 +382,7 @@ public class TestWriteReadStripedFile {
Assert.assertEquals("The length of file should be the same to write size",
length - startOffsetInFile, readLen);
- RSRawDecoder rsRawDecoder = new RSRawDecoder();
- rsRawDecoder.initialize(dataBlocks, parityBlocks, 1);
+ RSRawDecoder rsRawDecoder = new RSRawDecoder(dataBlocks, parityBlocks);
byte[] expected = new byte[readLen];
for (int i = startOffsetInFile; i < length; i++) {
//TODO: workaround (filling fixed bytes), to remove after HADOOP-11938