You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/04/13 22:11:09 UTC
[33/50] [abbrv] hadoop git commit: HADOOP-11740. Combine erasure
encoder and decoder interfaces. Contributed by Zhe Zhang.
HADOOP-11740. Combine erasure encoder and decoder interfaces. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a410b28a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a410b28a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a410b28a
Branch: refs/heads/HDFS-7285
Commit: a410b28a827f6d293df9eb9a8144ddfb3c314f9d
Parents: 35ecaea
Author: Zhe Zhang <zh...@apache.org>
Authored: Fri Apr 3 15:22:50 2015 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Mon Apr 13 13:09:56 2015 -0700
----------------------------------------------------------------------
.../coder/AbstractErasureDecoder.java | 7 ++--
.../coder/AbstractErasureEncoder.java | 7 ++--
.../io/erasurecode/coder/ErasureCoder.java | 12 ++++++
.../io/erasurecode/coder/ErasureDecoder.java | 41 --------------------
.../io/erasurecode/coder/ErasureEncoder.java | 39 -------------------
.../erasurecode/coder/TestErasureCoderBase.java | 20 +++++-----
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 14 ++++++-
7 files changed, 41 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a410b28a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
index 54a6d1e..cd31294 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
@@ -23,13 +23,12 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
/**
* An abstract erasure decoder that's to be inherited by new decoders.
*
- * It implements the {@link ErasureDecoder} interface.
+ * It implements the {@link ErasureCoder} interface.
*/
-public abstract class AbstractErasureDecoder extends AbstractErasureCoder
- implements ErasureDecoder {
+public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
@Override
- public ErasureCodingStep decode(ECBlockGroup blockGroup) {
+ public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
// We may have more than this when considering complicate cases. HADOOP-11550
return prepareDecodingStep(blockGroup);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a410b28a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
index 09b31e5..a836b75 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
@@ -23,13 +23,12 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
/**
* An abstract erasure encoder that's to be inherited by new encoders.
*
- * It implements the {@link ErasureEncoder} interface.
+ * It implements the {@link ErasureCoder} interface.
*/
-public abstract class AbstractErasureEncoder extends AbstractErasureCoder
- implements ErasureEncoder {
+public abstract class AbstractErasureEncoder extends AbstractErasureCoder {
@Override
- public ErasureCodingStep encode(ECBlockGroup blockGroup) {
+ public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
// We may have more than this when considering complicate cases. HADOOP-11550
return prepareEncodingStep(blockGroup);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a410b28a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
index c5922f3..fb90156 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
/**
* An erasure coder to perform encoding or decoding given a group. Generally it
@@ -64,6 +65,17 @@ public interface ErasureCoder extends Configurable {
public int getChunkSize();
/**
+ * Calculate the encoding or decoding steps given a block blockGroup.
+ *
+ * Note, currently only one coding step is supported. Will support complex
+ * cases of multiple coding steps.
+ *
+ * @param blockGroup the erasure coding block group containing all necessary
+ * information for codec calculation
+ */
+ public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
+
+ /**
* Tell if native or off-heap buffer is preferred or not. It's for callers to
* decide how to allocate coding chunk buffers, either on heap or off heap.
* It will return false by default.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a410b28a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
deleted file mode 100644
index dfd9e54..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.coder;
-
-
-import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-
-/**
- * Erasure decoder interface to perform decoding given a block group.
- *
- * It extends {@link ErasureCoder}.
- */
-public interface ErasureDecoder extends ErasureCoder {
-
- /**
- * Perform the decoding given a blockGroup. By default it will try the best to
- * attempt to recover all the missing blocks according to the codec logic.
- *
- * Note, currently only one coding step is supported. Will support complex
- * cases of multiple coding steps.
- *
- * @param blockGroup
- */
- public ErasureCodingStep decode(ECBlockGroup blockGroup);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a410b28a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
deleted file mode 100644
index e837d22..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.coder;
-
-import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-
-/**
- * Erasure encoder interface to perform encoding given a block group.
- *
- * It extends {@link ErasureCoder}.
- */
-public interface ErasureEncoder extends ErasureCoder {
-
- /**
- * Calculate the encoding steps given a block blockGroup.
- *
- * Note, currently only one coding step is supported. Will support complex
- * cases of multiple coding steps.
- *
- * @param blockGroup
- */
- public ErasureCodingStep encode(ECBlockGroup blockGroup);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a410b28a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
index b963a59..05a62a7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
@@ -26,8 +26,8 @@ import org.apache.hadoop.io.erasurecode.TestCoderBase;
* Erasure coder test base with utilities.
*/
public abstract class TestErasureCoderBase extends TestCoderBase {
- protected Class<? extends ErasureEncoder> encoderClass;
- protected Class<? extends ErasureDecoder> decoderClass;
+ protected Class<? extends ErasureCoder> encoderClass;
+ protected Class<? extends ErasureCoder> decoderClass;
protected int numChunksInBlock = 16;
@@ -55,7 +55,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
protected void testCoding(boolean usingDirectBuffer) {
this.usingDirectBuffer = usingDirectBuffer;
- ErasureEncoder encoder = createEncoder();
+ ErasureCoder encoder = createEncoder();
// Generate data and encode
ECBlockGroup blockGroup = prepareBlockGroupForEncoding();
@@ -68,7 +68,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
ErasureCodingStep codingStep;
try {
- codingStep = encoder.encode(blockGroup);
+ codingStep = encoder.calculateCoding(blockGroup);
performCodingStep(codingStep);
} finally {
encoder.release();
@@ -78,9 +78,9 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
//Decode
blockGroup = new ECBlockGroup(clonedDataBlocks, blockGroup.getParityBlocks());
- ErasureDecoder decoder = createDecoder();
+ ErasureCoder decoder = createDecoder();
try {
- codingStep = decoder.decode(blockGroup);
+ codingStep = decoder.calculateCoding(blockGroup);
performCodingStep(codingStep);
} finally {
decoder.release();
@@ -138,8 +138,8 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
* Create erasure encoder for test.
* @return
*/
- private ErasureEncoder createEncoder() {
- ErasureEncoder encoder;
+ private ErasureCoder createEncoder() {
+ ErasureCoder encoder;
try {
encoder = encoderClass.newInstance();
} catch (Exception e) {
@@ -155,8 +155,8 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
* Create the erasure decoder for the test.
* @return
*/
- private ErasureDecoder createDecoder() {
- ErasureDecoder decoder;
+ private ErasureCoder createDecoder() {
+ ErasureCoder decoder;
try {
decoder = decoderClass.newInstance();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a410b28a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index a686315..4e60a7c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -43,4 +43,16 @@
blocks in NameNode (Jing Zhao)
HDFS-8005. Erasure Coding: simplify striped block recovery work computation
- and add tests (Jing Zhao)
\ No newline at end of file
+ and add tests (Jing Zhao)
+
+ HDFS-7617. Add unit tests for editlog transactions for EC
+ (Hui Zheng via Zhe Zhang)
+
+ HADOOP-11782. Correct two thrown messages in ECSchema class
+ (Xinwei Qin via Kai Zheng)
+
+ HDFS-7839. Erasure coding: implement facilities in NameNode to create and
+ manage EC zones (Zhe Zhang)
+
+ HADOOP-11740. Combine erasure encoder and decoder interfaces (Zhe Zhang)
+