You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dr...@apache.org on 2015/05/22 09:50:44 UTC
hadoop git commit: HADOOP-12013 Generate fixed data to perform
erasure coder test. Contributed by Kai Zheng
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 24d0fbe60 -> 631322517
HADOOP-12013 Generate fixed data to perform erasure coder test. Contributed by Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63132251
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63132251
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63132251
Branch: refs/heads/HDFS-7285
Commit: 6313225177d3895b2bdf96cebea5ef86f2b925b7
Parents: 24d0fbe
Author: Kai Zheng <ka...@intel.com>
Authored: Fri May 22 23:47:52 2015 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Fri May 22 23:47:52 2015 +0800
----------------------------------------------------------------------
.../hadoop-common/CHANGES-HDFS-EC-7285.txt | 2 +
.../hadoop/io/erasurecode/TestCoderBase.java | 87 +++++++++++++++++++-
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 8 +-
.../rawcoder/TestRSRawCoderBase.java | 21 +++--
4 files changed, 110 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63132251/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
index c799b4f..531b8d5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -54,3 +54,5 @@
HADOOP-11938. Enhance ByteBuffer version encode/decode API of raw erasure
coder. (Kai Zheng via Zhe Zhang)
+
+ HADOOP-12013. Generate fixed data to perform erasure coder test. (Kai Zheng)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63132251/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
index cc3617c..3686695 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
@@ -52,6 +52,12 @@ public abstract class TestCoderBase {
// may go to different coding implementations.
protected boolean usingDirectBuffer = true;
+ protected boolean usingFixedData = true;
+ // Using this the generated data can be repeatable across multiple calls to
+ // encode(), in order for troubleshooting.
+ private static int FIXED_DATA_GENERATOR = 0;
+ protected byte[][] fixedData;
+
protected int getChunkSize() {
return chunkSize;
}
@@ -63,13 +69,17 @@ public abstract class TestCoderBase {
/**
* Prepare before running the case.
+ * @param conf
* @param numDataUnits
* @param numParityUnits
* @param erasedDataIndexes
+ * @param erasedParityIndexes
+ * @param usingFixedData Using fixed or pre-generated data to test instead of
+ * generating data
*/
protected void prepare(Configuration conf, int numDataUnits,
int numParityUnits, int[] erasedDataIndexes,
- int[] erasedParityIndexes) {
+ int[] erasedParityIndexes, boolean usingFixedData) {
this.conf = conf;
this.numDataUnits = numDataUnits;
this.numParityUnits = numParityUnits;
@@ -77,6 +87,38 @@ public abstract class TestCoderBase {
erasedDataIndexes : new int[] {0};
this.erasedParityIndexes = erasedParityIndexes != null ?
erasedParityIndexes : new int[] {0};
+ this.usingFixedData = usingFixedData;
+ if (usingFixedData) {
+ prepareFixedData();
+ }
+ }
+
+ /**
+ * Prepare before running the case.
+ * @param conf
+ * @param numDataUnits
+ * @param numParityUnits
+ * @param erasedDataIndexes
+ * @param erasedParityIndexes
+ */
+ protected void prepare(Configuration conf, int numDataUnits,
+ int numParityUnits, int[] erasedDataIndexes,
+ int[] erasedParityIndexes) {
+ prepare(conf, numDataUnits, numParityUnits, erasedDataIndexes,
+ erasedParityIndexes, false);
+ }
+
+ /**
+ * Prepare before running the case.
+ * @param numDataUnits
+ * @param numParityUnits
+ * @param erasedDataIndexes
+ * @param erasedParityIndexes
+ */
+ protected void prepare(int numDataUnits, int numParityUnits,
+ int[] erasedDataIndexes, int[] erasedParityIndexes) {
+ prepare(null, numDataUnits, numParityUnits, erasedDataIndexes,
+ erasedParityIndexes, false);
}
/**
@@ -278,6 +320,29 @@ public abstract class TestCoderBase {
* @return
*/
protected ECChunk[] prepareDataChunksForEncoding() {
+ if (usingFixedData) {
+ ECChunk[] chunks = new ECChunk[numDataUnits];
+ for (int i = 0; i < chunks.length; i++) {
+ chunks[i] = makeChunkUsingData(fixedData[i]);
+ }
+ return chunks;
+ }
+
+ return generateDataChunks();
+ }
+
+ private ECChunk makeChunkUsingData(byte[] data) {
+ ECChunk chunk = allocateOutputChunk();
+ ByteBuffer buffer = chunk.getBuffer();
+ int pos = buffer.position();
+ buffer.put(data, 0, chunkSize);
+ buffer.flip();
+ buffer.position(pos);
+
+ return chunk;
+ }
+
+ private ECChunk[] generateDataChunks() {
ECChunk[] chunks = new ECChunk[numDataUnits];
for (int i = 0; i < chunks.length; i++) {
chunks[i] = generateDataChunk();
@@ -286,6 +351,15 @@ public abstract class TestCoderBase {
return chunks;
}
+ private void prepareFixedData() {
+ // We may load test data from a resource, or just generate randomly.
+ // The generated data will be used across subsequent encode/decode calls.
+ this.fixedData = new byte[numDataUnits][];
+ for (int i = 0; i < numDataUnits; i++) {
+ fixedData[i] = generateFixedData(baseChunkSize * 2);
+ }
+ }
+
/**
* Generate data chunk by making random data.
* @return
@@ -319,6 +393,17 @@ public abstract class TestCoderBase {
return buffer;
}
+ protected byte[] generateFixedData(int len) {
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < buffer.length; i++) {
+ buffer[i] = (byte) FIXED_DATA_GENERATOR++;
+ if (FIXED_DATA_GENERATOR == 256) {
+ FIXED_DATA_GENERATOR = 0;
+ }
+ }
+ return buffer;
+ }
+
/**
* Prepare parity chunks for encoding, each chunk for each parity unit.
* @return
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63132251/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
index 02b9eea..80ec04d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
@@ -80,7 +80,13 @@ public class TestRSRawCoder extends TestRSRawCoderBase {
@Test
public void testCodingDirectBuffer_10x4_erasure_of_d2_d4_p0() {
- prepare(null, 10, 4, new int[] {2, 4}, new int[] {0});
+ prepare(null, 10, 4, new int[]{2, 4}, new int[]{0});
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_usingFixedData_10x4_erasure_of_d2_d4_p0() {
+ prepare(null, 10, 4, new int[] {2, 4}, new int[] {0}, true);
testCoding(true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63132251/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
index c06aded..efde332 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
@@ -17,12 +17,7 @@
*/
package org.apache.hadoop.io.erasurecode.rawcoder;
-import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
/**
* Test base for raw Reed-solomon coders.
@@ -32,6 +27,8 @@ public abstract class TestRSRawCoderBase extends TestRawCoderBase {
private static int symbolSize = 0;
private static int symbolMax = 0;
+ private static int RS_FIXED_DATA_GENERATOR = 0;
+
static {
symbolSize = (int) Math.round(Math.log(
RSUtil.GF.getFieldSize()) / Math.log(2));
@@ -41,9 +38,21 @@ public abstract class TestRSRawCoderBase extends TestRawCoderBase {
@Override
protected byte[] generateData(int len) {
byte[] buffer = new byte[len];
- for (int i = 0; i < len; i++) {
+ for (int i = 0; i < buffer.length; i++) {
buffer[i] = (byte) RAND.nextInt(symbolMax);
}
return buffer;
}
+
+ @Override
+ protected byte[] generateFixedData(int len) {
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < buffer.length; i++) {
+ buffer[i] = (byte) RS_FIXED_DATA_GENERATOR++;
+ if (RS_FIXED_DATA_GENERATOR == symbolMax) {
+ RS_FIXED_DATA_GENERATOR = 0;
+ }
+ }
+ return buffer;
+ }
}