You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2017/02/24 23:56:49 UTC
[22/26] hadoop git commit: HDFS-11427. Rename rs-default to rs.
HDFS-11427. Rename rs-default to rs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1a52b04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1a52b04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1a52b04
Branch: refs/heads/YARN-5734
Commit: c1a52b04d0cc5ad5c86ae93043655f313386f7f9
Parents: b32ffa2
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Feb 24 10:58:45 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Feb 24 10:58:45 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/io/erasurecode/CodecUtil.java | 28 +++++++--------
.../io/erasurecode/ErasureCodeConstants.java | 8 ++---
.../erasurecode/coder/HHXORErasureDecoder.java | 2 +-
.../erasurecode/coder/HHXORErasureEncoder.java | 2 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 2 +-
.../io/erasurecode/coder/RSErasureEncoder.java | 2 +-
.../src/main/resources/core-default.xml | 4 +--
.../erasurecode/TestCodecRawCoderMapping.java | 10 +++---
.../coder/TestHHXORErasureCoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 2 +-
.../src/site/markdown/HDFSErasureCoding.md | 6 ++--
.../TestDFSRSDefault10x4StripedInputStream.java | 2 +-
...TestDFSRSDefault10x4StripedOutputStream.java | 2 +-
...fault10x4StripedOutputStreamWithFailure.java | 4 +--
.../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +-
.../TestDFSStripedOutputStreamWithFailure.java | 2 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 2 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +-
.../test/resources/testErasureCodingConf.xml | 36 ++++++++++----------
20 files changed, 61 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 977bacb..861451a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -55,9 +55,9 @@ public final class CodecUtil {
public static final String IO_ERASURECODE_CODEC_XOR =
XORErasureCodec.class.getCanonicalName();
/** Erasure coder Reed-Solomon codec. */
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_KEY =
+ public static final String IO_ERASURECODE_CODEC_RS_KEY =
"io.erasurecode.codec.rs";
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT =
+ public static final String IO_ERASURECODE_CODEC_RS =
RSErasureCodec.class.getCanonicalName();
/** Erasure coder hitch hiker XOR codec. */
public static final String IO_ERASURECODE_CODEC_HHXOR_KEY =
@@ -67,10 +67,10 @@ public final class CodecUtil {
/** Supported erasure codec classes. */
- /** Raw coder factory for the RS default codec. */
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY =
- "io.erasurecode.codec.rs-default.rawcoder";
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT =
+ /** Raw coder factory for the RS codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
+ "io.erasurecode.codec.rs.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT =
RSRawErasureCoderFactory.class.getCanonicalName();
/** Raw coder factory for the RS legacy codec. */
@@ -183,10 +183,10 @@ public final class CodecUtil {
private static String getRawCoderFactNameFromCodec(Configuration conf,
String codec) {
switch (codec) {
- case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
+ case ErasureCodeConstants.RS_CODEC_NAME:
return conf.get(
- IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
- IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT);
+ IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT);
case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
return conf.get(
IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
@@ -233,15 +233,15 @@ public final class CodecUtil {
private static String getCodecClassName(Configuration conf, String codec) {
switch (codec) {
- case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
+ case ErasureCodeConstants.RS_CODEC_NAME:
return conf.get(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY,
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT);
+ CodecUtil.IO_ERASURECODE_CODEC_RS_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS);
case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
//TODO:rs-legacy should be handled differently.
return conf.get(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY,
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT);
+ CodecUtil.IO_ERASURECODE_CODEC_RS_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS);
case ErasureCodeConstants.XOR_CODEC_NAME:
return conf.get(
CodecUtil.IO_ERASURECODE_CODEC_XOR_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index e168909..c830bb2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -25,16 +25,16 @@ public final class ErasureCodeConstants {
private ErasureCodeConstants() {
}
- public static final String RS_DEFAULT_CODEC_NAME = "rs-default";
+ public static final String RS_CODEC_NAME = "rs";
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
public static final String XOR_CODEC_NAME = "xor";
public static final String HHXOR_CODEC_NAME = "hhxor";
public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
- RS_DEFAULT_CODEC_NAME, 6, 3);
+ RS_CODEC_NAME, 6, 3);
public static final ECSchema RS_3_2_SCHEMA = new ECSchema(
- RS_DEFAULT_CODEC_NAME, 3, 2);
+ RS_CODEC_NAME, 3, 2);
public static final ECSchema RS_6_3_LEGACY_SCHEMA = new ECSchema(
RS_LEGACY_CODEC_NAME, 6, 3);
@@ -43,5 +43,5 @@ public final class ErasureCodeConstants {
XOR_CODEC_NAME, 2, 1);
public static final ECSchema RS_10_4_SCHEMA = new ECSchema(
- RS_DEFAULT_CODEC_NAME, 10, 4);
+ RS_CODEC_NAME, 10, 4);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
index 05e9384..4747e49 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
@@ -67,7 +67,7 @@ public class HHXORErasureDecoder extends ErasureDecoder {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
rsRawDecoder = CodecUtil.createRawDecoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rsRawDecoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
index 7a15a05..6b858b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
@@ -61,7 +61,7 @@ public class HHXORErasureEncoder extends ErasureEncoder {
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rsRawEncoder == null) {
rsRawEncoder = CodecUtil.createRawEncoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rsRawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
index 6e679c3..ba7f773 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
@@ -52,7 +52,7 @@ public class RSErasureDecoder extends ErasureDecoder {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
rsRawDecoder = CodecUtil.createRawDecoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rsRawDecoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
index 7a09b92..193309d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
@@ -53,7 +53,7 @@ public class RSErasureEncoder extends ErasureEncoder {
if (rawEncoder == null) {
// TODO: we should create the raw coder according to codec.
rawEncoder = CodecUtil.createRawEncoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index d8136ee..08ca05b 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -657,10 +657,10 @@
</property>
<property>
- <name>io.erasurecode.codec.rs-default.rawcoder</name>
+ <name>io.erasurecode.codec.rs.rawcoder</name>
<value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
<description>
- Raw coder implementation for the rs-default codec. The default value is a
+ Raw coder implementation for the rs codec. The default value is a
pure Java implementation. There is also a native implementation. Its value
is org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
index 0db001a..7f7fcf3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
@@ -47,12 +47,12 @@ public class TestCodecRawCoderMapping {
public void testRSDefaultRawCoder() {
ErasureCoderOptions coderOptions = new ErasureCoderOptions(
numDataUnit, numParityUnit);
- // should return default raw coder of rs-default codec
+ // should return default raw coder of rs codec
RawErasureEncoder encoder = CodecUtil.createRawEncoder(
- conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
Assert.assertTrue(encoder instanceof RSRawEncoder);
RawErasureDecoder decoder = CodecUtil.createRawDecoder(
- conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
Assert.assertTrue(decoder instanceof RSRawDecoder);
// should return default raw coder of rs-legacy codec
@@ -71,11 +71,11 @@ public class TestCodecRawCoderMapping {
String dummyFactName = "DummyNoneExistingFactory";
// set the dummy factory to rs-legacy and create a raw coder
- // with rs-default, which is OK as the raw coder key is not used
+ // with rs, which is OK as the raw coder key is not used
conf.set(CodecUtil.
IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName);
RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf,
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
Assert.assertTrue(encoder instanceof RSRawEncoder);
// now create the raw coder with rs-legacy, which should throw exception
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
index 1eca1e1..91c4ba8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
@@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
index 5f36c99..3b18347 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
@@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 0283e2b..36fb61d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -64,7 +64,7 @@ Architecture
2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work.
- There are four policies currently being supported: RS-DEFAULT-3-2-64k, RS-DEFAULT-6-3-64k, RS-DEFAULT-10-4-64k and RS-LEGACY-6-3-64k. All with default cell size of 64KB. The system default policy is RS-DEFAULT-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB.
+ Five policies are currently supported: RS-3-2-64k, RS-6-3-64k, RS-10-4-64k, RS-LEGACY-6-3-64k, and XOR-2-1-64k. All with default cell size of 64KB. The system default policy is RS-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB.
* **Intel ISA-L**
Intel ISA-L stands for Intel Intelligent Storage Acceleration Library. ISA-L is a collection of optimized low-level functions used primarily in storage applications. It includes a fast block Reed-Solomon type erasure codes optimized for Intel AVX and AVX2 instruction sets.
@@ -91,7 +91,7 @@ Deployment
### Configuration keys
The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys:
- `io.erasurecode.codec.rs-default.rawcoder` for the default RS codec,
+ `io.erasurecode.codec.rs.rawcoder` for the default RS codec,
`io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec,
`io.erasurecode.codec.xor.rawcoder` for the XOR codec.
The default implementations for all of these codecs are pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information.
@@ -108,7 +108,7 @@ Deployment
HDFS native implementation of default RS codec leverages Intel ISA-L library to improve the encoding and decoding calculation. To enable and use Intel ISA-L, there are three steps.
1. Build ISA-L library. Please refer to the offical site "https://github.com/01org/isa-l/" for detail information.
2. Build Hadoop with ISA-L support. Please refer to "Intel ISA-L build options" section in "Build instructions for Hadoop"(BUILDING.txt) document. Use -Dbundle.isal to copy the contents of the isal.lib directory into the final tar file. Deploy hadoop with the tar file. Make sure ISA-L library is available on both HDFS client and DataNodes.
- 3. Configure the `io.erasurecode.codec.rs-default.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes.
+ 3. Configure the `io.erasurecode.codec.rs.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes.
To check ISA-L library enable state, try "Hadoop checknative" command. It will tell you if ISA-L library is enabled or not.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
index fc0ee37..3e6d1e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
/**
- * This tests read operation of DFS striped file with RS-DEFAULT-10-4-64k
+ * This tests read operation of DFS striped file with RS-10-4-64k
* erasure code policy.
*/
public class TestDFSRSDefault10x4StripedInputStream extends
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
index 37821c1..1ea839a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
/**
- * This tests write operation of DFS striped file with RS-DEFAULT-10-4-64k
+ * This tests write operation of DFS striped file with RS-10-4-64k
* erasure code policy.
*/
public class TestDFSRSDefault10x4StripedOutputStream
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
index 1b2ec42..340fec5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
/**
- * This tests write operation of DFS striped file with RS-DEFAULT-10-4-64k
+ * This tests write operation of DFS striped file with RS-10-4-64k
* erasure code policy under Datanode failure conditions.
*/
public class TestDFSRSDefault10x4StripedOutputStreamWithFailure
@@ -33,4 +33,4 @@ public class TestDFSRSDefault10x4StripedOutputStreamWithFailure
return ErasureCodingPolicyManager.getPolicyByPolicyID(
HdfsConstants.RS_10_4_POLICY_ID);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 121b9a4..9b69904 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -96,7 +96,7 @@ public class TestDFSStripedInputStream {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
SimulatedFSDataset.setFactory(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index 5bde16e..8834e14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -85,7 +85,7 @@ public class TestDFSStripedOutputStream {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index 0baf9cc..fabd417 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure {
final int numDNs = dataBlocks + parityBlocks;
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index 4960c58..08b53c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -100,7 +100,7 @@ public class TestReconstructStripedFile {
false);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
index 1a4086e..f26aa06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -69,7 +69,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index 82b71def..9b9003a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -133,7 +133,7 @@
<description>setPolicy : set erasure coding policy on a directory to encode files</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -141,7 +141,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output>
+ <expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -150,8 +150,8 @@
<description>setPolicy : set a policy twice</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -159,7 +159,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output>
+ <expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -168,7 +168,7 @@
<description>unsetPolicy : unset policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
@@ -187,8 +187,8 @@
<description>setPolicy : change different policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-3-2-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-3-2-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
@@ -197,7 +197,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-3-2-64k</expected-output>
+ <expected-output>RS-3-2-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -207,7 +207,7 @@
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<command>-fs NAMENODE -mkdir /ecdir/child</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/child/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/child/ecfile</ec-admin-command>
@@ -220,7 +220,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -246,7 +246,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
@@ -255,7 +255,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -264,7 +264,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/ecfile</ec-admin-command>
</test-commands>
@@ -275,7 +275,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -290,7 +290,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3</expected-output>
+ <expected-output>RS-6-3</expected-output>
</comparator>
</comparators>
</test>
@@ -334,7 +334,7 @@
<description>setPolicy : illegal parameters - too many arguments</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-DEFAULT-3-2-64k /ecdir2</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k /ecdir2</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -367,7 +367,7 @@
<test>
<description>setPolicy : illegal parameters - no such file</description>
<test-commands>
- <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-DEFAULT-3-2-64k</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org