You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2016/04/21 00:37:06 UTC
[01/13] hadoop git commit: HDFS-10302. BlockPlacementPolicyDefault
should use default replication considerload value. Contributed by Lin Yiqun.
Repository: hadoop
Updated Branches:
refs/heads/YARN-3368 d296832c2 -> 49caa1263
HDFS-10302. BlockPlacementPolicyDefault should use default replication considerload value. Contributed by Lin Yiqun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1bec4cb0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1bec4cb0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1bec4cb0
Branch: refs/heads/YARN-3368
Commit: 1bec4cb0f7d4e36c8b8555f55b61a946e293f230
Parents: 0c4b3f3
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Apr 18 07:58:55 2016 -0500
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:45 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bec4cb0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index f20f5fb..474a5e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -79,7 +79,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
NetworkTopology clusterMap,
Host2NodesMap host2datanodeMap) {
this.considerLoad = conf.getBoolean(
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, true);
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_DEFAULT);
this.considerLoadFactor = conf.getDouble(
DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR,
DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR_DEFAULT);
[12/13] hadoop git commit: HADOOP-12924. Configure raw erasure coders
for supported codecs. Contributed by Rui Li.
Posted by wa...@apache.org.
HADOOP-12924. Configure raw erasure coders for supported codecs. Contributed by Rui Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b37e3914
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b37e3914
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b37e3914
Branch: refs/heads/YARN-3368
Commit: b37e391490bbae9d85e298d3786d71102d664ae1
Parents: 323db7b
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Apr 20 15:47:01 2016 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:46 2016 -0700
----------------------------------------------------------------------
.../hadoop/fs/CommonConfigurationKeys.java | 19 +++-
.../apache/hadoop/io/erasurecode/CodecUtil.java | 114 +++++++++++--------
.../io/erasurecode/ErasureCodeConstants.java | 40 +++++++
.../erasurecode/coder/HHXORErasureEncoder.java | 2 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 1 +
.../io/erasurecode/coder/RSErasureEncoder.java | 3 +-
.../src/main/resources/core-default.xml | 8 ++
.../erasurecode/TestCodecRawCoderMapping.java | 85 ++++++++++++++
.../hadoop/io/erasurecode/TestCoderBase.java | 2 +-
.../coder/TestHHXORErasureCoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 2 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 2 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 2 +-
.../hdfs/protocol/ErasureCodingPolicy.java | 4 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 5 +-
.../erasurecode/StripedReconstructor.java | 2 +-
.../namenode/ErasureCodingPolicyManager.java | 14 ++-
.../src/site/markdown/HDFSErasureCoding.md | 6 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 3 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +-
.../test/resources/testErasureCodingConf.xml | 12 +-
21 files changed, 255 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index a708900..9f52447 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -21,6 +21,9 @@ package org.apache.hadoop.fs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.http.lib.StaticUserWebFilter;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
+import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
/**
* This class contains constants for configuration keys used
@@ -156,13 +159,23 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
/** Supported erasure codec classes */
public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs";
- /** Raw coder factory for the RS codec. */
- public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
- "io.erasurecode.codec.rs.rawcoder";
+ /** Raw coder factory for the RS default codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY =
+ "io.erasurecode.codec.rs-default.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT =
+ RSRawErasureCoderFactory.class.getCanonicalName();
+
+ /** Raw coder factory for the RS legacy codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY =
+ "io.erasurecode.codec.rs-legacy.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT =
+ RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
/** Raw coder factory for the XOR codec. */
public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY =
"io.erasurecode.codec.xor.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT =
+ XORRawErasureCoderFactory.class.getCanonicalName();
/**
* Service Authorization
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index a2354b6..fcce071 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -17,17 +17,14 @@
*/
package org.apache.hadoop.io.erasurecode;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
-import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
-import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
-import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
/**
* A codec & coder utility to help create raw coders conveniently.
@@ -42,40 +39,56 @@ public final class CodecUtil {
* @param conf configuration possibly with some items to configure the coder
* @param numDataUnits number of data units in a coding group
* @param numParityUnits number of parity units in a coding group
+ * @param codec the codec to use. If null, will use the default codec
* @return raw encoder
*/
public static RawErasureEncoder createRSRawEncoder(
- Configuration conf, int numDataUnits, int numParityUnits) {
- RawErasureCoder rawCoder = createRawCoder(conf,
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
- true, numDataUnits, numParityUnits);
- if (rawCoder == null) {
- rawCoder = new RSRawEncoder(numDataUnits, numParityUnits);
+ Configuration conf, int numDataUnits, int numParityUnits, String codec) {
+ Preconditions.checkNotNull(conf);
+ if (codec == null) {
+ codec = ErasureCodeConstants.RS_DEFAULT_CODEC_NAME;
}
-
+ RawErasureCoder rawCoder = createRawCoder(conf,
+ getFactNameFromCodec(conf, codec), true, numDataUnits, numParityUnits);
return (RawErasureEncoder) rawCoder;
}
/**
+ * Create RS raw encoder using the default codec.
+ */
+ public static RawErasureEncoder createRSRawEncoder(
+ Configuration conf, int numDataUnits, int numParityUnits) {
+ return createRSRawEncoder(conf, numDataUnits, numParityUnits, null);
+ }
+
+ /**
* Create RS raw decoder according to configuration.
* @param conf configuration possibly with some items to configure the coder
* @param numDataUnits number of data units in a coding group
* @param numParityUnits number of parity units in a coding group
+ * @param codec the codec to use. If null, will use the default codec
* @return raw decoder
*/
public static RawErasureDecoder createRSRawDecoder(
- Configuration conf, int numDataUnits, int numParityUnits) {
- RawErasureCoder rawCoder = createRawCoder(conf,
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
- false, numDataUnits, numParityUnits);
- if (rawCoder == null) {
- rawCoder = new RSRawDecoder(numDataUnits, numParityUnits);
+ Configuration conf, int numDataUnits, int numParityUnits, String codec) {
+ Preconditions.checkNotNull(conf);
+ if (codec == null) {
+ codec = ErasureCodeConstants.RS_DEFAULT_CODEC_NAME;
}
-
+ RawErasureCoder rawCoder = createRawCoder(conf,
+ getFactNameFromCodec(conf, codec), false, numDataUnits, numParityUnits);
return (RawErasureDecoder) rawCoder;
}
/**
+ * Create RS raw decoder using the default codec.
+ */
+ public static RawErasureDecoder createRSRawDecoder(
+ Configuration conf, int numDataUnits, int numParityUnits) {
+ return createRSRawDecoder(conf, numDataUnits, numParityUnits, null);
+ }
+
+ /**
* Create XOR raw encoder according to configuration.
* @param conf configuration possibly with some items to configure the coder
* @param numDataUnits number of data units in a coding group
@@ -84,13 +97,10 @@ public final class CodecUtil {
*/
public static RawErasureEncoder createXORRawEncoder(
Configuration conf, int numDataUnits, int numParityUnits) {
+ Preconditions.checkNotNull(conf);
RawErasureCoder rawCoder = createRawCoder(conf,
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
+ getFactNameFromCodec(conf, ErasureCodeConstants.XOR_CODEC_NAME),
true, numDataUnits, numParityUnits);
- if (rawCoder == null) {
- rawCoder = new XORRawEncoder(numDataUnits, numParityUnits);
- }
-
return (RawErasureEncoder) rawCoder;
}
@@ -103,51 +113,65 @@ public final class CodecUtil {
*/
public static RawErasureDecoder createXORRawDecoder(
Configuration conf, int numDataUnits, int numParityUnits) {
+ Preconditions.checkNotNull(conf);
RawErasureCoder rawCoder = createRawCoder(conf,
- CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
+ getFactNameFromCodec(conf, ErasureCodeConstants.XOR_CODEC_NAME),
false, numDataUnits, numParityUnits);
- if (rawCoder == null) {
- rawCoder = new XORRawDecoder(numDataUnits, numParityUnits);
- }
-
return (RawErasureDecoder) rawCoder;
}
/**
* Create raw coder using specified conf and raw coder factory key.
* @param conf configuration possibly with some items to configure the coder
- * @param rawCoderFactoryKey configuration key to find the raw coder factory
+ * @param rawCoderFactory name of the raw coder factory
* @param isEncoder is encoder or not we're going to create
* @param numDataUnits number of data units in a coding group
* @param numParityUnits number of parity units in a coding group
* @return raw coder
*/
public static RawErasureCoder createRawCoder(Configuration conf,
- String rawCoderFactoryKey, boolean isEncoder, int numDataUnits,
+ String rawCoderFactory, boolean isEncoder, int numDataUnits,
int numParityUnits) {
- if (conf == null) {
- return null;
- }
-
- Class<? extends RawErasureCoderFactory> factClass = null;
- factClass = conf.getClass(rawCoderFactoryKey,
- factClass, RawErasureCoderFactory.class);
-
- if (factClass == null) {
- return null;
- }
-
RawErasureCoderFactory fact;
try {
+ Class<? extends RawErasureCoderFactory> factClass = conf.getClassByName(
+ rawCoderFactory).asSubclass(RawErasureCoderFactory.class);
fact = factClass.newInstance();
- } catch (InstantiationException e) {
- throw new RuntimeException("Failed to create raw coder", e);
- } catch (IllegalAccessException e) {
+ } catch (ClassNotFoundException | InstantiationException |
+ IllegalAccessException e) {
throw new RuntimeException("Failed to create raw coder", e);
}
return isEncoder ? fact.createEncoder(numDataUnits, numParityUnits) :
fact.createDecoder(numDataUnits, numParityUnits);
}
+
+ private static String getFactNameFromCodec(Configuration conf, String codec) {
+ switch (codec) {
+ case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
+ return conf.get(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CommonConfigurationKeys.
+ IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT);
+ case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
+ return conf.get(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
+ CommonConfigurationKeys.
+ IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT);
+ case ErasureCodeConstants.XOR_CODEC_NAME:
+ return conf.get(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT);
+ default:
+ // For custom codec, we throw exception if the factory is not configured
+ String rawCoderKey = "io.erasurecode.codec." + codec + ".rawcoder";
+ String factName = conf.get(rawCoderKey);
+ if (factName == null) {
+ throw new IllegalArgumentException("Raw coder factory not configured " +
+ "for custom codec " + codec);
+ }
+ return factName;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
new file mode 100644
index 0000000..1fb9488
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+/**
+ * Constants related to the erasure code feature.
+ */
+public final class ErasureCodeConstants {
+
+ private ErasureCodeConstants(){
+ }
+
+ public static final String RS_DEFAULT_CODEC_NAME = "rs-default";
+ public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
+ public static final String XOR_CODEC_NAME = "xor";
+
+ public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
+ RS_DEFAULT_CODEC_NAME, 6, 3);
+
+ public static final ECSchema RS_3_2_SCHEMA = new ECSchema(
+ RS_DEFAULT_CODEC_NAME, 3, 2);
+
+ public static final ECSchema RS_6_3_LEGACY_SCHEMA = new ECSchema(
+ RS_LEGACY_CODEC_NAME, 6, 3);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
index f30572f..a402469 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
@@ -65,7 +65,7 @@ public class HHXORErasureEncoder extends AbstractErasureEncoder {
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rsRawEncoder == null) {
rsRawEncoder = CodecUtil.createRSRawEncoder(getConf(),
- getNumDataUnits(), getNumParityUnits());
+ getNumDataUnits(), getNumParityUnits());
}
return rsRawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
index 6728945..47efd29 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
@@ -54,6 +54,7 @@ public class RSErasureDecoder extends AbstractErasureDecoder {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
+ // TODO: we should create the raw coder according to codec.
rsRawDecoder = CodecUtil.createRSRawDecoder(getConf(),
getNumDataUnits(), getNumParityUnits());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
index 7784573..4806d9e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
@@ -54,8 +54,9 @@ public class RSErasureEncoder extends AbstractErasureEncoder {
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rawEncoder == null) {
+ // TODO: we should create the raw coder according to codec.
rawEncoder = CodecUtil.createRSRawEncoder(getConf(),
- getNumDataUnits(), getNumParityUnits());
+ getNumDataUnits(), getNumParityUnits());
}
return rawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 96b108f..ac8bc5a 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -506,6 +506,14 @@
</description>
</property>
+<property>
+ <name>io.erasurecode.codec.rs-default.rawcoder</name>
+ <value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
+ <description>
+ Raw coder implementation for the rs-default codec.
+ </description>
+</property>
+
<!-- file system properties -->
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
new file mode 100644
index 0000000..5075966
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoderLegacy;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test the codec to raw coder mapping.
+ */
+public class TestCodecRawCoderMapping {
+
+ private static Configuration conf;
+ private static final int numDataUnit = 6;
+ private static final int numParityUnit = 3;
+
+ @Before
+ public void setup() {
+ conf = new Configuration();
+ }
+
+ @Test
+ public void testRSDefaultRawCoder() {
+ // should return default raw coder of rs-default codec
+ RawErasureEncoder encoder = CodecUtil.createRSRawEncoder(
+ conf, numDataUnit, numParityUnit);
+ Assert.assertTrue(encoder instanceof RSRawEncoder);
+ RawErasureDecoder decoder = CodecUtil.createRSRawDecoder(
+ conf, numDataUnit, numParityUnit);
+ Assert.assertTrue(decoder instanceof RSRawDecoder);
+
+ // should return default raw coder of rs-legacy codec
+ encoder = CodecUtil.createRSRawEncoder(conf, numDataUnit, numParityUnit,
+ ErasureCodeConstants.RS_LEGACY_CODEC_NAME);
+ Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
+ decoder = CodecUtil.createRSRawDecoder(conf, numDataUnit, numParityUnit,
+ ErasureCodeConstants.RS_LEGACY_CODEC_NAME);
+ Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
+ }
+
+ @Test
+ public void testDedicatedRawCoderKey() {
+ String dummyFactName = "DummyNoneExistingFactory";
+ // set the dummy factory to rs-legacy and create a raw coder
+ // with rs-default, which is OK as the raw coder key is not used
+ conf.set(CommonConfigurationKeys.
+ IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName);
+ RawErasureEncoder encoder = CodecUtil.createRSRawEncoder(conf, numDataUnit,
+ numParityUnit, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME);
+ Assert.assertTrue(encoder instanceof RSRawEncoder);
+ // now create the raw coder with rs-legacy, which should throw exception
+ try {
+ CodecUtil.createRSRawEncoder(conf, numDataUnit, numParityUnit,
+ ErasureCodeConstants.RS_LEGACY_CODEC_NAME);
+ Assert.fail();
+ } catch (Exception e) {
+ GenericTestUtils.assertExceptionContains("Failed to create raw coder", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
index 5721336..633e064 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
@@ -112,7 +112,7 @@ public abstract class TestCoderBase {
protected void prepare(Configuration conf, int numDataUnits,
int numParityUnits, int[] erasedDataIndexes,
int[] erasedParityIndexes, boolean usingFixedData) {
- this.conf = conf;
+ this.conf = conf != null ? conf : new Configuration();
this.numDataUnits = numDataUnits;
this.numParityUnits = numParityUnits;
this.erasedDataIndexes = erasedDataIndexes != null ?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
index ad346e0..09d1ec2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
@@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
index ee2348e..b0fe8f8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
@@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 38236ad..1944782 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -185,7 +185,7 @@ public class DFSStripedInputStream extends DFSInputStream {
readingService =
new ExecutorCompletionService<>(dfsClient.getStripedReadsThreadPool());
decoder = CodecUtil.createRSRawDecoder(dfsClient.getConfiguration(),
- dataBlkNum, parityBlkNum);
+ dataBlkNum, parityBlkNum, ecPolicy.getCodecName());
if (DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug("Creating an striped input stream for file " + src);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 3dfdbee..b049286 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -287,7 +287,7 @@ public class DFSStripedOutputStream extends DFSOutputStream {
ExecutorCompletionService<>(flushAllExecutor);
encoder = CodecUtil.createRSRawEncoder(dfsClient.getConfiguration(),
- numDataBlocks, numParityBlocks);
+ numDataBlocks, numParityBlocks, ecPolicy.getCodecName());
coordinator = new Coordinator(numAllBlocks);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
index 2c748d2..c320fdf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
@@ -72,6 +72,10 @@ public final class ErasureCodingPolicy {
return schema.getNumParityUnits();
}
+ public String getCodecName() {
+ return schema.getCodecName();
+ }
+
public byte getId() {
return id;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 3c5c441..8df2d54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -22,7 +22,6 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.util.StringUtils;
@InterfaceAudience.Private
@@ -145,11 +144,9 @@ public final class HdfsConstants {
ALL, LIVE, DEAD, DECOMMISSIONING
}
- public static final ECSchema RS_6_3_SCHEMA = new ECSchema("rs", 6, 3);
public static final byte RS_6_3_POLICY_ID = 0;
-
- public static final ECSchema RS_3_2_SCHEMA = new ECSchema("rs", 3, 2);
public static final byte RS_3_2_POLICY_ID = 1;
+ public static final byte RS_6_3_LEGACY_POLICY_ID = 2;
/* Hidden constructor */
protected HdfsConstants() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
index a0a5f83..1b59b22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
@@ -216,7 +216,7 @@ class StripedReconstructor implements Runnable {
private void initDecoderIfNecessary() {
if (decoder == null) {
decoder = CodecUtil.createRSRawDecoder(conf, ecPolicy.getNumDataUnits(),
- ecPolicy.getNumParityUnits());
+ ecPolicy.getNumParityUnits(), ecPolicy.getCodecName());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index eee80d3..eaf63f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import java.util.Map;
import java.util.TreeMap;
@@ -39,15 +40,18 @@ public final class ErasureCodingPolicyManager {
*/
private static final int DEFAULT_CELLSIZE = 64 * 1024;
private static final ErasureCodingPolicy SYS_POLICY1 =
- new ErasureCodingPolicy(HdfsConstants.RS_6_3_SCHEMA, DEFAULT_CELLSIZE,
- HdfsConstants.RS_6_3_POLICY_ID);
+ new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_SCHEMA,
+ DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_POLICY_ID);
private static final ErasureCodingPolicy SYS_POLICY2 =
- new ErasureCodingPolicy(HdfsConstants.RS_3_2_SCHEMA, DEFAULT_CELLSIZE,
- HdfsConstants.RS_3_2_POLICY_ID);
+ new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA,
+ DEFAULT_CELLSIZE, HdfsConstants.RS_3_2_POLICY_ID);
+ private static final ErasureCodingPolicy SYS_POLICY3 =
+ new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_LEGACY_SCHEMA,
+ DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_LEGACY_POLICY_ID);
//We may add more later.
private static final ErasureCodingPolicy[] SYS_POLICIES =
- new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2};
+ new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2, SYS_POLICY3};
/**
* All active policies maintained in NN memory for fast querying,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 5b3aa34..9066a15 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -96,8 +96,10 @@ Deployment
### Configuration keys
The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys:
- `io.erasurecode.codec.rs.rawcoder` and `io.erasurecode.codec.xor.rawcoder`.
- The default implementations for both of these codecs are pure Java.
+ `io.erasurecode.codec.rs-default.rawcoder` for the default RS codec,
+ `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec,
+ `io.erasurecode.codec.xor.rawcoder` for the XOR codec.
+ The default implementations for all of these codecs are pure Java.
Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
index 6d0dfa8..ccc94a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
@@ -491,7 +491,8 @@ public class StripedFileTestUtil {
}
}
final RawErasureEncoder encoder =
- CodecUtil.createRSRawEncoder(conf, dataBytes.length, parityBytes.length);
+ CodecUtil.createRSRawEncoder(conf, dataBytes.length, parityBytes.length,
+ TEST_EC_POLICY.getCodecName());
encoder.encode(dataBytes, expectedParityBytes);
for (int i = 0; i < parityBytes.length; i++) {
if (checkSet.contains(i + dataBytes.length)){
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 4dcb654..e4f7ac0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -218,7 +218,7 @@ public class TestDFSStripedInputStream {
}
RawErasureDecoder rawDecoder = CodecUtil.createRSRawDecoder(conf,
- DATA_BLK_NUM, PARITY_BLK_NUM);
+ DATA_BLK_NUM, PARITY_BLK_NUM, ecPolicy.getCodecName());
// Update the expected content for decoded data
int[] missingBlkIdx = new int[PARITY_BLK_NUM];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b37e3914/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index fd7cf9d..1c782c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -109,7 +109,7 @@
<description>setPolicy : set erasure coding policy on a directory to encode files</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -179,7 +179,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
@@ -188,7 +188,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-6-3-64k</expected-output>
+ <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -197,7 +197,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy /ecdir/ecfile</ec-admin-command>
</test-commands>
@@ -208,7 +208,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-6-3-64k</expected-output>
+ <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -223,7 +223,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-6-3</expected-output>
+ <expected-output>RS-DEFAULT-6-3</expected-output>
</comparator>
</comparators>
</test>
[06/13] hadoop git commit: YARN-4934. Reserved Resource for
QueueMetrics needs to be handled correctly in few cases. (Sunil G via wangda)
Posted by wa...@apache.org.
YARN-4934. Reserved Resource for QueueMetrics needs to be handled correctly in few cases. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/931cd285
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/931cd285
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/931cd285
Branch: refs/heads/YARN-3368
Commit: 931cd2856433e3ac6447e80bfa7ad94e4b651c9c
Parents: 4f58a8b
Author: Wangda Tan <wa...@apache.org>
Authored: Sat Apr 16 22:47:41 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:45 2016 -0700
----------------------------------------------------------------------
.../scheduler/capacity/LeafQueue.java | 7 -
.../scheduler/common/fica/FiCaSchedulerApp.java | 2 +
.../capacity/TestContainerAllocation.java | 188 ++++++++++++++++++-
3 files changed, 189 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/931cd285/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index aabdf9c..fbcb91c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1348,13 +1348,6 @@ public class LeafQueue extends AbstractCSQueue {
// Book-keeping
if (removed) {
- // track reserved resource for metrics, for normal container
- // getReservedResource will be null.
- Resource reservedRes = rmContainer.getReservedResource();
- if (reservedRes != null && !reservedRes.equals(Resources.none())) {
- decReservedResource(node.getPartition(), reservedRes);
- }
-
// Inform the ordering policy
orderingPolicy.containerReleased(application, rmContainer);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/931cd285/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index f474aad..35329d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -246,6 +246,8 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
// Update reserved metrics
queue.getMetrics().unreserveResource(getUser(),
rmContainer.getReservedResource());
+ queue.decReservedResource(node.getPartition(),
+ rmContainer.getReservedResource());
return true;
}
return false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/931cd285/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 84eba10..f94c963 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.security.SecurityUtilTestHelper;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -37,6 +38,8 @@ import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ContainerType;
+import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -50,8 +53,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -417,5 +425,183 @@ public class TestContainerAllocation {
rm1.close();
}
-
+
+ @Test(timeout = 60000)
+ public void testAllocationForReservedContainer() throws Exception {
+ /**
+ * Test case: Submit two application (app1/app2) to a queue. And there's one
+ * node with 8G resource in the cluster. App1 allocates a 6G container, Then
+ * app2 asks for a 4G container. App2's request will be reserved on the
+ * node.
+ *
+ * Before next node heartbeat, app1 container is completed/killed. So app1
+ * container which was reserved will be allocated.
+ */
+ // inject node label manager
+ MockRM rm1 = new MockRM();
+
+ rm1.getRMContext().setNodeLabelManager(mgr);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+
+ // launch an app to queue, AM container should be launched in nm1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // launch another app to queue, AM container should be launched in nm1
+ RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+ am1.allocate("*", 4 * GB, 1, new ArrayList<ContainerId>());
+ am2.allocate("*", 4 * GB, 1, new ArrayList<ContainerId>());
+
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ LeafQueue leafQueue = (LeafQueue) cs.getQueue("default");
+
+ // Do node heartbeats 2 times
+ // First time will allocate container for app1, second time will reserve
+ // container for app2
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ // App2 will get preference to be allocated on node1, and node1 will be all
+ // used by App2.
+ FiCaSchedulerApp schedulerApp1 =
+ cs.getApplicationAttempt(am1.getApplicationAttemptId());
+ FiCaSchedulerApp schedulerApp2 =
+ cs.getApplicationAttempt(am2.getApplicationAttemptId());
+
+ // Check if a 4G container allocated for app1, and nothing allocated for app2
+ Assert.assertEquals(2, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(1, schedulerApp2.getLiveContainers().size());
+ Assert.assertTrue(schedulerApp2.getReservedContainers().size() > 0);
+
+ // NM1 has available resource = 2G (8G - 2 * 1G - 4G)
+ Assert.assertEquals(2 * GB, cs.getNode(nm1.getNodeId())
+ .getUnallocatedResource().getMemory());
+ Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ // Usage of queue = 4G + 2 * 1G + 4G (reserved)
+ Assert.assertEquals(10 * GB, cs.getRootQueue().getQueueResourceUsage()
+ .getUsed().getMemory());
+ Assert.assertEquals(4 * GB, cs.getRootQueue().getQueueResourceUsage()
+ .getReserved().getMemory());
+ Assert.assertEquals(4 * GB, leafQueue.getQueueResourceUsage().getReserved()
+ .getMemory());
+
+ // Mark one app1 container as killed/completed and re-kick RM
+ for (RMContainer container : schedulerApp1.getLiveContainers()) {
+ if (container.isAMContainer()) {
+ continue;
+ }
+ cs.markContainerForKillable(container);
+ }
+ // Cancel asks of app1 and re-kick RM
+ am1.allocate("*", 4 * GB, 0, new ArrayList<ContainerId>());
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ // Check 4G container cancelled for app1, and one container allocated for
+ // app2
+ Assert.assertEquals(1, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(2, schedulerApp2.getLiveContainers().size());
+ Assert.assertFalse(schedulerApp2.getReservedContainers().size() > 0);
+
+ // NM1 has available resource = 2G (8G - 2 * 1G - 4G)
+ Assert.assertEquals(2 * GB, cs.getNode(nm1.getNodeId())
+ .getUnallocatedResource().getMemory());
+ Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ // Usage of queue = 4G + 2 * 1G
+ Assert.assertEquals(6 * GB, cs.getRootQueue().getQueueResourceUsage()
+ .getUsed().getMemory());
+ Assert.assertEquals(0 * GB, cs.getRootQueue().getQueueResourceUsage()
+ .getReserved().getMemory());
+ Assert.assertEquals(0 * GB, leafQueue.getQueueResourceUsage().getReserved()
+ .getMemory());
+
+ rm1.close();
+ }
+
+ @Test(timeout = 60000)
+ public void testReservedContainerMetricsOnDecommisionedNode() throws Exception {
+ /**
+ * Test case: Submit two application (app1/app2) to a queue. And there's one
+ * node with 8G resource in the cluster. App1 allocates a 6G container, Then
+ * app2 asks for a 4G container. App2's request will be reserved on the
+ * node.
+ *
+ * Before next node heartbeat, app1 container is completed/killed. So app1
+ * container which was reserved will be allocated.
+ */
+ // inject node label manager
+ MockRM rm1 = new MockRM();
+
+ rm1.getRMContext().setNodeLabelManager(mgr);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+
+ // launch an app to queue, AM container should be launched in nm1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // launch another app to queue, AM container should be launched in nm1
+ RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+ am1.allocate("*", 4 * GB, 1, new ArrayList<ContainerId>());
+ am2.allocate("*", 4 * GB, 1, new ArrayList<ContainerId>());
+
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ LeafQueue leafQueue = (LeafQueue) cs.getQueue("default");
+
+ // Do node heartbeats 2 times
+ // First time will allocate container for app1, second time will reserve
+ // container for app2
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ // App2 will get preference to be allocated on node1, and node1 will be all
+ // used by App2.
+ FiCaSchedulerApp schedulerApp1 =
+ cs.getApplicationAttempt(am1.getApplicationAttemptId());
+ FiCaSchedulerApp schedulerApp2 =
+ cs.getApplicationAttempt(am2.getApplicationAttemptId());
+
+ // Check if a 4G container allocated for app1, and nothing allocated for app2
+ Assert.assertEquals(2, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(1, schedulerApp2.getLiveContainers().size());
+ Assert.assertTrue(schedulerApp2.getReservedContainers().size() > 0);
+
+ // NM1 has available resource = 2G (8G - 2 * 1G - 4G)
+ Assert.assertEquals(2 * GB, cs.getNode(nm1.getNodeId())
+ .getUnallocatedResource().getMemory());
+ Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ // Usage of queue = 4G + 2 * 1G + 4G (reserved)
+ Assert.assertEquals(10 * GB, cs.getRootQueue().getQueueResourceUsage()
+ .getUsed().getMemory());
+ Assert.assertEquals(4 * GB, cs.getRootQueue().getQueueResourceUsage()
+ .getReserved().getMemory());
+ Assert.assertEquals(4 * GB, leafQueue.getQueueResourceUsage().getReserved()
+ .getMemory());
+
+ // Remove the node
+ cs.handle(new NodeRemovedSchedulerEvent(rmNode1));
+
+ // Check all container cancelled for app1 and app2
+ Assert.assertEquals(0, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(0, schedulerApp2.getLiveContainers().size());
+ Assert.assertFalse(schedulerApp2.getReservedContainers().size() > 0);
+
+ // Usage and Reserved capacity of queue is 0
+ Assert.assertEquals(0 * GB, cs.getRootQueue().getQueueResourceUsage()
+ .getUsed().getMemory());
+ Assert.assertEquals(0 * GB, cs.getRootQueue().getQueueResourceUsage()
+ .getReserved().getMemory());
+ Assert.assertEquals(0 * GB, leafQueue.getQueueResourceUsage().getReserved()
+ .getMemory());
+
+ rm1.close();
+ }
}
[02/13] hadoop git commit: YARN-4965. Distributed shell AM failed due
to ClientHandlerException thrown by jersey. Contributed by Junping Du
Posted by wa...@apache.org.
YARN-4965. Distributed shell AM failed due to ClientHandlerException thrown by jersey. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f58a8b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f58a8b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f58a8b4
Branch: refs/heads/YARN-3368
Commit: 4f58a8b4f06dabf59337f62b9b7ae8ceb80bbc2f
Parents: 418f7f7
Author: Xuan <xg...@apache.org>
Authored: Sat Apr 16 19:39:18 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:45 2016 -0700
----------------------------------------------------------------------
.../jobhistory/JobHistoryEventHandler.java | 9 ++--
.../pom.xml | 6 +++
.../distributedshell/ApplicationMaster.java | 10 ++--
.../distributedshell/TestDistributedShell.java | 50 ++++++++++++++++++++
.../client/api/impl/TestTimelineClient.java | 2 +-
5 files changed, 67 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f58a8b4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 5690743..47d2389 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -77,6 +77,8 @@ import org.codehaus.jackson.node.JsonNodeFactory;
import org.codehaus.jackson.node.ObjectNode;
import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.api.client.ClientHandlerException;
+
/**
* The job history events get routed to this class. This class writes the Job
* history events to the DFS directly into a staging dir and then moved to a
@@ -1032,12 +1034,9 @@ public class JobHistoryEventHandler extends AbstractService
+ error.getErrorCode());
}
}
- } catch (IOException ex) {
- LOG.error("Error putting entity " + tEntity.getEntityId() + " to Timeline"
- + "Server", ex);
- } catch (YarnException ex) {
+ } catch (YarnException | IOException | ClientHandlerException ex) {
LOG.error("Error putting entity " + tEntity.getEntityId() + " to Timeline"
- + "Server", ex);
+ + "Server", ex);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f58a8b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index c118603..dba8fc0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -133,6 +133,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f58a8b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index cbe0348..2b85ba8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.log4j.LogManager;
import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.api.client.ClientHandlerException;
/**
* An ApplicationMaster for executing shell commands on a set of launched
@@ -1149,13 +1150,14 @@ public class ApplicationMaster {
putContainerEntity(timelineClient,
container.getId().getApplicationAttemptId(),
entity));
- } catch (YarnException | IOException e) {
+ } catch (YarnException | IOException | ClientHandlerException e) {
LOG.error("Container start event could not be published for "
+ container.getId().toString(), e);
}
}
- private void publishContainerEndEvent(
+ @VisibleForTesting
+ void publishContainerEndEvent(
final TimelineClient timelineClient, ContainerStatus container,
String domainId, UserGroupInformation ugi) {
final TimelineEntity entity = new TimelineEntity();
@@ -1177,7 +1179,7 @@ public class ApplicationMaster {
putContainerEntity(timelineClient,
container.getContainerId().getApplicationAttemptId(),
entity));
- } catch (YarnException | IOException e) {
+ } catch (YarnException | IOException | ClientHandlerException e) {
LOG.error("Container end event could not be published for "
+ container.getContainerId().toString(), e);
}
@@ -1212,7 +1214,7 @@ public class ApplicationMaster {
try {
TimelinePutResponse response = timelineClient.putEntities(entity);
processTimelineResponseErrors(response);
- } catch (YarnException | IOException e) {
+ } catch (YarnException | IOException | ClientHandlerException e) {
LOG.error("App Attempt "
+ (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
+ " event could not be published for "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f58a8b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 6536050..2b46fca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.yarn.applications.distributedshell;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -27,6 +31,7 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.InetAddress;
+import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
@@ -46,14 +51,24 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.ServerSocketUtil;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
+import org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster;
+import org.apache.hadoop.yarn.client.api.impl.DirectTimelineWriter;
+import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
+import org.apache.hadoop.yarn.client.api.impl.TimelineWriter;
+import org.apache.hadoop.yarn.client.api.impl.TestTimelineClient;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
+
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -61,6 +76,7 @@ import org.apache.hadoop.yarn.server.timeline.PluginStoreTestUtils;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.apache.hadoop.yarn.server.timeline.TimelineVersion;
import org.apache.hadoop.yarn.server.timeline.TimelineVersionWatcher;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
import org.junit.Assert;
@@ -69,6 +85,8 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
+import com.sun.jersey.api.client.ClientHandlerException;
+
public class TestDistributedShell {
private static final Log LOG =
@@ -77,6 +95,7 @@ public class TestDistributedShell {
protected MiniYARNCluster yarnCluster = null;
protected MiniDFSCluster hdfsCluster = null;
private FileSystem fs = null;
+ private TimelineWriter spyTimelineWriter;
protected YarnConfiguration conf = null;
private static final int NUM_NMS = 1;
private static final float DEFAULT_TIMELINE_VERSION = 1.0f;
@@ -865,6 +884,37 @@ public class TestDistributedShell {
}
}
+ @Test
+ public void testDSTimelineClientWithConnectionRefuse() throws Exception {
+ ApplicationMaster am = new ApplicationMaster();
+
+ TimelineClientImpl client = new TimelineClientImpl() {
+ @Override
+ protected TimelineWriter createTimelineWriter(Configuration conf,
+ UserGroupInformation authUgi, com.sun.jersey.api.client.Client client,
+ URI resURI) throws IOException {
+ TimelineWriter timelineWriter =
+ new DirectTimelineWriter(authUgi, client, resURI);
+ spyTimelineWriter = spy(timelineWriter);
+ return spyTimelineWriter;
+ }
+ };
+ client.init(conf);
+ client.start();
+ TestTimelineClient.mockEntityClientResponse(spyTimelineWriter, null,
+ false, true);
+ try {
+ UserGroupInformation ugi = mock(UserGroupInformation.class);
+ when(ugi.getShortUserName()).thenReturn("user1");
+ // verify no ClientHandlerException get thrown out.
+ am.publishContainerEndEvent(client, ContainerStatus.newInstance(
+ BuilderUtils.newContainerId(1, 1, 1, 1), ContainerState.COMPLETE, "",
+ 1), "domainId", ugi);
+ } finally {
+ client.stop();
+ }
+ }
+
protected void waitForNMsToRegister() throws Exception {
int sec = 60;
while (sec >= 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f58a8b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
index 39fc8de..d5e186c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
@@ -298,7 +298,7 @@ public class TestTimelineClient {
client.connectionRetry.getRetired());
}
- private static ClientResponse mockEntityClientResponse(
+ public static ClientResponse mockEntityClientResponse(
TimelineWriter spyTimelineWriter, ClientResponse.Status status,
boolean hasError, boolean hasRuntimeError) {
ClientResponse response = mock(ClientResponse.class);
[07/13] hadoop git commit: Fixed TimelineClient to retry
SocketTimeoutException too. Contributed by Xuan Gong.
Posted by wa...@apache.org.
Fixed TimelineClient to retry SocketTimeoutException too. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38aed9d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38aed9d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38aed9d8
Branch: refs/heads/YARN-3368
Commit: 38aed9d868f9aa6ecd0f8bd33e26ea06ba5ca90d
Parents: 1bec4cb
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Mon Apr 18 11:47:06 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:46 2016 -0700
----------------------------------------------------------------------
.../client/api/impl/TimelineClientImpl.java | 74 ++++++++++++++------
.../client/api/impl/TestTimelineClient.java | 41 +++++++++++
2 files changed, 93 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38aed9d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index ef46229..8c60041 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -24,6 +24,7 @@ import java.lang.reflect.UndeclaredThrowableException;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
@@ -116,7 +117,9 @@ public class TimelineClientImpl extends TimelineClient {
TimelineClientConnectionRetry connectionRetry;
// Abstract class for an operation that should be retried by timeline client
- private static abstract class TimelineClientRetryOp {
+ @Private
+ @VisibleForTesting
+ public static abstract class TimelineClientRetryOp {
// The operation that should be retried
public abstract Object run() throws IOException;
// The method to indicate if we should retry given the incoming exception
@@ -449,27 +452,8 @@ public class TimelineClientImpl extends TimelineClient {
final PrivilegedExceptionAction<?> action)
throws IOException, YarnException {
// Set up the retry operation
- TimelineClientRetryOp tokenRetryOp = new TimelineClientRetryOp() {
-
- @Override
- public Object run() throws IOException {
- // Try pass the request, if fail, keep retrying
- authUgi.checkTGTAndReloginFromKeytab();
- try {
- return authUgi.doAs(action);
- } catch (UndeclaredThrowableException e) {
- throw new IOException(e.getCause());
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public boolean shouldRetryOn(Exception e) {
- // Only retry on connection exceptions
- return (e instanceof ConnectException);
- }
- };
+ TimelineClientRetryOp tokenRetryOp =
+ createTimelineClientRetryOpForOperateDelegationToken(action);
return connectionRetry.retryOn(tokenRetryOp);
}
@@ -680,4 +664,50 @@ public class TimelineClientImpl extends TimelineClient {
public void setTimelineWriter(TimelineWriter writer) {
this.timelineWriter = writer;
}
+
+ @Private
+ @VisibleForTesting
+ public TimelineClientRetryOp
+ createTimelineClientRetryOpForOperateDelegationToken(
+ final PrivilegedExceptionAction<?> action) throws IOException {
+ return new TimelineClientRetryOpForOperateDelegationToken(
+ this.authUgi, action);
+ }
+
+ @Private
+ @VisibleForTesting
+ public class TimelineClientRetryOpForOperateDelegationToken
+ extends TimelineClientRetryOp {
+
+ private final UserGroupInformation authUgi;
+ private final PrivilegedExceptionAction<?> action;
+
+ public TimelineClientRetryOpForOperateDelegationToken(
+ UserGroupInformation authUgi, PrivilegedExceptionAction<?> action) {
+ this.authUgi = authUgi;
+ this.action = action;
+ }
+
+ @Override
+ public Object run() throws IOException {
+ // Try pass the request, if fail, keep retrying
+ authUgi.checkTGTAndReloginFromKeytab();
+ try {
+ return authUgi.doAs(action);
+ } catch (UndeclaredThrowableException e) {
+ throw new IOException(e.getCause());
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public boolean shouldRetryOn(Exception e) {
+ // retry on connection exceptions
+ // and SocketTimeoutException
+ return (e instanceof ConnectException
+ || e instanceof SocketTimeoutException);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38aed9d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
index d5e186c..41b788d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
@@ -27,7 +27,9 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.ConnectException;
+import java.net.SocketTimeoutException;
import java.net.URI;
+import java.security.PrivilegedExceptionAction;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -234,6 +236,8 @@ public class TestTimelineClient {
UserGroupInformation.setConfiguration(conf);
TimelineClientImpl client = createTimelineClient(conf);
+ TimelineClientImpl clientFake =
+ createTimelineClientFakeTimelineClientRetryOp(conf);
TestTimlineDelegationTokenSecretManager dtManager =
new TestTimlineDelegationTokenSecretManager();
try {
@@ -278,8 +282,24 @@ public class TestTimelineClient {
} catch (RuntimeException ce) {
assertException(client, ce);
}
+
+ // Test DelegationTokenOperationsRetry on SocketTimeoutException
+ try {
+ TimelineDelegationTokenIdentifier timelineDT =
+ new TimelineDelegationTokenIdentifier(
+ new Text("tester"), new Text("tester"), new Text("tester"));
+ clientFake.cancelDelegationToken(
+ new Token<TimelineDelegationTokenIdentifier>(timelineDT.getBytes(),
+ dtManager.createPassword(timelineDT),
+ timelineDT.getKind(),
+ new Text("0.0.0.0:8188")));
+ assertFail();
+ } catch (RuntimeException ce) {
+ assertException(clientFake, ce);
+ }
} finally {
client.stop();
+ clientFake.stop();
dtManager.stopThreads();
}
}
@@ -393,6 +413,27 @@ public class TestTimelineClient {
return client;
}
+ private TimelineClientImpl createTimelineClientFakeTimelineClientRetryOp(
+ YarnConfiguration conf) {
+ TimelineClientImpl client = new TimelineClientImpl() {
+
+ @Override
+ public TimelineClientRetryOp
+ createTimelineClientRetryOpForOperateDelegationToken(
+ final PrivilegedExceptionAction<?> action) throws IOException {
+ TimelineClientRetryOpForOperateDelegationToken op =
+ spy(new TimelineClientRetryOpForOperateDelegationToken(
+ UserGroupInformation.getCurrentUser(), action));
+ doThrow(new SocketTimeoutException("Test socketTimeoutException"))
+ .when(op).run();
+ return op;
+ }
+ };
+ client.init(conf);
+ client.start();
+ return client;
+ }
+
private static class TestTimlineDelegationTokenSecretManager extends
AbstractDelegationTokenSecretManager<TimelineDelegationTokenIdentifier> {
[08/13] hadoop git commit: HDFS-10291 TestShortCircuitLocalRead
failing (stevel)
Posted by wa...@apache.org.
HDFS-10291 TestShortCircuitLocalRead failing (stevel)
(cherry picked from commit dd64769699ea4c85abdf9c590b9db680837e8b27)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/11e53d25
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/11e53d25
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/11e53d25
Branch: refs/heads/YARN-3368
Commit: 11e53d259caaa349a2b3f84b05d6032107dc60a9
Parents: b37e391
Author: Steve Loughran <st...@apache.org>
Authored: Mon Apr 18 10:26:35 2016 +0100
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:46 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e53d25/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
index f4fbebc..116dc88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
@@ -153,8 +153,11 @@ public class TestShortCircuitLocalRead {
//Read a small number of bytes first.
int nread = stm.read(actual, 0, 3);
nread += stm.read(actual, nread, 2);
- //Read across chunk boundary
- nread += stm.read(actual, nread, 517);
+ int len = 517;
+ if (actual.length - nread >= len) {
+ //Read across chunk boundary
+ nread += stm.read(actual, nread, len);
+ }
checkData(actual, readOffset, expected, nread, "A few bytes");
//Now read rest of it
while (nread < actual.length) {
[03/13] hadoop git commit: MAPREDUCE-6649. getFailureInfo not
returning any failure info. Contributed by Eric Badger
Posted by wa...@apache.org.
MAPREDUCE-6649. getFailureInfo not returning any failure info. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/418f7f7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/418f7f7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/418f7f7c
Branch: refs/heads/YARN-3368
Commit: 418f7f7c3a0dd92dd6ea062759e084c081bca936
Parents: d296832
Author: Eric Payne <ep...@apache.org>
Authored: Sat Apr 16 22:07:27 2016 +0000
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:45 2016 -0700
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/hs/CompletedJob.java | 9 ++++++-
.../mapreduce/v2/hs/TestJobHistoryEntities.java | 27 ++++++++++++++++++++
2 files changed, 35 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/418f7f7c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
index ad4e6bc..4deb9ae 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
@@ -142,6 +142,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
report.setFinishTime(jobInfo.getFinishTime());
report.setJobName(jobInfo.getJobname());
report.setUser(jobInfo.getUsername());
+ report.setDiagnostics(jobInfo.getErrorInfo());
if ( getTotalMaps() == 0 ) {
report.setMapProgress(1.0f);
@@ -335,6 +336,12 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
}
}
+ protected JobHistoryParser createJobHistoryParser(Path historyFileAbsolute)
+ throws IOException {
+ return new JobHistoryParser(historyFileAbsolute.getFileSystem(conf),
+ historyFileAbsolute);
+ }
+
//History data is leisurely loaded when task level data is requested
protected synchronized void loadFullHistoryData(boolean loadTasks,
Path historyFileAbsolute) throws IOException {
@@ -347,7 +354,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
JobHistoryParser parser = null;
try {
final FileSystem fs = historyFileAbsolute.getFileSystem(conf);
- parser = new JobHistoryParser(fs, historyFileAbsolute);
+ parser = createJobHistoryParser(historyFileAbsolute);
final Path jobConfPath = new Path(historyFileAbsolute.getParent(),
JobHistoryUtils.getIntermediateConfFileName(jobId));
final Configuration conf = new Configuration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/418f7f7c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
index 9608fc8..c6ddae5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
@@ -19,14 +19,18 @@ package org.apache.hadoop.mapreduce.v2.hs;
import static org.junit.Assert.assertEquals;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobACLsManager;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
@@ -236,4 +240,27 @@ public class TestJobHistoryEntities {
}
+ @Test (timeout=30000)
+ public void testCompletedJobWithDiagnostics() throws Exception {
+ final String jobError = "Job Diagnostics";
+ JobInfo jobInfo = spy(new JobInfo());
+ when(jobInfo.getErrorInfo()).thenReturn(jobError);
+ when(jobInfo.getJobStatus()).thenReturn(JobState.FAILED.toString());
+ when(jobInfo.getAMInfos()).thenReturn(Collections.<JobHistoryParser.AMInfo>emptyList());
+ final JobHistoryParser mockParser = mock(JobHistoryParser.class);
+ when(mockParser.parse()).thenReturn(jobInfo);
+ HistoryFileInfo info = mock(HistoryFileInfo.class);
+ when(info.getConfFile()).thenReturn(fullConfPath);
+ when(info.getHistoryFile()).thenReturn(fullHistoryPath);
+ CompletedJob job =
+ new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
+ info, jobAclsManager) {
+ @Override
+ protected JobHistoryParser createJobHistoryParser(
+ Path historyFileAbsolute) throws IOException {
+ return mockParser;
+ }
+ };
+ assertEquals(jobError, job.getReport().getDiagnostics());
+ }
}
[05/13] hadoop git commit: HDFS-9412. getBlocks occupies FSLock and
takes too long to complete. Contributed by He Tianyi.
Posted by wa...@apache.org.
HDFS-9412. getBlocks occupies FSLock and takes too long to complete. Contributed by He Tianyi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22855e6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22855e6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22855e6f
Branch: refs/heads/YARN-3368
Commit: 22855e6f2a6d6a4656da0802891b35323591ef71
Parents: 931cd28
Author: Walter Su <wa...@apache.org>
Authored: Mon Apr 18 09:28:02 2016 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:45 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/blockmanagement/BlockManager.java | 17 +++++++++++++++++
.../java/org/apache/hadoop/hdfs/TestGetBlocks.java | 8 ++++++--
2 files changed, 23 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22855e6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 104d723..8b50ef8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -305,6 +305,14 @@ public class BlockManager implements BlockStatsMXBean {
* processed again after aquiring lock again.
*/
private int numBlocksPerIteration;
+
+ /**
+ * Minimum size that a block can be sent to Balancer through getBlocks.
+ * And after HDFS-8824, the small blocks are unused anyway, so there's no
+ * point to send them to balancer.
+ */
+ private long getBlocksMinBlockSize = -1;
+
/**
* Progress of the Reconstruction queues initialisation.
*/
@@ -414,6 +422,9 @@ public class BlockManager implements BlockStatsMXBean {
this.numBlocksPerIteration = conf.getInt(
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT,
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT);
+ this.getBlocksMinBlockSize = conf.getLongBytes(
+ DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY,
+ DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_DEFAULT);
this.blockReportLeaseManager = new BlockReportLeaseManager(conf);
bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, conf);
@@ -1179,6 +1190,9 @@ public class BlockManager implements BlockStatsMXBean {
while(totalSize<size && iter.hasNext()) {
curBlock = iter.next();
if(!curBlock.isComplete()) continue;
+ if (curBlock.getNumBytes() < getBlocksMinBlockSize) {
+ continue;
+ }
totalSize += addBlock(curBlock, results);
}
if(totalSize<size) {
@@ -1186,6 +1200,9 @@ public class BlockManager implements BlockStatsMXBean {
for(int i=0; i<startBlock&&totalSize<size; i++) {
curBlock = iter.next();
if(!curBlock.isComplete()) continue;
+ if (curBlock.getNumBytes() < getBlocksMinBlockSize) {
+ continue;
+ }
totalSize += addBlock(curBlock, results);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22855e6f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
index 741e641..6e4b0f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
@@ -179,11 +179,15 @@ public class TestGetBlocks {
final int DEFAULT_BLOCK_SIZE = 1024;
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
+ CONF.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY,
+ DEFAULT_BLOCK_SIZE);
+
MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(
REPLICATION_FACTOR).build();
try {
cluster.waitActive();
- long fileLen = 2 * DEFAULT_BLOCK_SIZE;
+ // the third block will not be visible to getBlocks
+ long fileLen = 2 * DEFAULT_BLOCK_SIZE + 1;
DFSTestUtil.createFile(cluster.getFileSystem(), new Path("/tmp.txt"),
fileLen, REPLICATION_FACTOR, 0L);
@@ -196,7 +200,7 @@ public class TestGetBlocks {
DFSUtilClient.getNNAddress(CONF), CONF);
locatedBlocks = dfsclient.getNamenode()
.getBlockLocations("/tmp.txt", 0, fileLen).getLocatedBlocks();
- assertEquals(2, locatedBlocks.size());
+ assertEquals(3, locatedBlocks.size());
notWritten = false;
for (int i = 0; i < 2; i++) {
dataNodes = locatedBlocks.get(i).getLocations();
[11/13] hadoop git commit: HDFS-10284.
o.a.h.hdfs.server.blockmanagement.TestBlockManagerSafeMode.testCheckSafeMode
fails intermittently. Contributed by Mingliang Liu.
Posted by wa...@apache.org.
HDFS-10284. o.a.h.hdfs.server.blockmanagement.TestBlockManagerSafeMode.testCheckSafeMode fails intermittently. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1fa9ff7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1fa9ff7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1fa9ff7
Branch: refs/heads/YARN-3368
Commit: b1fa9ff73270c95249995ff6e436888da1f2c3fb
Parents: 11e53d2
Author: Walter Su <wa...@apache.org>
Authored: Tue Apr 19 18:07:46 2016 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:46 2016 -0700
----------------------------------------------------------------------
.../TestBlockManagerSafeMode.java | 64 +++++++++++++++++---
1 file changed, 54 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fa9ff7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
index a347669..dc18a22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
@@ -124,57 +124,101 @@ public class TestBlockManagerSafeMode {
/**
* Test the state machine transition.
+ *
+ * We use separate test methods instead of a single one because the mocked
+ * {@link #fsn} is accessed concurrently by the current thread and the
+ * <code>smmthread</code> thread in the {@link BlockManagerSafeMode}. Stubbing
+ * or verification of a shared mock from different threads is NOT the proper
+ * way of testing, which leads to intermittent behavior. Meanwhile, previous
+ * state transition may have side effects that should be reset before next
+ * test case. For example, in {@link BlockManagerSafeMode}, there is a safe
+ * mode monitor thread, which is created when BlockManagerSafeMode is
+ * constructed. It will start the first time the safe mode enters extension
+ * stage and will stop if the safe mode leaves to OFF state. Across different
+ * test cases, this thread should be reset.
*/
- @Test(timeout = 30000)
- public void testCheckSafeMode() {
- bmSafeMode.activate(BLOCK_TOTAL);
-
+ @Test(timeout = 10000)
+ public void testCheckSafeMode1() {
// stays in PENDING_THRESHOLD: pending block threshold
+ bmSafeMode.activate(BLOCK_TOTAL);
setSafeModeStatus(BMSafeModeStatus.PENDING_THRESHOLD);
for (long i = 0; i < BLOCK_THRESHOLD; i++) {
setBlockSafe(i);
bmSafeMode.checkSafeMode();
assertEquals(BMSafeModeStatus.PENDING_THRESHOLD, getSafeModeStatus());
}
+ }
- // PENDING_THRESHOLD -> EXTENSION
+ /** Check safe mode transition from PENDING_THRESHOLD to EXTENSION. */
+ @Test(timeout = 10000)
+ public void testCheckSafeMode2() {
+ bmSafeMode.activate(BLOCK_TOTAL);
Whitebox.setInternalState(bmSafeMode, "extension", Integer.MAX_VALUE);
setSafeModeStatus(BMSafeModeStatus.PENDING_THRESHOLD);
setBlockSafe(BLOCK_THRESHOLD);
bmSafeMode.checkSafeMode();
assertEquals(BMSafeModeStatus.EXTENSION, getSafeModeStatus());
+ }
- // PENDING_THRESHOLD -> OFF
+ /** Check safe mode transition from PENDING_THRESHOLD to OFF. */
+ @Test(timeout = 10000)
+ public void testCheckSafeMode3() {
+ bmSafeMode.activate(BLOCK_TOTAL);
Whitebox.setInternalState(bmSafeMode, "extension", 0);
setSafeModeStatus(BMSafeModeStatus.PENDING_THRESHOLD);
setBlockSafe(BLOCK_THRESHOLD);
bmSafeMode.checkSafeMode();
assertEquals(BMSafeModeStatus.OFF, getSafeModeStatus());
+ }
- // stays in EXTENSION
+ /** Check safe mode stays in EXTENSION pending threshold. */
+ @Test(timeout = 10000)
+ public void testCheckSafeMode4() {
+ bmSafeMode.activate(BLOCK_TOTAL);
setBlockSafe(0);
setSafeModeStatus(BMSafeModeStatus.EXTENSION);
Whitebox.setInternalState(bmSafeMode, "extension", 0);
bmSafeMode.checkSafeMode();
assertEquals(BMSafeModeStatus.EXTENSION, getSafeModeStatus());
+ }
- // stays in EXTENSION: pending extension period
+ /** Check safe mode stays in EXTENSION pending extension period. */
+ @Test(timeout = 10000)
+ public void testCheckSafeMode5() {
+ bmSafeMode.activate(BLOCK_TOTAL);
Whitebox.setInternalState(bmSafeMode, "extension", Integer.MAX_VALUE);
setSafeModeStatus(BMSafeModeStatus.EXTENSION);
setBlockSafe(BLOCK_THRESHOLD);
bmSafeMode.checkSafeMode();
assertEquals(BMSafeModeStatus.EXTENSION, getSafeModeStatus());
+ }
- // should stay in PENDING_THRESHOLD during transitionToActive
+ /** Check it will not leave safe mode during NN transitionToActive. */
+ @Test(timeout = 10000)
+ public void testCheckSafeMode6() {
doReturn(true).when(fsn).inTransitionToActive();
+ bmSafeMode.activate(BLOCK_TOTAL);
Whitebox.setInternalState(bmSafeMode, "extension", 0);
setSafeModeStatus(BMSafeModeStatus.PENDING_THRESHOLD);
setBlockSafe(BLOCK_THRESHOLD);
bmSafeMode.checkSafeMode();
assertEquals(BMSafeModeStatus.PENDING_THRESHOLD, getSafeModeStatus());
+ }
+
+ /** Check smmthread will leave safe mode if NN is not in transitionToActive.*/
+ @Test(timeout = 10000)
+ public void testCheckSafeMode7() throws Exception {
doReturn(false).when(fsn).inTransitionToActive();
+ bmSafeMode.activate(BLOCK_TOTAL);
+ Whitebox.setInternalState(bmSafeMode, "extension", 5);
+ setBlockSafe(BLOCK_THRESHOLD);
bmSafeMode.checkSafeMode();
- assertEquals(BMSafeModeStatus.OFF, getSafeModeStatus());
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return getSafeModeStatus() == BMSafeModeStatus.OFF;
+ }
+ }, 100, 10000);
}
/**
[10/13] hadoop git commit: HDFS-10306. SafeModeMonitor should not
leave safe mode if name system is starting active service. Contributed by
Mingliang Liu.
Posted by wa...@apache.org.
HDFS-10306. SafeModeMonitor should not leave safe mode if name system is starting active service. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/323db7ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/323db7ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/323db7ba
Branch: refs/heads/YARN-3368
Commit: 323db7bab7a1a5d824776e37fe22571c0eea7f9a
Parents: d04d0ce
Author: Jing Zhao <ji...@apache.org>
Authored: Mon Apr 18 18:05:49 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:46 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/323db7ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
index e2d688e..ea70c86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
@@ -623,7 +623,9 @@ class BlockManagerSafeMode {
* the extension time has passed.
*/
private boolean canLeave() {
- if (timeToLeaveExtension() > 0) {
+ if (namesystem.inTransitionToActive()) {
+ return false;
+ } else if (timeToLeaveExtension() > 0) {
reportStatus("STATE* Safe mode ON, in safe mode extension.", false);
return false;
} else if (!areThresholdsMet()) {
[04/13] hadoop git commit: HDFS-10275. TestDataNodeMetrics failing
intermittently due to TotalWriteTime counted incorrectly. Contributed by Lin
Yiqun.
Posted by wa...@apache.org.
HDFS-10275. TestDataNodeMetrics failing intermittently due to TotalWriteTime counted incorrectly. Contributed by Lin Yiqun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c4b3f36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c4b3f36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c4b3f36
Branch: refs/heads/YARN-3368
Commit: 0c4b3f360bef0b26ab2d70f52795a7297cc398b5
Parents: 22855e6
Author: Walter Su <wa...@apache.org>
Authored: Mon Apr 18 20:29:29 2016 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:45 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/TestDataNodeMetrics.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c4b3f36/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
index 5f9b602..355f7a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
@@ -258,10 +258,9 @@ public class TestDataNodeMetrics {
* and reading causes totalReadTime to move.
* @throws Exception
*/
- @Test(timeout=60000)
+ @Test(timeout=120000)
public void testDataNodeTimeSpend() throws Exception {
Configuration conf = new HdfsConfiguration();
- SimulatedFSDataset.setFactory(conf);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
try {
final FileSystem fs = cluster.getFileSystem();
@@ -284,6 +283,7 @@ public class TestDataNodeMetrics {
DFSTestUtil.createFile(fs, new Path("/time.txt." + x.get()),
LONG_FILE_LEN, (short) 1, Time.monotonicNow());
DFSTestUtil.readFile(fs, new Path("/time.txt." + x.get()));
+ fs.delete(new Path("/time.txt." + x.get()), true);
} catch (IOException ioe) {
LOG.error("Caught IOException while ingesting DN metrics", ioe);
return false;
@@ -294,7 +294,7 @@ public class TestDataNodeMetrics {
return endWriteValue > startWriteValue
&& endReadValue > startReadValue;
}
- }, 30, 30000);
+ }, 30, 60000);
} finally {
if (cluster != null) {
cluster.shutdown();
[09/13] hadoop git commit: HDFS-10265. OEV tool fails to read edit
xml file if OP_UPDATE_BLOCKS has no BLOCK tag (Wan Chang via cmccabe)
Posted by wa...@apache.org.
HDFS-10265. OEV tool fails to read edit xml file if OP_UPDATE_BLOCKS has no BLOCK tag (Wan Chang via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d04d0ce8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d04d0ce8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d04d0ce8
Branch: refs/heads/YARN-3368
Commit: d04d0ce851d0992e91626df4fb6d852ebe89633c
Parents: 38aed9d
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Apr 18 11:45:18 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:46 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/namenode/FSEditLogOp.java | 3 ++-
.../test/java/org/apache/hadoop/hdfs/DFSTestUtil.java | 12 ++++++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04d0ce8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index c4e1a78..a3285a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -1096,7 +1096,8 @@ public abstract class FSEditLogOp {
@Override void fromXml(Stanza st) throws InvalidXmlException {
this.path = st.getValue("PATH");
- List<Stanza> blocks = st.getChildren("BLOCK");
+ List<Stanza> blocks = st.hasChildren("BLOCK") ?
+ st.getChildren("BLOCK") : new ArrayList<Stanza>();
this.blocks = new Block[blocks.size()];
for (int i = 0; i < blocks.size(); i++) {
this.blocks[i] = FSEditLogOp.blockFromXml(blocks.get(i));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04d0ce8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 8a52bbb..d159fc5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1272,6 +1272,18 @@ public class DFSTestUtil {
// OP_APPEND 47
FSDataOutputStream s2 = filesystem.append(pathFileCreate, 4096, null);
s2.close();
+
+ // OP_UPDATE_BLOCKS 25
+ final String updateBlockFile = "/update_blocks";
+ FSDataOutputStream fout = filesystem.create(new Path(updateBlockFile), true, 4096, (short)1, 4096L);
+ fout.write(1);
+ fout.hflush();
+ long fileId = ((DFSOutputStream)fout.getWrappedStream()).getFileId();
+ DFSClient dfsclient = DFSClientAdapter.getDFSClient(filesystem);
+ LocatedBlocks blocks = dfsclient.getNamenode().getBlockLocations(updateBlockFile, 0, Integer.MAX_VALUE);
+ dfsclient.getNamenode().abandonBlock(blocks.get(0).getBlock(), fileId, updateBlockFile, dfsclient.clientName);
+ fout.close();
+
// OP_SET_STORAGE_POLICY 45
filesystem.setStoragePolicy(pathFileCreate,
HdfsConstants.HOT_STORAGE_POLICY_NAME);
[13/13] hadoop git commit: HDFS-9744.
TestDirectoryScanner#testThrottling occasionally time out after 300 seconds.
Contributed by Lin Yiqun.
Posted by wa...@apache.org.
HDFS-9744. TestDirectoryScanner#testThrottling occasionally time out after 300 seconds. Contributed by Lin Yiqun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49caa126
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49caa126
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49caa126
Branch: refs/heads/YARN-3368
Commit: 49caa1263318a4cfdec0c30b91c0426342f033b0
Parents: b1fa9ff
Author: Walter Su <wa...@apache.org>
Authored: Tue Apr 19 18:54:11 2016 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:47 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49caa126/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index d860107..48d4681 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@ -560,7 +560,7 @@ public class TestDirectoryScanner {
*
* @throws Exception thrown on unexpected failure
*/
- @Test (timeout=300000)
+ @Test (timeout=600000)
public void testThrottling() throws Exception {
Configuration conf = new Configuration(CONF);