You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/03/03 02:45:12 UTC
[01/50] [abbrv] hadoop git commit: HADOOP-14100. Upgrade Jsch jar to
latest version to fix vulnerability in old versions. Contributed by
Vinayakumar B [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-5734 6136aff31 -> 0b869d925 (forced update)
HADOOP-14100. Upgrade Jsch jar to latest version to fix vulnerability in old versions. Contributed by Vinayakumar B
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/159d6c56
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/159d6c56
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/159d6c56
Branch: refs/heads/YARN-5734
Commit: 159d6c56e7f3aa3ebe45750cf88735287f047b42
Parents: 82607fc
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Feb 23 14:25:08 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Feb 23 14:25:08 2017 -0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/159d6c56/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 47e21d8..c8aa857 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -963,7 +963,7 @@
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
- <version>0.1.51</version>
+ <version>0.1.54</version>
</dependency>
<dependency>
<groupId>org.apache.htrace</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-6228:
EntityGroupFSTimelineStore should allow configurable cache stores.
Contributed by Li Lu
Posted by jh...@apache.org.
YARN-6228: EntityGroupFSTimelineStore should allow configurable cache
stores. Contributed by Li Lu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53d372a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53d372a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53d372a2
Branch: refs/heads/YARN-5734
Commit: 53d372a2550c970f3dd3c49738af3c1789ae589b
Parents: c1a52b0
Author: Xuan <xg...@apache.org>
Authored: Fri Feb 24 10:58:48 2017 -0800
Committer: Xuan <xg...@apache.org>
Committed: Fri Feb 24 10:59:35 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +++
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 5 +++++
.../apache/hadoop/yarn/server/timeline/EntityCacheItem.java | 9 +++++++--
3 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d372a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 094a424..cdccec6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1890,6 +1890,9 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX =
TIMELINE_SERVICE_PREFIX + "entity-group-fs-store.";
+ public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CACHE_STORE =
+ TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "cache-store-class";
+
public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR =
TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "active-dir";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d372a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 53beb5e..368946e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2096,6 +2096,11 @@
</property>
<!-- Timeline Service v1.5 Configuration -->
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.cache-store-class</name>
+ <value>org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore</value>
+ <description>Caching storage timeline server v1.5 is using. </description>
+ </property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d372a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java
index 7ed7c4a..8df60ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java
@@ -17,8 +17,10 @@
package org.apache.hadoop.yarn.server.timeline;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -95,8 +97,11 @@ public class EntityCacheItem {
}
if (!appLogs.getDetailLogs().isEmpty()) {
if (store == null) {
- store = new LevelDBCacheTimelineStore(groupId.toString(),
- "LeveldbCache." + groupId);
+ store = ReflectionUtils.newInstance(config.getClass(
+ YarnConfiguration
+ .TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CACHE_STORE,
+ MemoryTimelineStore.class, TimelineStore.class),
+ config);
store.init(config);
store.start();
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-12556. KafkaSink jar files
are created but not copied to target dist (Babak Behzad via aw)
Posted by jh...@apache.org.
HADOOP-12556. KafkaSink jar files are created but not copied to target dist (Babak Behzad via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/989bd56b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/989bd56b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/989bd56b
Branch: refs/heads/YARN-5734
Commit: 989bd56b9fd521a67afbc691c9915be13d7c8e3a
Parents: 235203d
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Feb 28 16:15:40 2017 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Feb 28 16:15:40 2017 -0800
----------------------------------------------------------------------
hadoop-tools/hadoop-tools-dist/pom.xml | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989bd56b/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 00f07ef..dd28404 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -95,6 +95,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-kafka</artifactId>
+ <scope>compile</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
<scope>compile</scope>
<version>${project.version}</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDFS-11295. Check storage
remaining instead of node remaining in
BlockPlacementPolicyDefault.chooseReplicaToDelete(). Contributed by Marton
Elek.
Posted by jh...@apache.org.
HDFS-11295. Check storage remaining instead of node remaining in BlockPlacementPolicyDefault.chooseReplicaToDelete(). Contributed by Marton Elek.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2b3ba9b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2b3ba9b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2b3ba9b
Branch: refs/heads/YARN-5734
Commit: d2b3ba9b8fb76753fa1b51661dacbde74aa5c6df
Parents: 289bc50
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Feb 24 15:44:11 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Feb 24 15:44:11 2017 -0800
----------------------------------------------------------------------
.../BlockPlacementPolicyDefault.java | 2 +-
.../blockmanagement/DatanodeStorageInfo.java | 5 +++
.../blockmanagement/TestReplicationPolicy.java | 35 ++++++++++++++------
.../TestReplicationPolicyWithNodeGroup.java | 23 ++++++++++---
4 files changed, 49 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2b3ba9b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index eb54667..7676334 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -968,7 +968,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
- long free = node.getRemaining();
+ long free = storage.getRemaining();
long lastHeartbeat = node.getLastUpdateMonotonic();
if (lastHeartbeat < oldestHeartbeat) {
oldestHeartbeat = lastHeartbeat;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2b3ba9b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index b4c8aaa..ab666b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -388,6 +388,11 @@ public class DatanodeStorageInfo {
return null;
}
+ @VisibleForTesting
+ void setRemainingForTests(int remaining) {
+ this.remaining = remaining;
+ }
+
static enum AddBlockResult {
ADDED, REPLACED, ALREADY_EXIST
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2b3ba9b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 1af013d..27dcbf1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -950,24 +950,31 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
List<DatanodeStorageInfo> replicaList = new ArrayList<>();
final Map<String, List<DatanodeStorageInfo>> rackMap
= new HashMap<String, List<DatanodeStorageInfo>>();
-
- dataNodes[0].setRemaining(4*1024*1024);
+
+ storages[0].setRemainingForTests(4*1024*1024);
+ dataNodes[0].setRemaining(calculateRemaining(dataNodes[0]));
replicaList.add(storages[0]);
-
- dataNodes[1].setRemaining(3*1024*1024);
+
+ storages[1].setRemainingForTests(3*1024*1024);
+ dataNodes[1].setRemaining(calculateRemaining(dataNodes[1]));
replicaList.add(storages[1]);
-
- dataNodes[2].setRemaining(2*1024*1024);
+
+ storages[2].setRemainingForTests(2*1024*1024);
+ dataNodes[2].setRemaining(calculateRemaining(dataNodes[2]));
replicaList.add(storages[2]);
-
- dataNodes[5].setRemaining(1*1024*1024);
+
+ //Even if this node has the most space, because the storage[5] has
+ //the lowest it should be chosen in case of block delete.
+ storages[4].setRemainingForTests(100 * 1024 * 1024);
+ storages[5].setRemainingForTests(512 * 1024);
+ dataNodes[5].setRemaining(calculateRemaining(dataNodes[5]));
replicaList.add(storages[5]);
-
+
// Refresh the last update time for all the datanodes
for (int i = 0; i < dataNodes.length; i++) {
DFSTestUtil.resetLastUpdatesWithOffset(dataNodes[i], 0);
}
-
+
List<DatanodeStorageInfo> first = new ArrayList<>();
List<DatanodeStorageInfo> second = new ArrayList<>();
replicator.splitNodesWithRack(replicaList, replicaList, rackMap, first,
@@ -999,6 +1006,14 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
assertEquals(chosen, storages[1]);
}
+ private long calculateRemaining(DatanodeDescriptor dataNode) {
+ long sum = 0;
+ for (DatanodeStorageInfo storageInfo: dataNode.getStorageInfos()){
+ sum += storageInfo.getRemaining();
+ }
+ return sum;
+ }
+
@Test
public void testChooseReplicasToDelete() throws Exception {
Collection<DatanodeStorageInfo> nonExcess = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2b3ba9b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
index 2f184bb..ebd4b81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
@@ -625,16 +625,21 @@ public class TestReplicationPolicyWithNodeGroup extends BaseReplicationPolicyTes
public void testChooseReplicaToDelete() throws Exception {
List<DatanodeStorageInfo> replicaList = new ArrayList<>();
final Map<String, List<DatanodeStorageInfo>> rackMap = new HashMap<>();
- dataNodes[0].setRemaining(4*1024*1024);
+ storages[0].setRemainingForTests(4*1024*1024);
+ dataNodes[0].setRemaining(calculateRemaining(dataNodes[0]));
replicaList.add(storages[0]);
- dataNodes[1].setRemaining(3*1024*1024);
+ storages[1].setRemainingForTests(3*1024*1024);
+ dataNodes[1].setRemaining(calculateRemaining(dataNodes[1]));
replicaList.add(storages[1]);
- dataNodes[2].setRemaining(2*1024*1024);
+ storages[2].setRemainingForTests(2*1024*1024);
+ dataNodes[2].setRemaining(calculateRemaining(dataNodes[2]));
replicaList.add(storages[2]);
- dataNodes[5].setRemaining(1*1024*1024);
+ storages[4].setRemainingForTests(100 * 1024 * 1024);
+ storages[5].setRemainingForTests(512 * 1024);
+ dataNodes[5].setRemaining(calculateRemaining(dataNodes[5]));
replicaList.add(storages[5]);
List<DatanodeStorageInfo> first = new ArrayList<>();
@@ -671,7 +676,15 @@ public class TestReplicationPolicyWithNodeGroup extends BaseReplicationPolicyTes
first, second, excessTypes, rackMap);
assertEquals(chosen, storages[5]);
}
-
+
+ private long calculateRemaining(DatanodeDescriptor dataNode) {
+ long sum = 0;
+ for (DatanodeStorageInfo storageInfo: dataNode.getStorageInfos()){
+ sum += storageInfo.getRemaining();
+ }
+ return sum;
+ }
+
/**
* Test replica placement policy in case of boundary topology.
* Rack 2 has only 1 node group & can't be placed with two replicas
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDFS-11412. Maintenance minimum
replication config value allowable range should be [0,
DefaultReplication]. (Manoj Govindassamy via mingma)
Posted by jh...@apache.org.
HDFS-11412. Maintenance minimum replication config value allowable range should be [0, DefaultReplication]. (Manoj Govindassamy via mingma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25c84d27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25c84d27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25c84d27
Branch: refs/heads/YARN-5734
Commit: 25c84d279bcefb72a3dd8058f25bba1713504849
Parents: 6f6dfe0
Author: Ming Ma <mi...@apache.org>
Authored: Wed Mar 1 20:23:52 2017 -0800
Committer: Ming Ma <mi...@apache.org>
Committed: Wed Mar 1 20:23:52 2017 -0800
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 9 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 ++
.../hadoop/hdfs/TestMaintenanceState.java | 111 +++++++++++++++++++
3 files changed, 124 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25c84d27/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5125b33..5ca0fa7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -484,12 +484,12 @@ public class BlockManager implements BlockStatsMXBean {
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY
+ " = " + minMaintenanceR + " < 0");
}
- if (minMaintenanceR > minR) {
+ if (minMaintenanceR > defaultReplication) {
throw new IOException("Unexpected configuration parameters: "
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY
+ " = " + minMaintenanceR + " > "
- + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY
- + " = " + minR);
+ + DFSConfigKeys.DFS_REPLICATION_KEY
+ + " = " + defaultReplication);
}
this.minReplicationToBeInMaintenance = (short)minMaintenanceR;
@@ -825,7 +825,8 @@ public class BlockManager implements BlockStatsMXBean {
if (block.isStriped()) {
return ((BlockInfoStriped) block).getRealDataBlockNum();
} else {
- return minReplicationToBeInMaintenance;
+ return (short) Math.min(minReplicationToBeInMaintenance,
+ block.getReplication());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25c84d27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 51dca41..f9908fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -807,6 +807,14 @@ public class MiniDFSCluster implements AutoCloseable {
int replication = conf.getInt(DFS_REPLICATION_KEY, 3);
conf.setInt(DFS_REPLICATION_KEY, Math.min(replication, numDataNodes));
+ int maintenanceMinReplication = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT);
+ if (maintenanceMinReplication ==
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT) {
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY,
+ Math.min(maintenanceMinReplication, numDataNodes));
+ }
int safemodeExtension = conf.getInt(
DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 0);
conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, safemodeExtension);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25c84d27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
index f3e2a0b..2432153 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
@@ -30,6 +31,7 @@ import java.util.List;
import java.util.Map;
import com.google.common.collect.Lists;
+import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileSystem;
@@ -67,6 +69,38 @@ public class TestMaintenanceState extends AdminStatesBaseTest {
}
/**
+ * Test valid value range for the config namenode.maintenance.replication.min.
+ */
+ @Test (timeout = 60000)
+ public void testMaintenanceMinReplConfigRange() {
+ LOG.info("Setting testMaintenanceMinReplConfigRange");
+
+ // Case 1: Maintenance min replication less allowed minimum 0
+ setMinMaintenanceR(-1);
+ try {
+ startCluster(1, 1);
+ fail("Cluster start should fail when 'dfs.namenode.maintenance" +
+ ".replication.min=-1'");
+ } catch (IOException e) {
+ LOG.info("Expected exception: " + e);
+ }
+
+ // Case 2: Maintenance min replication greater
+ // allowed max of DFSConfigKeys.DFS_REPLICATION_KEY
+ int defaultRepl = getConf().getInt(
+ DFSConfigKeys.DFS_REPLICATION_KEY,
+ DFSConfigKeys.DFS_REPLICATION_DEFAULT);
+ setMinMaintenanceR(defaultRepl + 1);
+ try {
+ startCluster(1, 1);
+ fail("Cluster start should fail when 'dfs.namenode.maintenance" +
+ ".replication.min > " + defaultRepl + "'");
+ } catch (IOException e) {
+ LOG.info("Expected exception: " + e);
+ }
+ }
+
+ /**
* Verify a node can transition from AdminStates.ENTERING_MAINTENANCE to
* AdminStates.NORMAL.
*/
@@ -407,6 +441,83 @@ public class TestMaintenanceState extends AdminStatesBaseTest {
}
/**
+ * Test file block replication lesser than maintenance minimum.
+ */
+ @Test(timeout = 360000)
+ public void testFileBlockReplicationAffectingMaintenance()
+ throws Exception {
+ int defaultReplication = getConf().getInt(DFSConfigKeys
+ .DFS_REPLICATION_KEY, DFSConfigKeys.DFS_REPLICATION_DEFAULT);
+ int defaultMaintenanceMinRepl = getConf().getInt(DFSConfigKeys
+ .DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT);
+
+ // Case 1:
+ // * Maintenance min larger than default min replication
+ // * File block replication larger than maintenance min
+ // * Initial data nodes not sufficient to remove all maintenance nodes
+ // as file block replication is greater than maintenance min.
+ // * Data nodes added later for the state transition to progress
+ int maintenanceMinRepl = defaultMaintenanceMinRepl + 1;
+ int fileBlockReplication = maintenanceMinRepl + 1;
+ int numAddedDataNodes = 1;
+ int numInitialDataNodes = (maintenanceMinRepl * 2 - numAddedDataNodes);
+ Assert.assertTrue(maintenanceMinRepl <= defaultReplication);
+ testFileBlockReplicationImpl(maintenanceMinRepl,
+ numInitialDataNodes, numAddedDataNodes, fileBlockReplication);
+
+ // Case 2:
+ // * Maintenance min larger than default min replication
+ // * File block replication lesser than maintenance min
+ // * Initial data nodes after removal of maintenance nodes is still
+ // sufficient for the file block replication.
+ // * No new data nodes to be added, still the state transition happens
+ maintenanceMinRepl = defaultMaintenanceMinRepl + 1;
+ fileBlockReplication = maintenanceMinRepl - 1;
+ numAddedDataNodes = 0;
+ numInitialDataNodes = (maintenanceMinRepl * 2 - numAddedDataNodes);
+ testFileBlockReplicationImpl(maintenanceMinRepl,
+ numInitialDataNodes, numAddedDataNodes, fileBlockReplication);
+ }
+
+ private void testFileBlockReplicationImpl(
+ int maintenanceMinRepl, int numDataNodes, int numNewDataNodes,
+ int fileBlockRepl)
+ throws Exception {
+ setup();
+ LOG.info("Starting testLargerMinMaintenanceReplication - maintMinRepl: "
+ + maintenanceMinRepl + ", numDNs: " + numDataNodes + ", numNewDNs: "
+ + numNewDataNodes + ", fileRepl: " + fileBlockRepl);
+ LOG.info("Setting maintenance minimum replication: " + maintenanceMinRepl);
+ setMinMaintenanceR(maintenanceMinRepl);
+ startCluster(1, numDataNodes);
+
+ final Path file = new Path("/testLargerMinMaintenanceReplication.dat");
+
+ FileSystem fileSys = getCluster().getFileSystem(0);
+ writeFile(fileSys, file, fileBlockRepl, 1);
+ final DatanodeInfo[] nodes = getFirstBlockReplicasDatanodeInfos(fileSys,
+ file);
+
+ ArrayList<String> nodeUuids = new ArrayList<>();
+ for (int i = 0; i < maintenanceMinRepl && i < nodes.length; i++) {
+ nodeUuids.add(nodes[i].getDatanodeUuid());
+ }
+
+ List<DatanodeInfo> maintenanceDNs = takeNodeOutofService(0, nodeUuids,
+ Long.MAX_VALUE, null, null, AdminStates.ENTERING_MAINTENANCE);
+
+ for (int i = 0; i < numNewDataNodes; i++) {
+ getCluster().startDataNodes(getConf(), 1, true, null, null);
+ }
+ getCluster().waitActive();
+ refreshNodes(0);
+ waitNodeState(maintenanceDNs, AdminStates.IN_MAINTENANCE);
+ cleanupFile(fileSys, file);
+ teardown();
+ }
+
+ /**
* Transition from IN_MAINTENANCE to DECOMMISSIONED.
*/
@Test(timeout = 360000)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDFS-11433. Document missing
usages of OfflineEditsViewer processors. Contributed by Yiqun Lin.
Posted by jh...@apache.org.
HDFS-11433. Document missing usages of OfflineEditsViewer processors. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a58870a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a58870a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a58870a
Branch: refs/heads/YARN-5734
Commit: 4a58870a0427ecdf4c5c895088b96a0b4d1c0c19
Parents: e24ed47
Author: Yiqun Lin <yq...@apache.org>
Authored: Sat Feb 25 10:00:05 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Sat Feb 25 10:00:05 2017 +0800
----------------------------------------------------------------------
.../src/site/markdown/HdfsEditsViewer.md | 73 +++++++++++++++++++-
1 file changed, 72 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a58870a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
index 7a46eb0..5e069bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
@@ -40,9 +40,80 @@ The Offline Edits Viewer provides several output processors (unless stated other
Usage
-----
+### XML Processor
+
+XML processor can create an XML file that contains the edits log information. Users can specify input and output file via -i and -o command-line.
+
+ bash$ bin/hdfs oev -p xml -i edits -o edits.xml
+
+XML processor is the default processor in Offline Edits Viewer, users can also use the following command:
+
bash$ bin/hdfs oev -i edits -o edits.xml
-| Flag | Description |
+This would result in the following output:
+
+ <?xml version="1.0" encoding="UTF-8"?>
+ <EDITS>
+ <EDITS_VERSION>-64</EDITS_VERSION>
+ <RECORD>
+ <OPCODE>OP_START_LOG_SEGMENT</OPCODE>
+ <DATA>
+ <TXID>1</TXID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_UPDATE_MASTER_KEY</OPCODE>
+ <DATA>
+ <TXID>2</TXID>
+ <DELEGATION_KEY>
+ <KEY_ID>1</KEY_ID>
+ <EXPIRY_DATE>1487921580728</EXPIRY_DATE>
+ <KEY>2e127ca41c7de215</KEY>
+ </DELEGATION_KEY>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ ...remaining output omitted...
+
+### Binary Processor
+
+Binary processor is the opposite of the XML processor. Users can specify input XML file and output file via -i and -o command-line.
+
+ bash$ bin/hdfs oev -p binary -i edits.xml -o edits
+
+This will reconstruct an edits log file from an XML file.
+
+### Stats Processor
+
+Stats processor is used to aggregate counts of op codes contained in the edits log file. Users can specify this processor by -p option.
+
+ bash$ bin/hdfs oev -p stats -i edits -o edits.stats
+
+The output result of this processor should be like the following output:
+
+ VERSION : -64
+ OP_ADD ( 0): 8
+ OP_RENAME_OLD ( 1): 1
+ OP_DELETE ( 2): 1
+ OP_MKDIR ( 3): 1
+ OP_SET_REPLICATION ( 4): 1
+ OP_DATANODE_ADD ( 5): 0
+ OP_DATANODE_REMOVE ( 6): 0
+ OP_SET_PERMISSIONS ( 7): 1
+ OP_SET_OWNER ( 8): 1
+ OP_CLOSE ( 9): 9
+ OP_SET_GENSTAMP_V1 ( 10): 0
+ ...some output omitted...
+ OP_APPEND ( 47): 1
+ OP_SET_QUOTA_BY_STORAGETYPE ( 48): 1
+ OP_INVALID ( -1): 0
+
+The output is formatted as a colon separated two column table: OpCode and OpCodeCount. Each OpCode corresponding to the specific operation(s) in NameNode.
+
+Options
+-------
+
+| Flag | Description |
|:---- |:---- |
| [`-i` ; `--inputFile`] *input file* | Specify the input edits log file to process. Xml (case insensitive) extension means XML format otherwise binary format is assumed. Required. |
| [`-o` ; `--outputFile`] *output file* | Specify the output filename, if the specified output processor generates one. If the specified file already exists, it is silently overwritten. Required. |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-6231. FairSchedulerTestBase
helper methods should call scheduler.update to avoid flakiness. (kasha)
Posted by jh...@apache.org.
YARN-6231. FairSchedulerTestBase helper methods should call scheduler.update to avoid flakiness. (kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f187d638
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f187d638
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f187d638
Branch: refs/heads/YARN-5734
Commit: f187d63816584b783fbfe238475c8f37decdb6dc
Parents: 258342e
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Mon Feb 27 22:10:57 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Mon Feb 27 22:10:57 2017 -0800
----------------------------------------------------------------------
.../resourcemanager/scheduler/fair/FairSchedulerTestBase.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f187d638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 2214c67..af4e1dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -190,6 +190,7 @@ public class FairSchedulerTestBase {
scheduler.allocate(id, ask, new ArrayList<ContainerId>(),
null, null, NULL_UPDATE_REQUESTS);
+ scheduler.update();
return id;
}
@@ -240,6 +241,7 @@ public class FairSchedulerTestBase {
ask.add(request);
scheduler.allocate(attId, ask, new ArrayList<ContainerId>(),
null, null, NULL_UPDATE_REQUESTS);
+ scheduler.update();
}
protected void createApplicationWithAMResource(ApplicationAttemptId attId,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDFS-11477. Simplify file IO
profiling configuration. Contributed by Hanisha Koneru.
Posted by jh...@apache.org.
HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e61491d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e61491d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e61491d4
Branch: refs/heads/YARN-5734
Commit: e61491d4769536ce8b77b416005ce531278ce729
Parents: a97833e
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Mar 2 15:38:25 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Mar 2 15:38:25 2017 -0800
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 7 ++++-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +----
.../hdfs/server/datanode/FileIoProvider.java | 2 +-
.../server/datanode/ProfilingFileIoEvents.java | 27 +++++++++++++++-----
.../datanode/TestDataNodeVolumeMetrics.java | 4 +--
.../hadoop/tools/TestHdfsConfigFields.java | 2 --
6 files changed, 31 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 7900692..a8bdbeb 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -332,7 +332,12 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
FsVolume
--------
-Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics.
+Per-volume metrics contain Datanode Volume IO related statistics. Per-volume
+metrics are off by default. They can be enabled by setting `dfs.datanode
+.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0.
+Setting this value to 0.0 would mean profiling is not enabled. But enabling
+per-volume metrics may have a performance impact. Each metrics record
+contains tags such as Hostname as additional information along with metrics.
| Name | Description |
|:---- |:---- |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index be20829..82d6073 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -718,10 +718,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT = 0; //no throttling
// Datanode File IO Stats
- public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
- "dfs.datanode.enable.fileio.profiling";
- public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
- false;
public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY =
"dfs.datanode.enable.fileio.fault.injection";
public static final boolean
@@ -730,7 +726,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
"dfs.datanode.fileio.profiling.sampling.fraction";
public static final double
- DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
+ DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0;
//Keys with no defaults
public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index 9def2e1..5508e0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*;
*
* Behavior can be injected into these events by enabling the
* profiling and/or fault injection event hooks through
- * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and
+ * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and
* {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}.
* These event hooks are disabled by default.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 9af84fa..7eab969 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.datanode;
+import static java.lang.Math.abs;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -43,18 +44,32 @@ class ProfilingFileIoEvents {
public ProfilingFileIoEvents(@Nullable Configuration conf) {
if (conf != null) {
- isEnabled = conf.getBoolean(DFSConfigKeys
- .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
- .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
- if (fileIOSamplingFraction > 1) {
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT);
+ if (abs(fileIOSamplingFraction) < 0.000001) {
+ LOG.info(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
+ + fileIOSamplingFraction + ". Disabling file IO profiling");
+ isEnabled = false;
+ } else if (fileIOSamplingFraction < 0.000001) {
LOG.warn(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
- " value cannot be more than 1. Setting value to 1");
+ " value cannot be less than 0. Disabling file IO profiling.");
+ isEnabled = false;
+ } else if (fileIOSamplingFraction > 1) {
+ LOG.warn(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
+ " value cannot be more than 1. Setting value to 1 and enabling " +
+ "file IO profiling");
+ isEnabled = true;
fileIOSamplingFraction = 1;
+ } else {
+ LOG.info(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
+ + fileIOSamplingFraction + ". Enabling file IO profiling");
+ isEnabled = true;
}
sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
index 6a8ac9c..03e1dee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
@@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics {
private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException {
Configuration conf = new HdfsConfiguration();
- conf.setBoolean(DFSConfigKeys
- .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true);
+ conf.setDouble(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
SimulatedFSDataset.setFactory(conf);
return new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATANODES)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index cdce342..1fdf713 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -103,8 +103,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
.add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
configurationPropsToSkipCompare
.add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
- configurationPropsToSkipCompare
- .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
.DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDFS-11426. Refactor EC CLI to be
similar to storage policies CLI.
Posted by jh...@apache.org.
HDFS-11426. Refactor EC CLI to be similar to storage policies CLI.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/132f758e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/132f758e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/132f758e
Branch: refs/heads/YARN-5734
Commit: 132f758e3dbe3a3f11c0d9b2de8edbee594fb475
Parents: 694e680
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Feb 23 16:00:11 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Feb 23 16:00:11 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/cli/CLITestHelper.java | 15 +-
.../hadoop-hdfs/src/main/bin/hdfs | 2 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 320 +++++++++++++++++++
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 2 +-
.../hadoop/hdfs/tools/erasurecode/ECCli.java | 62 ----
.../hdfs/tools/erasurecode/ECCommand.java | 248 --------------
.../src/site/markdown/HDFSErasureCoding.md | 16 +-
.../hadoop/cli/CLITestCmdErasureCoding.java | 4 +-
.../cli/util/ErasureCodingCliCmdExecutor.java | 6 +-
.../test/resources/testErasureCodingConf.xml | 135 ++++----
10 files changed, 405 insertions(+), 405 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
index b08af16..89d4e30 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.cli;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.cli.util.*;
import org.apache.hadoop.cli.util.CommandExecutor.Result;
import org.apache.hadoop.conf.Configuration;
@@ -28,6 +26,9 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
@@ -41,9 +42,9 @@ import java.util.ArrayList;
* Tests for the Command Line Interface (CLI)
*/
public class CLITestHelper {
- private static final Log LOG =
- LogFactory.getLog(CLITestHelper.class.getName());
-
+ private static final Logger LOG = LoggerFactory.getLogger(CLITestHelper
+ .class);
+
// In this mode, it runs the command and compares the actual output
// with the expected output
public static final String TESTMODE_TEST = "test"; // Run the tests
@@ -62,7 +63,6 @@ public class CLITestHelper {
// Storage for tests read in from the config file
protected ArrayList<CLITestData> testsFromConfigFile = null;
protected ArrayList<ComparatorData> testComparators = null;
- protected String thisTestCaseName = null;
protected ComparatorData comparatorData = null;
protected Configuration conf = null;
protected String clitestDataDir = null;
@@ -80,7 +80,8 @@ public class CLITestHelper {
p.parse(testConfigFile, getConfigParser());
success = true;
} catch (Exception e) {
- LOG.info("File: " + testConfigFile + " not found");
+ LOG.info("Exception while reading test config file {}:",
+ testConfigFile, e);
success = false;
}
assertTrue("Error reading test config file", success);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 617adbe..cf6d94a 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -130,7 +130,7 @@ function hdfscmd_case
exit 0
;;
ec)
- HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.erasurecode.ECCli
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.ECAdmin
;;
fetchdt)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
new file mode 100644
index 0000000..29c65b1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.tools.TableListing;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * CLI for the erasure code encoding operations.
+ */
+@InterfaceAudience.Private
+public class ECAdmin extends Configured implements Tool {
+
+ public static final String NAME = "ec";
+
+ public static void main(String[] args) throws Exception {
+ final ECAdmin admin = new ECAdmin(new Configuration());
+ int res = ToolRunner.run(admin, args);
+ System.exit(res);
+ }
+
+ public ECAdmin(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length == 0) {
+ AdminHelper.printUsage(false, NAME, COMMANDS);
+ ToolRunner.printGenericCommandUsage(System.err);
+ return 1;
+ }
+ final AdminHelper.Command command = AdminHelper.determineCommand(args[0],
+ COMMANDS);
+ if (command == null) {
+ System.err.println("Can't understand command '" + args[0] + "'");
+ if (!args[0].startsWith("-")) {
+ System.err.println("Command names must start with dashes.");
+ }
+ AdminHelper.printUsage(false, NAME, COMMANDS);
+ ToolRunner.printGenericCommandUsage(System.err);
+ return 1;
+ }
+ final List<String> argsList = new LinkedList<>();
+ argsList.addAll(Arrays.asList(args).subList(1, args.length));
+ try {
+ return command.run(getConf(), argsList);
+ } catch (IllegalArgumentException e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return -1;
+ }
+ }
+
+ /** Command to list the set of available erasure coding policies */
+ private static class ListECPoliciesCommand
+ implements AdminHelper.Command {
+ @Override
+ public String getName() {
+ return "-listPolicies";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + "]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ return getShortUsage() + "\n" +
+ "Get the list of supported erasure coding policies.\n";
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ if (args.size() > 0) {
+ System.err.println(getName() + ": Too many arguments");
+ return 1;
+ }
+
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+ try {
+ Collection<ErasureCodingPolicy> policies =
+ dfs.getAllErasureCodingPolicies();
+ System.out.println("Erasure Coding Policies:");
+ for (ErasureCodingPolicy policy : policies) {
+ if (policy != null) {
+ System.out.println("\t" + policy.getName());
+ }
+ }
+ } catch (IOException e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
+ /** Command to get the erasure coding policy for a file or directory */
+ private static class GetECPolicyCommand implements AdminHelper.Command {
+ @Override
+ public String getName() {
+ return "-getPolicy";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + " -path <path>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ final TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<path>",
+ "The path of the file/directory for getting the erasure coding " +
+ "policy");
+ return getShortUsage() + "\n" +
+ "Get the erasure coding policy of a file/directory.\n\n" +
+ listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ final String path = StringUtils.popOptionWithArgument("-path", args);
+ if (path == null) {
+ System.err.println("Please specify the path with -path.\nUsage: " +
+ getLongUsage());
+ return 1;
+ }
+
+ if (args.size() > 0) {
+ System.err.println(getName() + ": Too many arguments");
+ return 1;
+ }
+
+ final Path p = new Path(path);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ try {
+ ErasureCodingPolicy ecPolicy = dfs.getErasureCodingPolicy(p);
+ if (ecPolicy != null) {
+ System.out.println(ecPolicy.getName());
+ } else {
+ System.out.println("The erasure coding policy of " + path + " is " +
+ "unspecified");
+ }
+ } catch (Exception e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
+ /** Command to set the erasure coding policy to a file/directory */
+ private static class SetECPolicyCommand implements AdminHelper.Command {
+ @Override
+ public String getName() {
+ return "-setPolicy";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + " -path <path> -policy <policy>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<path>", "The path of the file/directory to set " +
+ "the erasure coding policy");
+ listing.addRow("<policy>", "The name of the erasure coding policy");
+ return getShortUsage() + "\n" +
+ "Set the erasure coding policy for a file/directory.\n\n" +
+ listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ final String path = StringUtils.popOptionWithArgument("-path", args);
+ if (path == null) {
+ System.err.println("Please specify the path for setting the EC " +
+ "policy.\nUsage: " + getLongUsage());
+ return 1;
+ }
+
+ final String ecPolicyName = StringUtils.popOptionWithArgument("-policy",
+ args);
+ if (ecPolicyName == null) {
+ System.err.println("Please specify the policy name.\nUsage: " +
+ getLongUsage());
+ return 1;
+ }
+
+ if (args.size() > 0) {
+ System.err.println(getName() + ": Too many arguments");
+ return 1;
+ }
+
+ final Path p = new Path(path);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ try {
+ ErasureCodingPolicy ecPolicy = null;
+ ErasureCodingPolicy[] ecPolicies =
+ dfs.getClient().getErasureCodingPolicies();
+ for (ErasureCodingPolicy policy : ecPolicies) {
+ if (ecPolicyName.equals(policy.getName())) {
+ ecPolicy = policy;
+ break;
+ }
+ }
+ if (ecPolicy == null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Policy '");
+ sb.append(ecPolicyName);
+ sb.append("' does not match any of the supported policies.");
+ sb.append(" Please select any one of ");
+ List<String> ecPolicyNames = new ArrayList<String>();
+ for (ErasureCodingPolicy policy : ecPolicies) {
+ ecPolicyNames.add(policy.getName());
+ }
+ sb.append(ecPolicyNames);
+ System.err.println(sb.toString());
+ return 3;
+ }
+ dfs.setErasureCodingPolicy(p, ecPolicy);
+ System.out.println("Set erasure coding policy " + ecPolicyName +
+ " on " + path);
+ } catch (Exception e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
+ /** Command to unset the erasure coding policy set for a file/directory */
+ private static class UnsetECPolicyCommand
+ implements AdminHelper.Command {
+
+ @Override
+ public String getName() {
+ return "-unsetPolicy";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + " -path <path>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<path>", "The path of the directory "
+ + "from which the erasure coding policy will be unset.");
+ return getShortUsage() + "\n"
+ + "Unset the erasure coding policy for a directory.\n\n"
+ + listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ final String path = StringUtils.popOptionWithArgument("-path", args);
+ if (path == null) {
+ System.err.println("Please specify a path.\nUsage: " + getLongUsage());
+ return 1;
+ }
+
+ if (args.size() > 0) {
+ System.err.println(getName() + ": Too many arguments");
+ return 1;
+ }
+
+ final Path p = new Path(path);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ try {
+ dfs.unsetErasureCodingPolicy(p);
+ System.out.println("Unset erasure coding policy from " + path);
+ } catch (Exception e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
+ private static final AdminHelper.Command[] COMMANDS = {
+ new ListECPoliciesCommand(),
+ new GetECPolicyCommand(),
+ new SetECPolicyCommand(),
+ new UnsetECPolicyCommand()
+ };
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
index f0643b2..9c7d048 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
@@ -143,7 +143,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
public int run(Configuration conf, List<String> args) throws IOException {
final String path = StringUtils.popOptionWithArgument("-path", args);
if (path == null) {
- System.err.println("Please specify the path with -path.\nUsage:" +
+ System.err.println("Please specify the path with -path.\nUsage: " +
getLongUsage());
return 1;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java
deleted file mode 100644
index 89dd4ee..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdfs.tools.erasurecode;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FsShell;
-import org.apache.hadoop.fs.shell.CommandFactory;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.util.ToolRunner;
-
-import java.io.IOException;
-
-/**
- * CLI for the erasure code encoding operations.
- */
-@InterfaceAudience.Private
-public class ECCli extends FsShell {
-
- private final static String usagePrefix =
- "Usage: hdfs ec [generic options]";
-
- @Override
- protected String getUsagePrefix() {
- return usagePrefix;
- }
-
- @Override
- protected void init() throws IOException {
- getConf().setQuietMode(true);
- if (commandFactory == null) {
- commandFactory = new CommandFactory(getConf());
- commandFactory.addObject(getHelp(), "-help");
- registerCommands(commandFactory);
- }
- }
-
- @Override
- protected void registerCommands(CommandFactory factory) {
- factory.registerCommands(ECCommand.class);
- }
-
- public static void main(String[] args) throws Exception {
- Configuration conf = new HdfsConfiguration();
- int res = ToolRunner.run(conf, new ECCli(), args);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
deleted file mode 100644
index fc732e0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdfs.tools.erasurecode;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.HadoopIllegalArgumentException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.shell.Command;
-import org.apache.hadoop.fs.shell.CommandFactory;
-import org.apache.hadoop.fs.shell.PathData;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * Erasure Coding CLI commands
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class ECCommand extends Command {
-
- public static void registerCommands(CommandFactory factory) {
- // Register all commands of Erasure CLI, with a '-' at the beginning in name
- // of the command.
- factory.addClass(SetECPolicyCommand.class, "-" + SetECPolicyCommand.NAME);
- factory.addClass(GetECPolicyCommand.class, "-"
- + GetECPolicyCommand.NAME);
- factory.addClass(UnsetECPolicyCommand.class, "-"
- + UnsetECPolicyCommand.NAME);
- factory.addClass(ListPolicies.class, "-" + ListPolicies.NAME);
- }
-
- @Override
- public String getCommandName() {
- return getName();
- }
-
- @Override
- protected void run(Path path) throws IOException {
- throw new RuntimeException("Not suppose to get here");
- }
-
- @Deprecated
- @Override
- public int runAll() {
- return run(args);
- }
-
- @Override
- protected void processPath(PathData item) throws IOException {
- if (!(item.fs instanceof DistributedFileSystem)) {
- throw new UnsupportedActionException(
- "Erasure commands are only supported for the HDFS paths");
- }
- }
-
- /**
- * A command to set the erasure coding policy for a directory, with the name
- * of the policy.
- */
- static class SetECPolicyCommand extends ECCommand {
- public static final String NAME = "setPolicy";
- public static final String USAGE = "[-p <policyName>] <path>";
- public static final String DESCRIPTION =
- "Set a specified erasure coding policy to a directory\n"
- + "Options :\n"
- + " -p <policyName> : erasure coding policy name to encode files. "
- + "If not passed the default policy will be used\n"
- + " <path> : Path to a directory. Under this directory "
- + "files will be encoded using specified erasure coding policy";
- private String ecPolicyName;
- private ErasureCodingPolicy ecPolicy = null;
-
- @Override
- protected void processOptions(LinkedList<String> args) throws IOException {
- ecPolicyName = StringUtils.popOptionWithArgument("-p", args);
- if (args.isEmpty()) {
- throw new HadoopIllegalArgumentException("<path> is missing");
- }
- if (args.size() > 1) {
- throw new HadoopIllegalArgumentException("Too many arguments");
- }
- }
-
- @Override
- protected void processPath(PathData item) throws IOException {
- super.processPath(item);
- DistributedFileSystem dfs = (DistributedFileSystem) item.fs;
- try {
- if (ecPolicyName != null) {
- ErasureCodingPolicy[] ecPolicies = dfs.getClient().getErasureCodingPolicies();
- for (ErasureCodingPolicy ecPolicy : ecPolicies) {
- if (ecPolicyName.equals(ecPolicy.getName())) {
- this.ecPolicy = ecPolicy;
- break;
- }
- }
- if (ecPolicy == null) {
- StringBuilder sb = new StringBuilder();
- sb.append("Policy '");
- sb.append(ecPolicyName);
- sb.append("' does not match any of the supported policies.");
- sb.append(" Please select any one of ");
- List<String> ecPolicyNames = new ArrayList<String>();
- for (ErasureCodingPolicy ecPolicy : ecPolicies) {
- ecPolicyNames.add(ecPolicy.getName());
- }
- sb.append(ecPolicyNames);
- throw new HadoopIllegalArgumentException(sb.toString());
- }
- }
- dfs.setErasureCodingPolicy(item.path, ecPolicy);
- out.println("EC policy set successfully at " + item.path);
- } catch (IOException e) {
- throw new IOException("Unable to set EC policy for the path "
- + item.path + ". " + e.getMessage());
- }
- }
- }
-
- /**
- * Get the erasure coding policy of a file or directory
- */
- static class GetECPolicyCommand extends ECCommand {
- public static final String NAME = "getPolicy";
- public static final String USAGE = "<path>";
- public static final String DESCRIPTION =
- "Get erasure coding policy information about at specified path\n";
-
- @Override
- protected void processOptions(LinkedList<String> args) throws IOException {
- if (args.isEmpty()) {
- throw new HadoopIllegalArgumentException("<path> is missing");
- }
- if (args.size() > 1) {
- throw new HadoopIllegalArgumentException("Too many arguments");
- }
- }
-
- @Override
- protected void processPath(PathData item) throws IOException {
- super.processPath(item);
- DistributedFileSystem dfs = (DistributedFileSystem) item.fs;
- try {
- ErasureCodingPolicy ecPolicy = dfs.getErasureCodingPolicy(item.path);
- if (ecPolicy != null) {
- out.println(ecPolicy.toString());
- } else {
- out.println("Path " + item.path + " is not erasure coded.");
- }
- } catch (IOException e) {
- throw new IOException("Unable to get EC policy for the path "
- + item.path + ". " + e.getMessage());
- }
- }
- }
-
- /**
- * List all supported erasure coding policies
- */
- static class ListPolicies extends ECCommand {
- public static final String NAME = "listPolicies";
- public static final String USAGE = "";
- public static final String DESCRIPTION =
- "Get the list of erasure coding policies supported\n";
-
- @Override
- protected void processOptions(LinkedList<String> args) throws IOException {
- if (!args.isEmpty()) {
- throw new HadoopIllegalArgumentException("Too many parameters");
- }
-
- FileSystem fs = FileSystem.get(getConf());
- if (fs instanceof DistributedFileSystem == false) {
- throw new UnsupportedActionException(
- "Erasure commands are only supported for the HDFS");
- }
- DistributedFileSystem dfs = (DistributedFileSystem) fs;
-
- ErasureCodingPolicy[] ecPolicies = dfs.getClient().getErasureCodingPolicies();
- StringBuilder sb = new StringBuilder();
- int i = 0;
- while (i < ecPolicies.length) {
- ErasureCodingPolicy ecPolicy = ecPolicies[i];
- sb.append(ecPolicy.getName());
- i++;
- if (i < ecPolicies.length) {
- sb.append(", ");
- }
- }
- out.println(sb.toString());
- }
- }
-
- /**
- * Unset the erasure coding policy from a directory.
- */
- static class UnsetECPolicyCommand extends ECCommand {
- public static final String NAME = "unsetPolicy";
- public static final String USAGE = "<path>";
- public static final String DESCRIPTION =
- "Unset erasure coding policy from a directory\n";
-
- @Override
- protected void processOptions(LinkedList<String> args) throws IOException {
- if (args.isEmpty()) {
- throw new HadoopIllegalArgumentException("<path> is missing");
- }
- if (args.size() > 1) {
- throw new HadoopIllegalArgumentException("Too many arguments");
- }
- }
-
- @Override
- protected void processPath(PathData item) throws IOException {
- super.processPath(item);
- DistributedFileSystem dfs = (DistributedFileSystem) item.fs;
- try {
- dfs.unsetErasureCodingPolicy(item.path);
- } catch (IOException e) {
- throw new IOException("Unable to unset EC policy from directory "
- + item.path + ". " + e.getMessage());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 6e4891f..0283e2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -119,30 +119,30 @@ Deployment
HDFS provides an `ec` subcommand to perform administrative commands related to erasure coding.
hdfs ec [generic options]
- [-setPolicy [-p <policyName>] <path>]
- [-getPolicy <path>]
- [-unsetPolicy <path>]
+ [-setPolicy -policy <policyName> -path <path>]
+ [-getPolicy -path <path>]
+ [-unsetPolicy -path <path>]
[-listPolicies]
[-usage [cmd ...]]
[-help [cmd ...]]
Below are the details about each command.
- * `[-setPolicy [-p <policyName>] <path>]`
+ * `[-setPolicy -policy <policyName> -path <path>]`
Sets an ErasureCoding policy on a directory at the specified path.
`path`: An directory in HDFS. This is a mandatory parameter. Setting a policy only affects newly created files, and does not affect existing files.
- `policyName`: The ErasureCoding policy to be used for files under this directory. This is an optional parameter, specified using \u2018-p\u2019 flag. If no policy is specified, the system default ErasureCodingPolicy will be used.
+ `policyName`: The ErasureCoding policy to be used for files under this directory.
- * `[-getPolicy <path>]`
+ * `[-getPolicy -path <path>]`
Get details of the ErasureCoding policy of a file or directory at the specified path.
- * `[-unsetPolicy <path>]`
+ * `[-unsetPolicy -path <path>]`
- Unset an ErasureCoding policy from a directory at the specified path when previously user sets the ErasureCoding policy on this directory via "setPolicy" command. If the directory inherits the ErasureCoding policy from its parent group, "unsetPolicy" command on this directory will not have any effect. Unset ErasureCoding policy on a directory which doesn't have ErasureCoding policy will not return an error.
+ Unset an ErasureCoding policy set by a previous call to "setPolicy" on a directory. If the directory inherits the ErasureCoding policy from an ancestor directory, "unsetPolicy" is a no-op. Unsetting the policy on a directory which doesn't have an explicit policy set will not return an error.
* `[-listPolicies]`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
index 0499a2b..28e5f98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.cli.util.CLITestCmd;
import org.apache.hadoop.cli.util.CommandExecutor;
import org.apache.hadoop.cli.util.ErasureCodingCliCmdExecutor;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.tools.erasurecode.ECCli;
+import org.apache.hadoop.hdfs.tools.ECAdmin;
public class CLITestCmdErasureCoding extends CLITestCmd {
public CLITestCmdErasureCoding(String str, CLICommandTypes type) {
@@ -33,7 +33,7 @@ public class CLITestCmdErasureCoding extends CLITestCmd {
@Override
public CommandExecutor getExecutor(String tag, Configuration conf) throws IllegalArgumentException {
if (getType() instanceof CLICommandErasureCodingCli)
- return new ErasureCodingCliCmdExecutor(tag, new ECCli());
+ return new ErasureCodingCliCmdExecutor(tag, new ECAdmin(conf));
return super.getExecutor(tag, conf);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
index e993313..59b2a73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
@@ -17,14 +17,14 @@
*/
package org.apache.hadoop.cli.util;
-import org.apache.hadoop.hdfs.tools.erasurecode.ECCli;
+import org.apache.hadoop.hdfs.tools.ECAdmin;
import org.apache.hadoop.util.ToolRunner;
public class ErasureCodingCliCmdExecutor extends CommandExecutor {
protected String namenode = null;
- protected ECCli admin = null;
+ protected ECAdmin admin = null;
- public ErasureCodingCliCmdExecutor(String namenode, ECCli admin) {
+ public ErasureCodingCliCmdExecutor(String namenode, ECAdmin admin) {
this.namenode = namenode;
this.admin = admin;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index 0753d4d..82b71def 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -42,7 +42,11 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>-usage: Unknown command</expected-output>
+ <expected-output>Can't understand command '-usage'</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Usage: bin/hdfs ec [COMMAND]</expected-output>
</comparator>
</comparators>
</test>
@@ -58,7 +62,11 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Usage: hdfs ec [generic options]</expected-output>
+ <expected-output>[-listPolicies]</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>[-unsetPolicy -path</expected-output>
</comparator>
</comparators>
</test>
@@ -72,12 +80,12 @@
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^[ \t]*Set a specified erasure coding policy to a directory( )*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>Set the erasure coding policy for a file/directory.</expected-output>
</comparator>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-setPolicy \[-p <policyName>\] <path>(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>[-setPolicy -path <path> -policy <policy>]</expected-output>
</comparator>
</comparators>
</test>
@@ -92,11 +100,11 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Get erasure coding policy information about at specified path</expected-output>
+ <expected-output>Get the erasure coding policy of a file/directory</expected-output>
</comparator>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-getPolicy <path>(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>[-getPolicy -path <path>]</expected-output>
</comparator>
</comparators>
</test>
@@ -111,11 +119,11 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Get the list of erasure coding policies supported</expected-output>
+ <expected-output>Get the list of supported erasure coding policies</expected-output>
</comparator>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-listPolicies (.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>[-listPolicies]</expected-output>
</comparator>
</comparators>
</test>
@@ -125,7 +133,7 @@
<description>setPolicy : set erasure coding policy on a directory to encode files</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -133,7 +141,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>EC policy set successfully at NAMENODE/ecdir</expected-output>
+ <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -142,8 +150,8 @@
<description>setPolicy : set a policy twice</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -151,25 +159,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>EC policy set successfully at NAMENODE/ecdir</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test>
- <description>setPolicy : default policy</description>
- <test-commands>
- <command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
- </test-commands>
- <cleanup-commands>
- <command>-fs NAMENODE -rmdir /ecdir</command>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=</expected-output>
+ <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -178,9 +168,9 @@
<description>unsetPolicy : unset policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -unsetPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -188,7 +178,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>is not erasure coded.</expected-output>
+ <expected-output>is unspecified</expected-output>
</comparator>
</comparators>
</test>
@@ -197,10 +187,9 @@
<description>setPolicy : change different policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-3-2-64k
- /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-3-2-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -208,7 +197,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-3-2-64k</expected-output>
+ <expected-output>RS-DEFAULT-3-2-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -218,10 +207,10 @@
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<command>-fs NAMENODE -mkdir /ecdir/child</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -unsetPolicy /ecdir/child</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/child/ecfile</command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir/child/ecfile</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/child/ecfile</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /ecdir/child/ecfile</command>
@@ -231,7 +220,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -240,7 +229,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /noec</command>
- <ec-admin-command>-fs NAMENODE -getPolicy /noec</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /noec</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /noec</command>
@@ -248,7 +237,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Path NAMENODE/noec is not erasure coded</expected-output>
+ <expected-output>is unspecified</expected-output>
</comparator>
</comparators>
</test>
@@ -257,8 +246,8 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -266,7 +255,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -275,9 +264,9 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/ecfile</command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir/ecfile</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/ecfile</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /ecdir/ecfile</command>
@@ -286,7 +275,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -318,8 +307,8 @@
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-setPolicy: <path> is missing(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>Please specify the path for setting the EC policy.</expected-output>
</comparator>
</comparators>
</test>
@@ -328,15 +317,15 @@
<description>setPolicy : illegal parameters - policy name is missing</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-setPolicy: option -p requires 1 argument(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>option -path requires 1 argument</expected-output>
</comparator>
</comparators>
</test>
@@ -345,7 +334,7 @@
<description>setPolicy : illegal parameters - too many arguments</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir1 /ecdir2</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-DEFAULT-3-2-64k /ecdir2</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -362,7 +351,7 @@
<description>setPolicy : illegal parameters - invalidpolicy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p invalidpolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy invalidpolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -378,14 +367,14 @@
<test>
<description>setPolicy : illegal parameters - no such file</description>
<test-commands>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-DEFAULT-3-2-64k</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^setPolicy: `/ecdir': No such file or directory(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>Path not found: /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -399,8 +388,8 @@
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-getPolicy: <path> is missing(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>Please specify the path with -path</expected-output>
</comparator>
</comparators>
</test>
@@ -408,7 +397,7 @@
<test>
<description>getPolicy : illegal parameters - too many arguments</description>
<test-commands>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /ecdir</command>
@@ -424,14 +413,14 @@
<test>
<description>getPolicy : illegal parameters - no such file</description>
<test-commands>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^getPolicy: `/ecdir': No such file or directory(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>Path not found: /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -446,7 +435,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>-listPolicies: Too many parameters</expected-output>
+ <expected-output>-listPolicies: Too many arguments</expected-output>
</comparator>
</comparators>
</test>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-14114 S3A can no longer
handle unencoded + in URIs. Contributed by Sean Mackrory.
Posted by jh...@apache.org.
HADOOP-14114 S3A can no longer handle unencoded + in URIs. Contributed by Sean Mackrory.
(cherry picked from commit ff87ca84418a710c6dc884fe8c70947fcc6489d5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c22a916
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c22a916
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c22a916
Branch: refs/heads/YARN-5734
Commit: 9c22a91662af24569191ce45289ef8266e8755cc
Parents: 132f758
Author: Steve Loughran <st...@apache.org>
Authored: Fri Feb 24 10:41:36 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Feb 24 10:41:36 2017 +0000
----------------------------------------------------------------------
.../hadoop/fs/s3native/S3xLoginHelper.java | 15 ++++++++++-
.../hadoop/fs/s3native/TestS3xLoginHelper.java | 28 ++++++++++++++++++++
2 files changed, 42 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c22a916/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
index 97ece37..862ce6b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
@@ -48,6 +48,13 @@ public final class S3xLoginHelper {
"The Filesystem URI contains login details."
+" This is insecure and may be unsupported in future.";
+ public static final String PLUS_WARNING =
+ "Secret key contains a special character that should be URL encoded! " +
+ "Attempting to resolve...";
+
+ public static final String PLUS_UNENCODED = "+";
+ public static final String PLUS_ENCODED = "%2B";
+
/**
* Build the filesystem URI. This can include stripping down of part
* of the URI.
@@ -112,7 +119,13 @@ public final class S3xLoginHelper {
int loginSplit = login.indexOf(':');
if (loginSplit > 0) {
String user = login.substring(0, loginSplit);
- String password = URLDecoder.decode(login.substring(loginSplit + 1),
+ String encodedPassword = login.substring(loginSplit + 1);
+ if (encodedPassword.contains(PLUS_UNENCODED)) {
+ LOG.warn(PLUS_WARNING);
+ encodedPassword = encodedPassword.replaceAll("\\" + PLUS_UNENCODED,
+ PLUS_ENCODED);
+ }
+ String password = URLDecoder.decode(encodedPassword,
"UTF-8");
return new Login(user, password);
} else if (loginSplit == 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c22a916/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java
index bd2ac1e..3761cb7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java
@@ -32,9 +32,13 @@ public class TestS3xLoginHelper extends Assert {
public static final String BUCKET = "s3a://bucket";
private static final URI ENDPOINT = uri(BUCKET);
public static final String S = "%2f";
+ public static final String P = "%2b";
+ public static final String P_RAW = "+";
public static final String USER = "user";
public static final String PASS = "pass";
public static final String PASLASHSLASH = "pa" + S + S;
+ public static final String PAPLUS = "pa" + P;
+ public static final String PAPLUS_RAW = "pa" + P_RAW;
public static final URI WITH_USER_AND_PASS = uri("s3a://user:pass@bucket");
public static final Path PATH_WITH_LOGIN =
@@ -42,6 +46,10 @@ public class TestS3xLoginHelper extends Assert {
public static final URI WITH_SLASH_IN_PASS = uri(
"s3a://user:" + PASLASHSLASH + "@bucket");
+ public static final URI WITH_PLUS_IN_PASS = uri(
+ "s3a://user:" + PAPLUS + "@bucket");
+ public static final URI WITH_PLUS_RAW_IN_PASS = uri(
+ "s3a://user:" + PAPLUS_RAW + "@bucket");
public static final URI USER_NO_PASS = uri("s3a://user@bucket");
public static final URI WITH_USER_AND_COLON = uri("s3a://user:@bucket");
public static final URI NO_USER = uri("s3a://:pass@bucket");
@@ -117,6 +125,16 @@ public class TestS3xLoginHelper extends Assert {
}
@Test
+ public void testLoginWithPlusInPass() throws Throwable {
+ assertMatchesLogin(USER, "pa+", WITH_PLUS_IN_PASS);
+ }
+
+ @Test
+ public void testLoginWithPlusRawInPass() throws Throwable {
+ assertMatchesLogin(USER, "pa+", WITH_PLUS_RAW_IN_PASS);
+ }
+
+ @Test
public void testLoginWithUser() throws Throwable {
assertMatchesLogin(USER, "", USER_NO_PASS);
}
@@ -152,6 +170,16 @@ public class TestS3xLoginHelper extends Assert {
}
@Test
+ public void testFsUriWithPlusInPass() throws Throwable {
+ assertMatchesEndpoint(WITH_PLUS_IN_PASS);
+ }
+
+ @Test
+ public void testFsUriWithPlusRawInPass() throws Throwable {
+ assertMatchesEndpoint(WITH_PLUS_RAW_IN_PASS);
+ }
+
+ @Test
public void testFsUriWithUser() throws Throwable {
assertMatchesEndpoint(USER_NO_PASS);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDFS-11478. Update EC commands in
HDFSCommands.md. Contributed by Yiqun Lin.
Posted by jh...@apache.org.
HDFS-11478. Update EC commands in HDFSCommands.md. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/555d0c39
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/555d0c39
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/555d0c39
Branch: refs/heads/YARN-5734
Commit: 555d0c39950078e80a373f188c3b1529995d0af7
Parents: 6f6dfe0
Author: Yiqun Lin <yq...@apache.org>
Authored: Thu Mar 2 12:10:24 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Thu Mar 2 12:10:24 2017 +0800
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/555d0c39/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index b65cc78..d031dad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -436,8 +436,9 @@ Runs the diskbalancer CLI. See [HDFS Diskbalancer](./HDFSDiskbalancer.html) for
Usage:
hdfs ec [generic options]
- [-setPolicy [-p <policyName>] <path>]
- [-getPolicy <path>]
+ [-setPolicy -policy <policyName> -path <path>]
+ [-getPolicy -path <path>]
+ [-unsetPolicy -path <path>]
[-listPolicies]
[-usage [cmd ...]]
[-help [cmd ...]]
@@ -446,6 +447,7 @@ Usage:
|:---- |:---- |
|-setPolicy| Set a specified ErasureCoding policy to a directory|
|-getPolicy| Get ErasureCoding policy information about a specified path|
+|-unsetPolicy| Unset an ErasureCoding policy set by a previous call to "setPolicy" on a directory |
|-listPolicies| Lists all supported ErasureCoding policies|
Runs the ErasureCoding CLI. See [HDFS ErasureCoding](./HDFSErasureCoding.html#Administrative_commands) for more information on this command.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-5946: Create
YarnConfigurationStore interface and InMemoryConfigurationStore class.
Contributed by Jonathan Hung
Posted by jh...@apache.org.
YARN-5946: Create YarnConfigurationStore interface and
InMemoryConfigurationStore class. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b869d92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b869d92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b869d92
Branch: refs/heads/YARN-5734
Commit: 0b869d92544ad76fbd2fd03125cc8346c6874b71
Parents: d5d40ae
Author: Xuan <xg...@apache.org>
Authored: Fri Feb 24 15:58:12 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Thu Mar 2 18:47:45 2017 -0800
----------------------------------------------------------------------
.../conf/InMemoryConfigurationStore.java | 86 +++++++++++
.../capacity/conf/YarnConfigurationStore.java | 154 +++++++++++++++++++
.../conf/TestYarnConfigurationStore.java | 70 +++++++++
3 files changed, 310 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b869d92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
new file mode 100644
index 0000000..a208fb9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A default implementation of {@link YarnConfigurationStore}. Doesn't offer
+ * persistent configuration storage, just stores the configuration in memory.
+ */
+public class InMemoryConfigurationStore implements YarnConfigurationStore {
+
+ private Configuration schedConf;
+ private LinkedList<LogMutation> pendingMutations;
+ private long pendingId;
+
+ @Override
+ public void initialize(Configuration conf, Configuration schedConf) {
+ this.schedConf = schedConf;
+ this.pendingMutations = new LinkedList<>();
+ this.pendingId = 0;
+ }
+
+ @Override
+ public synchronized long logMutation(LogMutation logMutation) {
+ logMutation.setId(++pendingId);
+ pendingMutations.add(logMutation);
+ return pendingId;
+ }
+
+ @Override
+ public synchronized boolean confirmMutation(long id, boolean isValid) {
+ LogMutation mutation = pendingMutations.poll();
+ // If confirmMutation is called out of order, discard mutations until id
+ // is reached.
+ while (mutation != null) {
+ if (mutation.getId() == id) {
+ if (isValid) {
+ Map<String, String> mutations = mutation.getUpdates();
+ for (Map.Entry<String, String> kv : mutations.entrySet()) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ }
+ return true;
+ }
+ mutation = pendingMutations.poll();
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized Configuration retrieve() {
+ return schedConf;
+ }
+
+ @Override
+ public synchronized List<LogMutation> getPendingMutations() {
+ return pendingMutations;
+ }
+
+ @Override
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ // Unimplemented.
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b869d92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
new file mode 100644
index 0000000..22c0ef8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * YarnConfigurationStore exposes the methods needed for retrieving and
+ * persisting {@link CapacityScheduler} configuration via key-value
+ * using write-ahead logging. When configuration mutation is requested, caller
+ * should first log it with {@code logMutation}, which persists this pending
+ * mutation. This mutation is merged to the persisted configuration only after
+ * {@code confirmMutation} is called.
+ *
+ * On startup/recovery, caller should call {@code retrieve} to get all
+ * confirmed mutations, then get pending mutations which were not confirmed via
+ * {@code getPendingMutations}, and replay/confirm them via
+ * {@code confirmMutation} as in the normal case.
+ */
+public interface YarnConfigurationStore {
+
+ /**
+ * LogMutation encapsulates the fields needed for configuration mutation
+ * audit logging and recovery.
+ */
+ class LogMutation {
+ private Map<String, String> updates;
+ private String user;
+ private long id;
+
+ /**
+ * Create log mutation prior to logging.
+ * @param updates key-value configuration updates
+ * @param user user who requested configuration change
+ */
+ public LogMutation(Map<String, String> updates, String user) {
+ this(updates, user, 0);
+ }
+
+ /**
+ * Create log mutation for recovery.
+ * @param updates key-value configuration updates
+ * @param user user who requested configuration change
+ * @param id transaction id of configuration change
+ */
+ LogMutation(Map<String, String> updates, String user, long id) {
+ this.updates = updates;
+ this.user = user;
+ this.id = id;
+ }
+
+ /**
+ * Get key-value configuration updates.
+ * @return map of configuration updates
+ */
+ public Map<String, String> getUpdates() {
+ return updates;
+ }
+
+ /**
+ * Get user who requested configuration change.
+ * @return user who requested configuration change
+ */
+ public String getUser() {
+ return user;
+ }
+
+ /**
+ * Get transaction id of this configuration change.
+ * @return transaction id
+ */
+ public long getId() {
+ return id;
+ }
+
+ /**
+ * Set transaction id of this configuration change.
+ * @param id transaction id
+ */
+ public void setId(long id) {
+ this.id = id;
+ }
+ }
+
+ /**
+ * Initialize the configuration store.
+ * @param conf configuration to initialize store with
+ * @param schedConf Initial key-value configuration to persist
+ */
+ void initialize(Configuration conf, Configuration schedConf);
+
+ /**
+ * Logs the configuration change to backing store. Generates an id associated
+ * with this mutation, sets it in {@code logMutation}, and returns it.
+ * @param logMutation configuration change to be persisted in write ahead log
+ * @return id which configuration store associates with this mutation
+ */
+ long logMutation(LogMutation logMutation);
+
+ /**
+ * Should be called after {@code logMutation}. Gets the pending mutation
+ * associated with {@code id} and marks the mutation as persisted (no longer
+ * pending). If isValid is true, merge the mutation with the persisted
+ * configuration.
+ *
+ * If {@code confirmMutation} is called with ids in a different order than
+ * was returned by {@code logMutation}, the result is implementation
+ * dependent.
+ * @param id id of mutation to be confirmed
+ * @param isValid if true, update persisted configuration with mutation
+ * associated with {@code id}.
+ * @return true on success
+ */
+ boolean confirmMutation(long id, boolean isValid);
+
+ /**
+ * Retrieve the persisted configuration.
+ * @return configuration as key-value
+ */
+ Configuration retrieve();
+
+ /**
+ * Get the list of pending mutations, in the order they were logged.
+ * @return list of mutations
+ */
+ List<LogMutation> getPendingMutations();
+
+ /**
+ * Get a list of confirmed configuration mutations starting from a given id.
+ * @param fromId id from which to start getting mutations, inclusive
+ * @return list of configuration mutations
+ */
+ List<LogMutation> getConfirmedConfHistory(long fromId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b869d92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
new file mode 100644
index 0000000..dff4e77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class TestYarnConfigurationStore {
+
+ private YarnConfigurationStore confStore;
+ private Configuration schedConf;
+
+ private static final String testUser = "testUser";
+
+ @Before
+ public void setUp() {
+ schedConf = new Configuration(false);
+ schedConf.set("key1", "val1");
+ }
+
+ @Test
+ public void testInMemoryConfigurationStore() {
+ confStore = new InMemoryConfigurationStore();
+ confStore.initialize(new Configuration(), schedConf);
+ assertEquals("val1", confStore.retrieve().get("key1"));
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("keyUpdate1", "valUpdate1");
+ LogMutation mutation1 = new LogMutation(update1, testUser);
+ long id = confStore.logMutation(mutation1);
+ assertEquals(1, confStore.getPendingMutations().size());
+ confStore.confirmMutation(id, true);
+ assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1"));
+ assertEquals(0, confStore.getPendingMutations().size());
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("keyUpdate2", "valUpdate2");
+ LogMutation mutation2 = new LogMutation(update2, testUser);
+ id = confStore.logMutation(mutation2);
+ assertEquals(1, confStore.getPendingMutations().size());
+ confStore.confirmMutation(id, false);
+ assertNull("Configuration should not be updated",
+ confStore.retrieve().get("keyUpdate2"));
+ assertEquals(0, confStore.getPendingMutations().size());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDFS-11428. Change
setErasureCodingPolicy to take a required string EC policy name. Contributed
by Andrew Wang.
Posted by jh...@apache.org.
HDFS-11428. Change setErasureCodingPolicy to take a required string EC policy name. Contributed by Andrew Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82ef9acc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82ef9acc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82ef9acc
Branch: refs/heads/YARN-5734
Commit: 82ef9accafe7318278efb169678e17065e082c8e
Parents: dcd03df
Author: Rakesh Radhakrishnan <ra...@apache.org>
Authored: Wed Mar 1 16:06:02 2017 +0530
Committer: Rakesh Radhakrishnan <ra...@apache.org>
Committed: Wed Mar 1 16:06:02 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +--
.../hadoop/hdfs/DistributedFileSystem.java | 11 +++---
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 10 +++---
.../hadoop/hdfs/protocol/ClientProtocol.java | 5 ++-
.../ClientNamenodeProtocolTranslatorPB.java | 6 ++--
.../src/main/proto/erasurecoding.proto | 2 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 4 +--
.../server/namenode/FSDirErasureCodingOp.java | 13 +++++--
.../hdfs/server/namenode/FSNamesystem.java | 9 ++---
.../hdfs/server/namenode/NameNodeRpcServer.java | 4 +--
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 26 +-------------
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 6 ++--
.../hdfs/ErasureCodeBenchmarkThroughput.java | 3 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 3 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 3 +-
.../TestDFSStripedOutputStreamWithFailure.java | 2 +-
.../hdfs/TestDecommissionWithStriped.java | 3 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 37 +++++++++++---------
.../TestErasureCodingPolicyWithSnapshot.java | 10 +++---
.../apache/hadoop/hdfs/TestFileChecksum.java | 3 +-
.../hadoop/hdfs/TestFileStatusWithECPolicy.java | 2 +-
.../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +-
.../hdfs/TestReadStripedFileWithDecoding.java | 3 +-
.../TestReadStripedFileWithMissingBlocks.java | 2 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 3 +-
.../hdfs/TestSafeModeWithStripedFile.java | 3 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 18 +++++-----
.../hadoop/hdfs/TestWriteReadStripedFile.java | 3 +-
.../hdfs/TestWriteStripedFileWithFailure.java | 3 +-
.../hdfs/server/balancer/TestBalancer.java | 3 +-
.../TestBlockTokenWithDFSStriped.java | 4 +--
...constructStripedBlocksWithRackAwareness.java | 9 +++--
.../TestSequentialBlockGroupId.java | 4 +--
.../TestDataNodeErasureCodingMetrics.java | 3 +-
.../hadoop/hdfs/server/mover/TestMover.java | 3 +-
.../TestAddOverReplicatedStripedBlocks.java | 3 +-
.../namenode/TestAddStripedBlockInFBR.java | 3 +-
.../server/namenode/TestAddStripedBlocks.java | 3 +-
.../server/namenode/TestFSEditLogLoader.java | 4 +--
.../hdfs/server/namenode/TestFSImage.java | 6 ++--
.../hadoop/hdfs/server/namenode/TestFsck.java | 8 +++--
.../server/namenode/TestNameNodeMXBean.java | 3 +-
.../namenode/TestQuotaWithStripedBlocks.java | 3 +-
.../namenode/TestReconstructStripedBlocks.java | 6 ++--
.../server/namenode/TestStripedINodeFile.java | 6 ++--
.../TestOfflineImageViewer.java | 3 +-
...TestOfflineImageViewerWithStripedBlocks.java | 3 +-
.../test/resources/testErasureCodingConf.xml | 2 +-
.../java/org/apache/hadoop/fs/TestDFSIO.java | 2 +-
49 files changed, 152 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index e0ccd62..ae1d821 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2611,12 +2611,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
- public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
+ public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
checkOpen();
try (TraceScope ignored =
newPathTraceScope("setErasureCodingPolicy", src)) {
- namenode.setErasureCodingPolicy(src, ecPolicy);
+ namenode.setErasureCodingPolicy(src, ecPolicyName);
} catch (RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
SafeModeException.class,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 30dcfa4..1eef560 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2397,17 +2397,18 @@ public class DistributedFileSystem extends FileSystem {
* Set the source path to the specified erasure coding policy.
*
* @param path The directory to set the policy
- * @param ecPolicy The erasure coding policy. If not specified default will
- * be used.
+ * @param ecPolicyName The erasure coding policy name.
* @throws IOException
*/
public void setErasureCodingPolicy(final Path path,
- final ErasureCodingPolicy ecPolicy) throws IOException {
+ final String ecPolicyName) throws IOException {
Path absF = fixRelativePart(path);
+ Preconditions.checkNotNull(ecPolicyName, "Erasure coding policy cannot be" +
+ " null.");
new FileSystemLinkResolver<Void>() {
@Override
public Void doCall(final Path p) throws IOException {
- dfs.setErasureCodingPolicy(getPathName(p), ecPolicy);
+ dfs.setErasureCodingPolicy(getPathName(p), ecPolicyName);
return null;
}
@@ -2415,7 +2416,7 @@ public class DistributedFileSystem extends FileSystem {
public Void next(final FileSystem fs, final Path p) throws IOException {
if (fs instanceof DistributedFileSystem) {
DistributedFileSystem myDfs = (DistributedFileSystem) fs;
- myDfs.setErasureCodingPolicy(p, ecPolicy);
+ myDfs.setErasureCodingPolicy(p, ecPolicyName);
return null;
}
throw new UnsupportedOperationException(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index a192fa8..456f280 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -465,13 +465,15 @@ public class HdfsAdmin {
* Set the source path to the specified erasure coding policy.
*
* @param path The source path referring to a directory.
- * @param ecPolicy The erasure coding policy for the directory.
- * If null, the default will be used.
+ * @param ecPolicyName The erasure coding policy name for the directory.
+ *
* @throws IOException
+ * @throws HadoopIllegalArgumentException if the specified EC policy is not
+ * enabled on the cluster
*/
public void setErasureCodingPolicy(final Path path,
- final ErasureCodingPolicy ecPolicy) throws IOException {
- dfs.setErasureCodingPolicy(path, ecPolicy);
+ final String ecPolicyName) throws IOException {
+ dfs.setErasureCodingPolicy(path, ecPolicyName);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index eaebd6f..041d226 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1510,11 +1510,10 @@ public interface ClientProtocol {
/**
* Set an erasure coding policy on a specified path.
* @param src The path to set policy on.
- * @param ecPolicy The erasure coding policy. If null, default policy will
- * be used
+ * @param ecPolicyName The erasure coding policy name.
*/
@AtMostOnce
- void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
+ void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index de474b5..da4a17f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -1459,14 +1459,12 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
- public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
+ public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
final SetErasureCodingPolicyRequestProto.Builder builder =
SetErasureCodingPolicyRequestProto.newBuilder();
builder.setSrc(src);
- if (ecPolicy != null) {
- builder.setEcPolicy(PBHelperClient.convertErasureCodingPolicy(ecPolicy));
- }
+ builder.setEcPolicyName(ecPolicyName);
SetErasureCodingPolicyRequestProto req = builder.build();
try {
rpcProxy.setErasureCodingPolicy(null, req);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
index 7ea8d32..4eab4d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
@@ -25,7 +25,7 @@ import "hdfs.proto";
message SetErasureCodingPolicyRequestProto {
required string src = 1;
- optional ErasureCodingPolicyProto ecPolicy = 2;
+ required string ecPolicyName = 2;
}
message SetErasureCodingPolicyResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 1aa15c8..1944fe7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -1442,9 +1442,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RpcController controller, SetErasureCodingPolicyRequestProto req)
throws ServiceException {
try {
- ErasureCodingPolicy ecPolicy = req.hasEcPolicy() ?
- PBHelperClient.convertErasureCodingPolicy(req.getEcPolicy()) : null;
- server.setErasureCodingPolicy(req.getSrc(), ecPolicy);
+ server.setErasureCodingPolicy(req.getSrc(), req.getEcPolicyName());
return SetErasureCodingPolicyResponseProto.newBuilder().build();
} catch (IOException e) {
throw new ServiceException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 0ab8c89..f4c4ad7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -59,14 +59,16 @@ final class FSDirErasureCodingOp {
*
* @param fsn The namespace
* @param srcArg The path of the target directory.
- * @param ecPolicy The erasure coding policy to set on the target directory.
+ * @param ecPolicyName The erasure coding policy name to set on the target
+ * directory.
* @param logRetryCache whether to record RPC ids in editlog for retry
* cache rebuilding
* @return {@link HdfsFileStatus}
* @throws IOException
+ * @throws HadoopIllegalArgumentException if the policy is not enabled
*/
static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn,
- final String srcArg, final ErasureCodingPolicy ecPolicy,
+ final String srcArg, final String ecPolicyName,
final boolean logRetryCache) throws IOException {
assert fsn.hasWriteLock();
@@ -78,6 +80,13 @@ final class FSDirErasureCodingOp {
List<XAttr> xAttrs;
fsd.writeLock();
try {
+ ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicyManager()
+ .getPolicyByName(ecPolicyName);
+ if (ecPolicy == null) {
+ throw new HadoopIllegalArgumentException("Policy '" +
+ ecPolicyName + "' does not match any supported erasure coding " +
+ "policies.");
+ }
iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
src = iip.getPath();
xAttrs = setErasureCodingPolicyXAttr(fsn, iip, ecPolicy);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 38a326c..254ccfa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -6772,13 +6772,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Set an erasure coding policy on the given path.
* @param srcArg The path of the target directory.
- * @param ecPolicy The erasure coding policy to set on the target directory.
+ * @param ecPolicyName The erasure coding policy to set on the target
+ * directory.
* @throws AccessControlException if the caller is not the superuser.
* @throws UnresolvedLinkException if the path can't be resolved.
* @throws SafeModeException if the Namenode is in safe mode.
*/
- void setErasureCodingPolicy(final String srcArg, final ErasureCodingPolicy
- ecPolicy, final boolean logRetryCache) throws IOException,
+ void setErasureCodingPolicy(final String srcArg, final String ecPolicyName,
+ final boolean logRetryCache) throws IOException,
UnresolvedLinkException, SafeModeException, AccessControlException {
final String operationName = "setErasureCodingPolicy";
checkSuperuserPrivilege();
@@ -6790,7 +6791,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg);
resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this,
- srcArg, ecPolicy, logRetryCache);
+ srcArg, ecPolicyName, logRetryCache);
success = true;
} catch (AccessControlException ace) {
logAuditEvent(success, operationName, srcArg, null,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index f9cfa42..cf14e8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -2018,7 +2018,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
}
@Override // ClientProtocol
- public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
+ public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
checkNNStartup();
final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
@@ -2027,7 +2027,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
}
boolean success = false;
try {
- namesystem.setErasureCodingPolicy(src, ecPolicy, cacheEntry != null);
+ namesystem.setErasureCodingPolicy(src, ecPolicyName, cacheEntry != null);
success = true;
} finally {
RetryCache.setState(cacheEntry, success);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 29c65b1..548f754 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
@@ -227,30 +226,7 @@ public class ECAdmin extends Configured implements Tool {
final Path p = new Path(path);
final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
try {
- ErasureCodingPolicy ecPolicy = null;
- ErasureCodingPolicy[] ecPolicies =
- dfs.getClient().getErasureCodingPolicies();
- for (ErasureCodingPolicy policy : ecPolicies) {
- if (ecPolicyName.equals(policy.getName())) {
- ecPolicy = policy;
- break;
- }
- }
- if (ecPolicy == null) {
- StringBuilder sb = new StringBuilder();
- sb.append("Policy '");
- sb.append(ecPolicyName);
- sb.append("' does not match any of the supported policies.");
- sb.append(" Please select any one of ");
- List<String> ecPolicyNames = new ArrayList<String>();
- for (ErasureCodingPolicy policy : ecPolicies) {
- ecPolicyNames.add(policy.getName());
- }
- sb.append(ecPolicyNames);
- System.err.println(sb.toString());
- return 3;
- }
- dfs.setErasureCodingPolicy(p, ecPolicy);
+ dfs.setErasureCodingPolicy(p, ecPolicyName);
System.out.println("Set erasure coding policy " + ecPolicyName +
" on " + path);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 1fbc1d9..60a1a4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -136,6 +136,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -1898,7 +1899,7 @@ public class DFSTestUtil {
Path dir, int numBlocks, int numStripesPerBlk, boolean toMkdir)
throws Exception {
createStripedFile(cluster, file, dir, numBlocks, numStripesPerBlk,
- toMkdir, null);
+ toMkdir, ErasureCodingPolicyManager.getSystemDefaultPolicy());
}
/**
@@ -1922,7 +1923,8 @@ public class DFSTestUtil {
assert dir != null;
dfs.mkdirs(dir);
try {
- dfs.getClient().setErasureCodingPolicy(dir.toString(), ecPolicy);
+ dfs.getClient()
+ .setErasureCodingPolicy(dir.toString(), ecPolicy.getName());
} catch (IOException e) {
if (!e.getMessage().contains("non-empty directory")) {
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
index 13dc997..d1a7569 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
@@ -192,7 +192,8 @@ public class ErasureCodeBenchmarkThroughput
}
if (!dfs.exists(ecPath)) {
dfs.mkdirs(ecPath);
- dfs.getClient().setErasureCodingPolicy(ecPath.toString(), ecPolicy);
+ dfs.getClient()
+ .setErasureCodingPolicy(ecPath.toString(), ecPolicy.getName());
} else {
Preconditions.checkArgument(
dfs.getClient().
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 9b69904..50b366b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -108,7 +108,8 @@ public class TestDFSStripedInputStream {
}
fs = cluster.getFileSystem();
fs.mkdirs(dirPath);
- fs.getClient().setErasureCodingPolicy(dirPath.toString(), ecPolicy);
+ fs.getClient()
+ .setErasureCodingPolicy(dirPath.toString(), ecPolicy.getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index 8834e14..5956cc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -89,7 +89,8 @@ public class TestDFSStripedOutputStream {
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy
+ .getName());
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index fabd417..01bd3c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -221,7 +221,7 @@ public class TestDFSStripedOutputStreamWithFailure {
cluster.waitActive();
dfs = cluster.getFileSystem();
dfs.mkdirs(dir);
- dfs.setErasureCodingPolicy(dir, ecPolicy);
+ dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
}
private void tearDown() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
index 2eab309..f8ed0c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
@@ -142,7 +142,8 @@ public class TestDecommissionWithStriped {
client = getDfsClient(cluster.getNameNode(0), conf);
dfs.mkdirs(ecDir);
- dfs.setErasureCodingPolicy(ecDir, null);
+ dfs.setErasureCodingPolicy(ecDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 0e431fa..8608626 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -47,6 +47,8 @@ public class TestErasureCodingPolicies {
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private static final int BLOCK_SIZE = 1024;
+ private static final ErasureCodingPolicy EC_POLICY =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
private FSNamesystem namesystem;
@Before
@@ -80,7 +82,8 @@ public class TestErasureCodingPolicies {
DFSTestUtil.createFile(fs, replicatedFile, 0, (short) 3, 0L);
// set ec policy on dir
- fs.setErasureCodingPolicy(dir, null);
+ fs.setErasureCodingPolicy(dir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// create a file which should be using ec
final Path ecSubDir = new Path(dir, "ecSubDir");
final Path ecFile = new Path(ecSubDir, "ecFile");
@@ -132,7 +135,7 @@ public class TestErasureCodingPolicies {
fs.mkdir(testDir, FsPermission.getDirDefault());
/* Normal creation of an erasure coding directory */
- fs.getClient().setErasureCodingPolicy(testDir.toString(), null);
+ fs.setErasureCodingPolicy(testDir, EC_POLICY.getName());
/* Verify files under the directory are striped */
final Path ECFilePath = new Path(testDir, "foo");
@@ -148,7 +151,7 @@ public class TestErasureCodingPolicies {
fs.mkdir(notEmpty, FsPermission.getDirDefault());
final Path oldFile = new Path(notEmpty, "old");
fs.create(oldFile);
- fs.getClient().setErasureCodingPolicy(notEmpty.toString(), null);
+ fs.setErasureCodingPolicy(notEmpty, EC_POLICY.getName());
final Path newFile = new Path(notEmpty, "new");
fs.create(newFile);
INode oldInode = namesystem.getFSDirectory().getINode(oldFile.toString());
@@ -160,10 +163,10 @@ public class TestErasureCodingPolicies {
final Path dir1 = new Path("/dir1");
final Path dir2 = new Path(dir1, "dir2");
fs.mkdir(dir1, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(dir1.toString(), null);
+ fs.setErasureCodingPolicy(dir1, EC_POLICY.getName());
fs.mkdir(dir2, FsPermission.getDirDefault());
try {
- fs.getClient().setErasureCodingPolicy(dir2.toString(), null);
+ fs.setErasureCodingPolicy(dir2, EC_POLICY.getName());
} catch (IOException e) {
fail("Nested erasure coding policies are supported");
}
@@ -172,7 +175,7 @@ public class TestErasureCodingPolicies {
final Path fPath = new Path("/file");
fs.create(fPath);
try {
- fs.getClient().setErasureCodingPolicy(fPath.toString(), null);
+ fs.setErasureCodingPolicy(fPath, EC_POLICY.getName());
fail("Erasure coding policy on file");
} catch (IOException e) {
assertExceptionContains("erasure coding policy for a file", e);
@@ -185,8 +188,8 @@ public class TestErasureCodingPolicies {
final Path dstECDir = new Path("/dstEC");
fs.mkdir(srcECDir, FsPermission.getDirDefault());
fs.mkdir(dstECDir, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(srcECDir.toString(), null);
- fs.getClient().setErasureCodingPolicy(dstECDir.toString(), null);
+ fs.setErasureCodingPolicy(srcECDir, EC_POLICY.getName());
+ fs.setErasureCodingPolicy(dstECDir, EC_POLICY.getName());
final Path srcFile = new Path(srcECDir, "foo");
fs.create(srcFile);
@@ -220,7 +223,8 @@ public class TestErasureCodingPolicies {
public void testReplication() throws IOException {
final Path testDir = new Path("/ec");
fs.mkdir(testDir, FsPermission.getDirDefault());
- fs.setErasureCodingPolicy(testDir, null);
+ fs.setErasureCodingPolicy(testDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
final Path fooFile = new Path(testDir, "foo");
// create ec file with replication=0
fs.create(fooFile, FsPermission.getFileDefault(), true,
@@ -241,8 +245,9 @@ public class TestErasureCodingPolicies {
// dir EC policy should be null
assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
// dir EC policy after setting
- fs.getClient().setErasureCodingPolicy(src, null); //Default one will be used.
- ErasureCodingPolicy sysDefaultECPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ ErasureCodingPolicy sysDefaultECPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ fs.getClient().setErasureCodingPolicy(src, sysDefaultECPolicy.getName());
verifyErasureCodingInfo(src, sysDefaultECPolicy);
fs.create(new Path(ecDir, "child1")).close();
// verify for the files in ec dir
@@ -263,7 +268,7 @@ public class TestErasureCodingPolicies {
// dir ECInfo before being set
assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
// dir ECInfo after set
- fs.getClient().setErasureCodingPolicy(src, usingECPolicy);
+ fs.getClient().setErasureCodingPolicy(src, usingECPolicy.getName());
verifyErasureCodingInfo(src, usingECPolicy);
fs.create(new Path(ecDir, "child1")).close();
// verify for the files in ec dir
@@ -291,12 +296,12 @@ public class TestErasureCodingPolicies {
final Path ecDir = new Path(src);
try {
fs.mkdir(ecDir, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(src, ecPolicy);
+ fs.getClient().setErasureCodingPolicy(src, ecPolicy.getName());
fail("HadoopIllegalArgumentException should be thrown for"
+ "setting an invalid erasure coding policy");
} catch (Exception e) {
- assertExceptionContains("Policy [ RS-4-2-128k ] does not match " +
- "any of the supported policies",e);
+ assertExceptionContains("Policy 'RS-4-2-128k' does not match " +
+ "any supported erasure coding policies",e);
}
}
@@ -338,7 +343,7 @@ public class TestErasureCodingPolicies {
for (ErasureCodingPolicy policy : sysPolicies) {
Path dir = new Path("/policy_" + policy.getId());
fs.mkdir(dir, FsPermission.getDefault());
- fs.setErasureCodingPolicy(dir, policy);
+ fs.setErasureCodingPolicy(dir, policy.getName());
Path file = new Path(dir, "child");
fs.create(file).close();
assertEquals(policy, fs.getErasureCodingPolicy(file));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
index e02c0bf..6cf4ef4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
@@ -75,7 +75,7 @@ public class TestErasureCodingPolicyWithSnapshot {
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDirParent);
// set erasure coding policy
- fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
String contents = DFSTestUtil.readFile(fs, ecFile);
final Path snap1 = fs.createSnapshot(ecDirParent, "snap1");
@@ -93,7 +93,7 @@ public class TestErasureCodingPolicyWithSnapshot {
fs.getErasureCodingPolicy(snap2ECDir));
// Make dir again with system default ec policy
- fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
final Path snap3 = fs.createSnapshot(ecDirParent, "snap3");
final Path snap3ECDir = new Path(snap3, ecDir.getName());
// Check that snap3's ECPolicy has the correct settings
@@ -134,7 +134,7 @@ public class TestErasureCodingPolicyWithSnapshot {
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDir);
- fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap1));
@@ -150,7 +150,7 @@ public class TestErasureCodingPolicyWithSnapshot {
fs.allowSnapshot(ecDir);
// set erasure coding policy
- fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1);
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
@@ -182,7 +182,7 @@ public class TestErasureCodingPolicyWithSnapshot {
fs.allowSnapshot(ecDir);
// set erasure coding policy
- fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
index 9d60798..8ae176f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
@@ -81,7 +81,8 @@ public class TestFileChecksum {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
Path ecPath = new Path(ecDir);
cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault());
- cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir, null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
client = fs.getClient();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
index 2210fbe..d7b7327 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
@@ -73,7 +73,7 @@ public class TestFileStatusWithECPolicy {
final ErasureCodingPolicy ecPolicy1 = ErasureCodingPolicyManager.getSystemDefaultPolicy();
// set EC policy on dir
- fs.setErasureCodingPolicy(dir, ecPolicy1);
+ fs.setErasureCodingPolicy(dir, ecPolicy1.getName());
final ErasureCodingPolicy ecPolicy2 = client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy();
assertNotNull(ecPolicy2);
assertTrue(ecPolicy1.equals(ecPolicy2));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index 246818b..710ff62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -94,7 +94,7 @@ public class TestLeaseRecoveryStriped {
cluster.waitActive();
dfs = cluster.getFileSystem();
dfs.mkdirs(dir);
- dfs.setErasureCodingPolicy(dir, ecPolicy);
+ dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
index 506a005..d7d9cc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
@@ -102,7 +102,8 @@ public class TestReadStripedFileWithDecoding {
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
index 2ee8663..b65626e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
@@ -61,7 +61,7 @@ public class TestReadStripedFileWithMissingBlocks {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy(
- "/", ecPolicy);
+ "/", ecPolicy.getName());
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index 08b53c2..38939f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -107,7 +107,8 @@ public class TestReconstructStripedFile {
cluster.waitActive();
fs = cluster.getFileSystem();
- fs.getClient().setErasureCodingPolicy("/", null);
+ fs.getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
List<DataNode> datanodes = cluster.getDataNodes();
for (int i = 0; i < dnNum; i++) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
index 2d37c06..0731779 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
@@ -63,7 +63,8 @@ public class TestSafeModeWithStripedFile {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
cluster.waitActive();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
index f26aa06..86624cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -101,7 +101,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
// Test unset a directory which has no EC policy
fs.unsetErasureCodingPolicy(dirPath);
// Set EC policy on directory
- fs.setErasureCodingPolicy(dirPath, ecPolicy);
+ fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
fs.unsetErasureCodingPolicy(dirPath);
@@ -142,12 +142,12 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
.getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID);
fs.mkdirs(parentDir);
- fs.setErasureCodingPolicy(parentDir, ecPolicy);
+ fs.setErasureCodingPolicy(parentDir, ecPolicy.getName());
fs.mkdirs(childDir);
// Create RS(6,3) EC policy file
DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L);
// Set RS(3,2) EC policy on child directory
- fs.setErasureCodingPolicy(childDir, ec32Policy);
+ fs.setErasureCodingPolicy(childDir, ec32Policy.getName());
// Create RS(3,2) EC policy file
DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L);
@@ -202,7 +202,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
// Test unset root path which has no EC policy
fs.unsetErasureCodingPolicy(rootPath);
// Set EC policy on root path
- fs.setErasureCodingPolicy(rootPath, ecPolicy);
+ fs.setErasureCodingPolicy(rootPath, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
fs.unsetErasureCodingPolicy(rootPath);
DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 1, 0L);
@@ -240,11 +240,11 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
.getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID);
fs.unsetErasureCodingPolicy(rootPath);
- fs.setErasureCodingPolicy(rootPath, ecPolicy);
+ fs.setErasureCodingPolicy(rootPath, ecPolicy.getName());
// Create RS(6,3) EC policy file
DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L);
// Change EC policy from RS(6,3) to RS(3,2)
- fs.setErasureCodingPolicy(rootPath, ec32Policy);
+ fs.setErasureCodingPolicy(rootPath, ec32Policy.getName());
DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L);
// start to check
@@ -281,7 +281,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
final Path replicateFilePath2 = new Path(ecDirPath, "rep_file2");
fs.mkdirs(ecDirPath);
- fs.setErasureCodingPolicy(ecDirPath, ecPolicy);
+ fs.setErasureCodingPolicy(ecDirPath, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
fs.unsetErasureCodingPolicy(ecDirPath);
DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 3, 0L);
@@ -328,7 +328,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
// Set EC policy on non-existent directory
try {
- fs.setErasureCodingPolicy(dirPath, ecPolicy);
+ fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
fail("FileNotFoundException should be thrown for a non-existent"
+ " file path");
} catch (FileNotFoundException e) {
@@ -347,7 +347,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
// Set EC policy on file
try {
- fs.setErasureCodingPolicy(ecFilePath, ecPolicy);
+ fs.setErasureCodingPolicy(ecFilePath, ecPolicy.getName());
fail("IOException should be thrown for setting EC policy on file");
} catch (IOException e) {
assertExceptionContains("Attempt to set an erasure coding policy " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
index 712f326..76ca704 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
@@ -80,7 +80,8 @@ public class TestWriteReadStripedFile {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
fs = cluster.getFileSystem();
fs.mkdirs(new Path("/ec"));
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
index b89e169..23a9821 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
@@ -59,7 +59,8 @@ public class TestWriteStripedFileWithFailure {
public void setup() throws IOException {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 1906d74..70aa4e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -1940,7 +1940,8 @@ public class TestBalancer {
cluster.waitActive();
client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
ClientProtocol.class).getProxy();
- client.setErasureCodingPolicy("/", null);
+ client.setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
long totalCapacity = sum(capacities);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
index 834df6f..20c3acc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
@@ -83,8 +83,8 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS {
.nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
.numDataNodes(numDNs)
.build();
- cluster.getFileSystem().getClient()
- .setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
try {
cluster.waitActive();
doTestRead(conf, cluster, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
index ace8732..832876e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
@@ -150,7 +150,8 @@ public class TestReconstructStripedBlocksWithRackAwareness {
.numDataNodes(hosts.length).build();
cluster.waitActive();
fs = cluster.getFileSystem();
- fs.setErasureCodingPolicy(new Path("/"), null);
+ fs.setErasureCodingPolicy(new Path("/"),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
FSNamesystem fsn = cluster.getNamesystem();
BlockManager bm = fsn.getBlockManager();
@@ -220,7 +221,8 @@ public class TestReconstructStripedBlocksWithRackAwareness {
.numDataNodes(hosts.length).build();
cluster.waitActive();
fs = cluster.getFileSystem();
- fs.setErasureCodingPolicy(new Path("/"), null);
+ fs.setErasureCodingPolicy(new Path("/"),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
MiniDFSCluster.DataNodeProperties lastHost = stopDataNode(
hosts[hosts.length - 1]);
@@ -273,7 +275,8 @@ public class TestReconstructStripedBlocksWithRackAwareness {
.numDataNodes(hostNames.length).build();
cluster.waitActive();
fs = cluster.getFileSystem();
- fs.setErasureCodingPolicy(new Path("/"), null);
+ fs.setErasureCodingPolicy(new Path("/"),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
final BlockManager bm = cluster.getNamesystem().getBlockManager();
final DatanodeManager dm = bm.getDatanodeManager();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
index 10ea01f..7920c59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
@@ -88,8 +88,8 @@ public class TestSequentialBlockGroupId {
blockGrpIdGenerator = cluster.getNamesystem().getBlockManager()
.getBlockIdManager().getBlockGroupIdGenerator();
fs.mkdirs(ecDir);
- cluster.getFileSystem().getClient()
- .setErasureCodingPolicy("/ecDir", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/ecDir",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
index 7e64214..7036c7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
@@ -75,7 +75,8 @@ public class TestDataNodeErasureCodingMetrics {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index d565548..a403ff4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -537,7 +537,8 @@ public class TestMover {
client.setStoragePolicy(barDir,
HdfsConstants.HOT_STORAGE_POLICY_NAME);
// set an EC policy on "/bar" directory
- client.setErasureCodingPolicy(barDir, null);
+ client.setErasureCodingPolicy(barDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// write file to barDir
final String fooFile = "/bar/foo";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
index c8b25564..670efd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
@@ -81,7 +81,8 @@ public class TestAddOverReplicatedStripedBlocks {
cluster.waitActive();
fs = cluster.getFileSystem();
fs.mkdirs(dirPath);
- fs.getClient().setErasureCodingPolicy(dirPath.toString(), null);
+ fs.getClient().setErasureCodingPolicy(dirPath.toString(),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
index 6e02372..87fbcc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
@@ -87,7 +87,8 @@ public class TestAddStripedBlockInFBR {
final Path repDir = new Path("/rep");
dfs.mkdirs(ecDir);
dfs.mkdirs(repDir);
- dfs.getClient().setErasureCodingPolicy(ecDir.toString(), null);
+ dfs.getClient().setErasureCodingPolicy(ecDir.toString(),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// create several non-EC files and one EC file
final Path[] repFiles = new Path[groupSize];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
index ffd3fa7..2df1aa4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
@@ -86,7 +86,8 @@ public class TestAddStripedBlocks {
.numDataNodes(groupSize).build();
cluster.waitActive();
dfs = cluster.getFileSystem();
- dfs.getClient().setErasureCodingPolicy("/", null);
+ dfs.getClient().setErasureCodingPolicy("/", ErasureCodingPolicyManager
+ .getSystemDefaultPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
index a57a936..72d76b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
@@ -479,7 +479,7 @@ public class TestFSEditLogLoader {
//set the storage policy of the directory
fs.mkdir(new Path(testDir), new FsPermission("755"));
fs.getClient().getNamenode().setErasureCodingPolicy(
- testDir, testECPolicy);
+ testDir, testECPolicy.getName());
// Create a file with striped block
Path p = new Path(testFilePath);
@@ -552,7 +552,7 @@ public class TestFSEditLogLoader {
//set the storage policy of the directory
fs.mkdir(new Path(testDir), new FsPermission("755"));
fs.getClient().getNamenode().setErasureCodingPolicy(
- testDir, testECPolicy);
+ testDir, testECPolicy.getName());
//create a file with striped blocks
Path p = new Path(testFilePath);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 179e297..ae15491 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -156,7 +156,7 @@ public class TestFSImage {
private void testSaveAndLoadStripedINodeFile(FSNamesystem fsn, Configuration conf,
boolean isUC) throws IOException{
// Construct an INode with StripedBlock for saving and loading
- fsn.setErasureCodingPolicy("/", testECPolicy, false);
+ fsn.setErasureCodingPolicy("/", testECPolicy.getName(), false);
long id = 123456789;
byte[] name = "testSaveAndLoadInodeFile_testfile".getBytes();
PermissionStatus permissionStatus = new PermissionStatus("testuser_a",
@@ -472,8 +472,8 @@ public class TestFSImage {
// Create directories and files
fs.mkdirs(parentDir);
fs.mkdirs(childDir);
- fs.setErasureCodingPolicy(parentDir, testECPolicy);
- fs.setErasureCodingPolicy(childDir, ec32Policy);
+ fs.setErasureCodingPolicy(parentDir, testECPolicy.getName());
+ fs.setErasureCodingPolicy(childDir, ec32Policy.getName());
Path file_10_4 = new Path(parentDir, "striped_file_10_4");
Path file_3_2 = new Path(childDir, "striped_file_3_2");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index c02e880..76c5378 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -690,7 +690,7 @@ public class TestFsck {
util.createFiles(fs, topDir);
// set topDir to EC when it has replicated files
cluster.getFileSystem().getClient().setErasureCodingPolicy(
- topDir, ecPolicy);
+ topDir, ecPolicy.getName());
// create a new file under topDir
DFSTestUtil.createFile(fs, new Path(topDir, "ecFile"), 1024, (short) 1, 0L);
@@ -2307,7 +2307,8 @@ public class TestFsck {
// create file
Path ecDirPath = new Path("/striped");
fs.mkdir(ecDirPath, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null);
+ fs.getClient().setErasureCodingPolicy(ecDirPath.toString(),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
Path file = new Path(ecDirPath, "corrupted");
final int length = cellSize * dataBlocks;
final byte[] bytes = StripedFileTestUtil.generateBytes(length);
@@ -2372,7 +2373,8 @@ public class TestFsck {
// create file
Path ecDirPath = new Path("/striped");
fs.mkdir(ecDirPath, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null);
+ fs.getClient().setErasureCodingPolicy(ecDirPath.toString(),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
Path file = new Path(ecDirPath, "missing");
final int length = cellSize * dataBlocks;
final byte[] bytes = StripedFileTestUtil.generateBytes(length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index ef1e6e4..037fd40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -738,7 +738,8 @@ public class TestNameNodeMXBean {
// create file
Path ecDirPath = new Path("/striped");
fs.mkdir(ecDirPath, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null);
+ fs.getClient().setErasureCodingPolicy(ecDirPath.toString(),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
Path file = new Path(ecDirPath, "corrupted");
final int length = cellSize * dataBlocks;
final byte[] bytes = StripedFileTestUtil.generateBytes(length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
index 9ec34d2..326ddc8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
@@ -71,7 +71,8 @@ public class TestQuotaWithStripedBlocks {
dfs = cluster.getFileSystem();
dfs.mkdirs(ecDir);
- dfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy);
+ dfs.getClient()
+ .setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName());
dfs.setQuota(ecDir, Long.MAX_VALUE - 1, DISK_QUOTA);
dfs.setQuotaByStorageType(ecDir, StorageType.DISK, DISK_QUOTA);
dfs.setStoragePolicy(ecDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
index cf1585c..5e4a9db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
@@ -201,7 +201,8 @@ public class TestReconstructStripedBlocks {
cluster.waitActive();
DistributedFileSystem fs = cluster.getFileSystem();
BlockManager bm = cluster.getNamesystem().getBlockManager();
- fs.getClient().setErasureCodingPolicy("/", null);
+ fs.getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
int fileLen = dataBlocks * blockSize;
Path p = new Path("/test2RecoveryTasksForSameBlockGroup");
final byte[] data = new byte[fileLen];
@@ -266,7 +267,8 @@ public class TestReconstructStripedBlocks {
try {
fs.mkdirs(dirPath);
- fs.setErasureCodingPolicy(dirPath, null);
+ fs.setErasureCodingPolicy(dirPath,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
DFSTestUtil.createFile(fs, filePath,
cellSize * dataBlocks * 2, (short) 1, 0L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
index 03bd124..a712498 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -307,7 +307,8 @@ public class TestStripedINodeFile {
dfs.mkdirs(ecDir);
// set erasure coding policy
- dfs.setErasureCodingPolicy(ecDir, null);
+ dfs.setErasureCodingPolicy(ecDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
DFSTestUtil.createFile(dfs, ecFile, len, (short) 1, 0xFEED);
DFSTestUtil.createFile(dfs, contiguousFile, len, (short) 1, 0xFEED);
final FSDirectory fsd = fsn.getFSDirectory();
@@ -408,7 +409,8 @@ public class TestStripedINodeFile {
client.mkdirs(fooDir, new FsPermission((short) 777), true);
client.setStoragePolicy(fooDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
// set an EC policy on "/foo" directory
- client.setErasureCodingPolicy(fooDir, null);
+ client.setErasureCodingPolicy(fooDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// write file to fooDir
final String barFile = "/foo/bar";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index 053b856..b587017 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -236,7 +236,8 @@ public class TestOfflineImageViewer {
ErasureCodingPolicy ecPolicy =
ErasureCodingPolicyManager.getPolicyByPolicyID(
HdfsConstants.XOR_2_1_POLICY_ID);
- hdfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy);
+ hdfs.getClient().setErasureCodingPolicy(ecDir.toString(),
+ ecPolicy.getName());
writtenFiles.put(ecDir.toString(), hdfs.getFileStatus(ecDir));
// Create an empty Erasure Coded file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
index db21217..e7794d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
@@ -63,7 +63,8 @@ public class TestOfflineImageViewerWithStripedBlocks {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
Path eczone = new Path("/eczone");
fs.mkdirs(eczone);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index 9b9003a..278963a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -359,7 +359,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Policy 'invalidpolicy' does not match any of the supported policies. Please select any one of [</expected-output>
+ <expected-output>Policy 'invalidpolicy' does not match any supported erasure coding policies.</expected-output>
</comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
index d218169..2eb4b29 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
@@ -989,7 +989,7 @@ public class TestDFSIO implements Tool {
((DistributedFileSystem) fs).getAllErasureCodingPolicies();
for (ErasureCodingPolicy ec : list) {
if (erasureCodePolicyName.equals(ec.getName())) {
- ((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec);
+ ((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec.getName());
LOG.info("enable erasureCodePolicy = " + erasureCodePolicyName +
" on " + path.toString());
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDFS-11421. Make WebHDFS' ACLs
RegEx configurable. Contributed by Harsh J.
Posted by jh...@apache.org.
HDFS-11421. Make WebHDFS' ACLs RegEx configurable. Contributed by Harsh J.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e24ed47d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e24ed47d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e24ed47d
Branch: refs/heads/YARN-5734
Commit: e24ed47d9a19f34a4dd8d4bad9b5c78ca3dd1c2e
Parents: d2b3ba9
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Feb 24 16:49:16 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Feb 24 16:49:46 2017 -0800
----------------------------------------------------------------------
.../hdfs/client/HdfsClientConfigKeys.java | 2 ++
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 6 +++-
.../hdfs/web/resources/AclPermissionParam.java | 17 +++++++++-
.../datanode/web/webhdfs/WebHdfsHandler.java | 10 ++++--
.../server/namenode/NameNodeHttpServer.java | 4 +++
.../src/main/resources/hdfs-default.xml | 8 +++++
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 29 ++++++++++++++++-
.../hadoop/hdfs/web/resources/TestParam.java | 34 ++++++++++++++++++++
8 files changed, 104 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 7ad79e0..6f8c661 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -35,6 +35,8 @@ public interface HdfsClientConfigKeys {
String DFS_WEBHDFS_USER_PATTERN_KEY =
"dfs.webhdfs.user.provider.user.pattern";
String DFS_WEBHDFS_USER_PATTERN_DEFAULT = "^[A-Za-z_][A-Za-z0-9._-]*[$]?$";
+ String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY =
+ "dfs.webhdfs.acl.provider.permission.pattern";
String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 135eef7..a9bc795 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -183,10 +183,14 @@ public class WebHdfsFileSystem extends FileSystem
) throws IOException {
super.initialize(uri, conf);
setConf(conf);
- /** set user pattern based on configuration file */
+
+ // set user and acl patterns based on configuration file
UserParam.setUserPattern(conf.get(
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+ AclPermissionParam.setAclPermissionPattern(conf.get(
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
boolean isOAuth = conf.getBoolean(
HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java
index 130c8fd..9ab3ad5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.permission.AclEntry;
/** AclPermission parameter. */
@@ -33,7 +34,7 @@ public class AclPermissionParam extends StringParam {
/** Default parameter value. */
public static final String DEFAULT = "";
- private static final Domain DOMAIN = new Domain(NAME,
+ private static Domain DOMAIN = new Domain(NAME,
Pattern.compile(DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
/**
@@ -49,6 +50,20 @@ public class AclPermissionParam extends StringParam {
super(DOMAIN,parseAclSpec(acl).equals(DEFAULT) ? null : parseAclSpec(acl));
}
+ @VisibleForTesting
+ public static Domain getAclPermissionPattern() {
+ return DOMAIN;
+ }
+
+ @VisibleForTesting
+ public static void setAclPermissionPattern(Domain dm) {
+ DOMAIN = dm;
+ }
+
+ public static void setAclPermissionPattern(String pattern) {
+ DOMAIN = new Domain(NAME, Pattern.compile(pattern));
+ }
+
@Override
public String getName() {
return NAME;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index f8c15fc..d2b2ec2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -48,11 +48,12 @@ import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
import org.apache.hadoop.hdfs.web.resources.PostOpParam;
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
@@ -112,8 +113,11 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
this.confForCreate = confForCreate;
/** set user pattern based on configuration file */
UserParam.setUserPattern(
- conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
- DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+ conf.get(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
+ HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+ AclPermissionParam.setAclPermissionPattern(
+ conf.get(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
index a1959e4..e7e5f51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.web.AuthFilter;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
import org.apache.hadoop.hdfs.web.resources.Param;
import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.http.HttpConfig;
@@ -80,6 +81,9 @@ public class NameNodeHttpServer {
UserParam.setUserPattern(conf.get(
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+ AclPermissionParam.setAclPermissionPattern(conf.get(
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
// add authentication filter for webhdfs
final String className = conf.get(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 652b216..d23b967 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2478,6 +2478,14 @@
</property>
<property>
+ <name>dfs.webhdfs.acl.provider.permission.pattern</name>
+ <value>^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$</value>
+ <description>
+ Valid pattern for user and group names in webhdfs acl operations, it must be a valid java regex.
+ </description>
+</property>
+
+<property>
<name>dfs.webhdfs.socket.connect-timeout</name>
<value>60s</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 1ff04de..7cfaa99 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -49,6 +49,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType;
+import com.google.common.collect.ImmutableList;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -64,6 +65,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclEntryScope;
+import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -379,10 +383,17 @@ public class TestWebHDFS {
}
@Test(timeout=300000)
- public void testNumericalUserName() throws Exception {
+ public void testCustomizedUserAndGroupNames() throws Exception {
final Configuration conf = WebHdfsTestUtil.createConf();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ // Modify username pattern to allow numeric usernames
conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, "^[A-Za-z0-9_][A-Za-z0-9" +
"._-]*[$]?$");
+ // Modify acl pattern to allow numeric and "@" characters user/groups in ACL spec
+ conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ "^(default:)?(user|group|mask|other):" +
+ "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?" +
+ "(user|group|mask|other):[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?)*$");
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
try {
@@ -391,6 +402,7 @@ public class TestWebHDFS {
.setPermission(new Path("/"),
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
+ // Test a numeric username
UserGroupInformation.createUserForTesting("123", new String[]{"my-group"})
.doAs(new PrivilegedExceptionAction<Void>() {
@Override
@@ -399,6 +411,21 @@ public class TestWebHDFS {
WebHdfsConstants.WEBHDFS_SCHEME);
Path d = new Path("/my-dir");
Assert.assertTrue(fs.mkdirs(d));
+ // Test also specifying a default ACL with a numeric username
+ // and another of a groupname with '@'
+ fs.modifyAclEntries(d, ImmutableList.of(
+ new AclEntry.Builder()
+ .setPermission(FsAction.READ)
+ .setScope(AclEntryScope.DEFAULT)
+ .setType(AclEntryType.USER)
+ .setName("11010")
+ .build(),
+ new AclEntry.Builder()
+ .setPermission(FsAction.READ_WRITE)
+ .setType(AclEntryType.GROUP)
+ .setName("foo@bar")
+ .build()
+ ));
return null;
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
index 6449bf7..d444cb4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
@@ -351,6 +351,40 @@ public class TestParam {
LOG.info("EXPECTED: " + e);
}
}
+
+ @Test
+ public void testUserGroupOkAfterAlteringAclPattern() {
+ // Preserve default pattern value
+ AclPermissionParam.Domain oldDomain =
+ AclPermissionParam.getAclPermissionPattern();
+
+ // Override the pattern with one that accepts '@' and numbers
+ // in the first character of usernames/groupnames
+ String newPattern =
+ "^(default:)?(user|group|mask|other):" +
+ "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?" +
+ "(,(default:)?(user|group|mask|other):" +
+ "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?)*$";
+
+ try {
+ AclPermissionParam.setAclPermissionPattern(newPattern);
+
+ String numericUserSpec = "user:110201:rwx";
+ AclPermissionParam aclNumericUserParam =
+ new AclPermissionParam(numericUserSpec);
+ Assert.assertEquals(numericUserSpec, aclNumericUserParam.getValue());
+
+ String oddGroupSpec = "group:foo@bar:rwx";
+ AclPermissionParam aclGroupWithDomainParam =
+ new AclPermissionParam(oddGroupSpec);
+ Assert.assertEquals(oddGroupSpec, aclGroupWithDomainParam.getValue());
+
+ } finally {
+ // Revert back to the default rules for remainder of tests
+ AclPermissionParam.setAclPermissionPattern(oldDomain);
+ }
+
+ }
@Test
public void testXAttrNameParam() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-6153. KeepContainer does not
work when AM retry window is set. Contributed by kyungwan nam
Posted by jh...@apache.org.
YARN-6153. KeepContainer does not work when AM retry window is set. Contributed by kyungwan nam
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/235203df
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/235203df
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/235203df
Branch: refs/heads/YARN-5734
Commit: 235203dffda1482fb38762fde544c4dd9c3e1fa8
Parents: e0bb867
Author: Jian He <ji...@apache.org>
Authored: Tue Feb 28 13:23:36 2017 -0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Feb 28 13:23:36 2017 -0800
----------------------------------------------------------------------
.../server/resourcemanager/rmapp/RMAppImpl.java | 17 +---
.../rmapp/attempt/RMAppAttemptImpl.java | 46 +++++------
.../yarn/server/resourcemanager/MockRM.java | 6 +-
.../resourcemanager/TestClientRMService.java | 4 +-
.../applicationsmanager/TestAMRestart.java | 84 +++++++++++++++++---
.../TestRMAppAttemptImplDiagnostics.java | 2 +-
.../attempt/TestRMAppAttemptTransitions.java | 19 +++--
7 files changed, 112 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/235203df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 516109b..9f00b2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -994,13 +994,7 @@ public class RMAppImpl implements RMApp, Recoverable {
}
RMAppAttempt attempt =
new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
- submissionContext, conf,
- // The newly created attempt maybe last attempt if (number of
- // previously failed attempts(which should not include Preempted,
- // hardware error and NM resync) + 1) equal to the max-attempt
- // limit.
- maxAppAttempts == (getNumFailedAppAttempts() + 1), amReq,
- currentAMBlacklistManager);
+ submissionContext, conf, amReq, this, currentAMBlacklistManager);
attempts.put(appAttemptId, attempt);
currentAttempt = attempt;
}
@@ -1498,18 +1492,13 @@ public class RMAppImpl implements RMApp, Recoverable {
};
}
- private int getNumFailedAppAttempts() {
+ public int getNumFailedAppAttempts() {
int completedAttempts = 0;
- long endTime = this.systemClock.getTime();
// Do not count AM preemption, hardware failures or NM resync
// as attempt failure.
for (RMAppAttempt attempt : attempts.values()) {
if (attempt.shouldCountTowardsMaxAttemptRetry()) {
- if (this.attemptFailuresValidityInterval <= 0
- || (attempt.getFinishTime() > endTime
- - this.attemptFailuresValidityInterval)) {
- completedAttempts++;
- }
+ completedAttempts++;
}
}
return completedAttempts;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/235203df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 25138c5..5c0f48e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -143,6 +143,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
private final EventHandler eventHandler;
private final YarnScheduler scheduler;
private final ApplicationMasterService masterService;
+ private final RMApp rmApp;
private final ReadLock readLock;
private final WriteLock writeLock;
@@ -179,12 +180,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
private int amContainerExitStatus = ContainerExitStatus.INVALID;
private Configuration conf;
- // Since AM preemption, hardware error and NM resync are not counted towards
- // AM failure count, even if this flag is true, a new attempt can still be
- // re-created if this attempt is eventually failed because of preemption,
- // hardware error or NM resync. So this flag indicates that this may be
- // last attempt.
- private final boolean maybeLastAttempt;
private static final ExpiredTransition EXPIRED_TRANSITION =
new ExpiredTransition();
private static final AttemptFailedTransition FAILED_TRANSITION =
@@ -490,16 +485,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
- Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq) {
+ Configuration conf, ResourceRequest amReq, RMApp rmApp) {
this(appAttemptId, rmContext, scheduler, masterService, submissionContext,
- conf, maybeLastAttempt, amReq, new DisabledBlacklistManager());
+ conf, amReq, rmApp, new DisabledBlacklistManager());
}
public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
- Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq,
+ Configuration conf, ResourceRequest amReq, RMApp rmApp,
BlacklistManager amBlacklistManager) {
this.conf = conf;
this.applicationAttemptId = appAttemptId;
@@ -514,7 +509,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.writeLock = lock.writeLock();
this.proxiedTrackingUrl = generateProxyUriWithScheme();
- this.maybeLastAttempt = maybeLastAttempt;
this.stateMachine = stateMachineFactory.make(this);
this.attemptMetrics =
@@ -531,6 +525,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
this.diagnostics = new BoundedAppender(diagnosticsLimitKC * 1024);
+ this.rmApp = rmApp;
}
private int getDiagnosticsLimitKCOrThrow(final Configuration configuration) {
@@ -1215,8 +1210,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
@Override
public RMAppAttemptState transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
- RMApp rmApp = appAttempt.rmContext.getRMApps().get(
- appAttempt.getAppAttemptId().getApplicationId());
+ RMApp rmApp = appAttempt.rmApp;
/*
* If last attempt recovered final state is null .. it means attempt was
@@ -1462,14 +1456,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
if (appAttempt.submissionContext
.getKeepContainersAcrossApplicationAttempts()
&& !appAttempt.submissionContext.getUnmanagedAM()) {
- // See if we should retain containers for non-unmanaged applications
- if (!appAttempt.shouldCountTowardsMaxAttemptRetry()) {
- // Premption, hardware failures, NM resync doesn't count towards
- // app-failures and so we should retain containers.
- keepContainersAcrossAppAttempts = true;
- } else if (!appAttempt.maybeLastAttempt) {
- // Not preemption, hardware failures or NM resync.
- // Not last-attempt too - keep containers.
+ int numberOfFailure = ((RMAppImpl)appAttempt.rmApp)
+ .getNumFailedAppAttempts();
+ if (numberOfFailure < appAttempt.rmApp.getMaxAppAttempts()) {
keepContainersAcrossAppAttempts = true;
}
}
@@ -1496,9 +1485,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
.applicationAttemptFinished(appAttempt, finalAttemptState);
appAttempt.rmContext.getSystemMetricsPublisher()
.appAttemptFinished(appAttempt, finalAttemptState,
- appAttempt.rmContext.getRMApps().get(
- appAttempt.applicationAttemptId.getApplicationId()),
- System.currentTimeMillis());
+ appAttempt.rmApp, System.currentTimeMillis());
}
}
@@ -1545,6 +1532,14 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
@Override
public boolean shouldCountTowardsMaxAttemptRetry() {
+ long attemptFailuresValidityInterval = this.submissionContext
+ .getAttemptFailuresValidityInterval();
+ long end = System.currentTimeMillis();
+ if (attemptFailuresValidityInterval > 0
+ && this.getFinishTime() > 0
+ && this.getFinishTime() < (end - attemptFailuresValidityInterval)) {
+ return false;
+ }
try {
this.readLock.lock();
int exitStatus = getAMContainerExitStatus();
@@ -2222,11 +2217,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
return attemptReport;
}
- // for testing
- public boolean mayBeLastAttempt() {
- return maybeLastAttempt;
- }
-
@Override
public RMAppAttemptMetrics getRMAppAttemptMetrics() {
// didn't use read/write lock here because RMAppAttemptMetrics has its own
http://git-wip-us.apache.org/repos/asf/hadoop/blob/235203df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 7d19dab..f9f42ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -618,15 +618,15 @@ public class MockRM extends ResourceManager {
false, null, 0, null, true, Priority.newInstance(0));
}
- public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval)
- throws Exception {
+ public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval,
+ boolean keepContainers) throws Exception {
Resource resource = Records.newRecord(Resource.class);
resource.setMemorySize(masterMemory);
Priority priority = Priority.newInstance(0);
return submitApp(resource, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null,
super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
- YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, keepContainers,
false, null, attemptFailuresValidityInterval, null, true, priority);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/235203df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 9076ac3..7a67aa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -396,7 +396,7 @@ public class TestClientRMService {
mock(ApplicationSubmissionContext.class);
YarnConfiguration config = new YarnConfiguration();
RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
- rmContext, yarnScheduler, null, asContext, config, false, null);
+ rmContext, yarnScheduler, null, asContext, config, null, null);
ApplicationResourceUsageReport report = rmAppAttemptImpl
.getApplicationResourceUsageReport();
assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
@@ -1327,7 +1327,7 @@ public class TestClientRMService {
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
ApplicationId.newInstance(123456, 1), 1);
RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId,
- rmContext, yarnScheduler, null, asContext, config, false, null));
+ rmContext, yarnScheduler, null, asContext, config, null, app));
Container container = Container.newInstance(
ContainerId.newContainerId(attemptId, 1), null, "", null, null, null);
RMContainerImpl containerimpl = spy(new RMContainerImpl(container,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/235203df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
index 03a9645..4fa8287 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
@@ -411,7 +411,6 @@ public class TestAMRestart {
MockAM am2 =
rm1.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
RMAppAttempt attempt2 = app1.getCurrentAppAttempt();
- Assert.assertTrue(((RMAppAttemptImpl) attempt2).mayBeLastAttempt());
// Preempt the second attempt.
ContainerId amContainer2 =
@@ -427,7 +426,6 @@ public class TestAMRestart {
MockAM am3 =
rm1.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 3, nm1);
RMAppAttempt attempt3 = app1.getCurrentAppAttempt();
- Assert.assertTrue(((RMAppAttemptImpl) attempt3).mayBeLastAttempt());
// mimic NM disk_failure
ContainerStatus containerStatus = Records.newRecord(ContainerStatus.class);
@@ -454,7 +452,6 @@ public class TestAMRestart {
MockAM am4 =
rm1.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 4, nm1);
RMAppAttempt attempt4 = app1.getCurrentAppAttempt();
- Assert.assertTrue(((RMAppAttemptImpl) attempt4).mayBeLastAttempt());
// create second NM, and register to rm1
MockNM nm2 =
@@ -475,7 +472,6 @@ public class TestAMRestart {
MockAM am5 =
rm1.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 5, nm2);
RMAppAttempt attempt5 = app1.getCurrentAppAttempt();
- Assert.assertTrue(((RMAppAttemptImpl) attempt5).mayBeLastAttempt());
// fail the AM normally
nm2
.nodeHeartbeat(am5.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
@@ -584,7 +580,6 @@ public class TestAMRestart {
// AM should be restarted even though max-am-attempt is 1.
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
- Assert.assertTrue(((RMAppAttemptImpl) attempt1).mayBeLastAttempt());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
@@ -645,7 +640,7 @@ public class TestAMRestart {
// set window size to a larger number : 60s
// we will verify the app should be failed if
// two continuous attempts failed in 60s.
- RMApp app = rm1.submitApp(200, 60000);
+ RMApp app = rm1.submitApp(200, 60000, false);
MockAM am = MockRM.launchAM(app, rm1, nm1);
// Fail current attempt normally
@@ -655,8 +650,7 @@ public class TestAMRestart {
// launch the second attempt
rm1.waitForState(app.getApplicationId(), RMAppState.ACCEPTED);
Assert.assertEquals(2, app.getAppAttempts().size());
- Assert.assertTrue(((RMAppAttemptImpl) app.getCurrentAppAttempt())
- .mayBeLastAttempt());
+
MockAM am_2 = MockRM.launchAndRegisterAM(app, rm1, nm1);
rm1.waitForState(am_2.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
nm1.nodeHeartbeat(am_2.getApplicationAttemptId(),
@@ -667,7 +661,7 @@ public class TestAMRestart {
ControlledClock clock = new ControlledClock();
// set window size to 10s
- RMAppImpl app1 = (RMAppImpl)rm1.submitApp(200, 10000);
+ RMAppImpl app1 = (RMAppImpl)rm1.submitApp(200, 10000, false);
app1.setSystemClock(clock);
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
@@ -684,7 +678,6 @@ public class TestAMRestart {
Assert.assertEquals(2, app1.getAppAttempts().size());
RMAppAttempt attempt2 = app1.getCurrentAppAttempt();
- Assert.assertTrue(((RMAppAttemptImpl) attempt2).mayBeLastAttempt());
MockAM am2 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
@@ -863,4 +856,75 @@ public class TestAMRestart {
rm1.stop();
}
+
+ // Test restarting AM launched with the KeepContainers and AM reset window.
+ // after AM reset window, even if AM who was the last is failed,
+ // all containers are launched by previous AM should be kept.
+ @Test (timeout = 20000)
+ public void testAMRestartNotLostContainerAfterAttemptFailuresValidityInterval()
+ throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ // explicitly set max-am-retry count as 2.
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
+
+ MockRM rm1 = new MockRM(conf);
+ rm1.start();
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ // set window size to 10s and enable keepContainers
+ RMAppImpl app1 = (RMAppImpl)rm1.submitApp(200, 10000, true);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ int NUM_CONTAINERS = 2;
+ allocateContainers(nm1, am1, NUM_CONTAINERS);
+
+ // launch the 2nd container, for testing running container transferred.
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
+ ContainerId containerId2 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
+
+ // Fail attempt1 normally
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(),
+ 1, ContainerState.COMPLETE);
+ rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
+
+ // launch the second attempt
+ rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
+ Assert.assertEquals(2, app1.getAppAttempts().size());
+
+ // It will be the last attempt.
+ RMAppAttempt attempt2 = app1.getCurrentAppAttempt();
+ MockAM am2 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
+
+ // wait for 10 seconds to reset AM failure count
+ Thread.sleep(10 * 1000);
+
+ // Fail attempt2 normally
+ nm1.nodeHeartbeat(am2.getApplicationAttemptId(),
+ 1, ContainerState.COMPLETE);
+ rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED);
+
+ // can launch the third attempt successfully
+ rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
+ Assert.assertEquals(3, app1.getAppAttempts().size());
+ MockAM am3 = rm1.launchAM(app1, rm1, nm1);
+ RegisterApplicationMasterResponse registerResponse =
+ am3.registerAppAttempt();
+
+ // keepContainers is applied, even if attempt2 was the last attempt.
+ Assert.assertEquals(1, registerResponse.getContainersFromPreviousAttempts()
+ .size());
+ boolean containerId2Exists = false;
+ Container container = registerResponse.getContainersFromPreviousAttempts().get(0);
+ if (container.getId().equals(containerId2)) {
+ containerId2Exists = true;
+ }
+ Assert.assertTrue(containerId2Exists);
+
+ rm1.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
+ rm1.stop();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/235203df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java
index a160eb8..19b5dd9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java
@@ -106,6 +106,6 @@ public class TestRMAppAttemptImplDiagnostics {
when(mockRMContext.getDispatcher()).thenReturn(mockDispatcher);
return new RMAppAttemptImpl(mockApplicationAttemptId, mockRMContext, null,
- null, null, configuration, false, null);
+ null, null, configuration, null, null);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/235203df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 786cc50..ced5bd9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -327,10 +327,10 @@ public class TestRMAppAttemptTransitions {
application = mock(RMAppImpl.class);
applicationAttempt =
new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler,
- masterService, submissionContext, new Configuration(), false,
+ masterService, submissionContext, new Configuration(),
BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1), application);
when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
when(application.getApplicationId()).thenReturn(applicationId);
@@ -1107,10 +1107,10 @@ public class TestRMAppAttemptTransitions {
RMAppAttempt myApplicationAttempt =
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
spyRMContext, scheduler,masterService,
- submissionContext, myConf, false,
+ submissionContext, myConf,
BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1), application);
//submit, schedule and allocate app attempt
myApplicationAttempt.handle(
@@ -1536,6 +1536,9 @@ public class TestRMAppAttemptTransitions {
// create a failed attempt.
when(submissionContext.getKeepContainersAcrossApplicationAttempts())
.thenReturn(true);
+ when(application.getMaxAppAttempts()).thenReturn(2);
+ when(application.getNumFailedAppAttempts()).thenReturn(1);
+
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
@@ -1581,9 +1584,9 @@ public class TestRMAppAttemptTransitions {
applicationAttempt =
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext,
scheduler, masterService, submissionContext, new Configuration(),
- true, BuilderUtils.newResourceRequest(
+ BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1), application);
when(submissionContext.getKeepContainersAcrossApplicationAttempts())
.thenReturn(true);
when(submissionContext.getMaxAppAttempts()).thenReturn(1);
@@ -1642,9 +1645,9 @@ public class TestRMAppAttemptTransitions {
applicationAttempt =
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
spyRMContext, scheduler, masterService, submissionContext,
- new Configuration(), true, ResourceRequest.newInstance(
+ new Configuration(), ResourceRequest.newInstance(
Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3,
- false, "label-expression"));
+ false, "label-expression"), application);
new RMAppAttemptImpl.ScheduleTransition().transition(
(RMAppAttemptImpl) applicationAttempt, null);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HADOOP-14113. Review ADL Docs.
Contributed by Steve Loughran
Posted by jh...@apache.org.
HADOOP-14113. Review ADL Docs. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e60c6543
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e60c6543
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e60c6543
Branch: refs/heads/YARN-5734
Commit: e60c6543d57611039b0438d5dcb4cb19ee239bb6
Parents: 9c22a91
Author: Steve Loughran <st...@apache.org>
Authored: Fri Feb 24 13:24:59 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Feb 24 13:24:59 2017 +0000
----------------------------------------------------------------------
.../src/site/markdown/index.md | 237 ++++++++++---------
1 file changed, 124 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e60c6543/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index 6d9e173..9355241 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -20,20 +20,20 @@
* [Usage](#Usage)
* [Concepts](#Concepts)
* [OAuth2 Support](#OAuth2_Support)
- * [Configuring Credentials & FileSystem](#Configuring_Credentials)
+ * [Configuring Credentials and FileSystem](#Configuring_Credentials)
* [Using Refresh Token](#Refresh_Token)
* [Using Client Keys](#Client_Credential_Token)
* [Protecting the Credentials with Credential Providers](#Credential_Provider)
* [Enabling ADL Filesystem](#Enabling_ADL)
- * [Accessing adl URLs](#Accessing_adl_URLs)
+ * [Accessing `adl` URLs](#Accessing_adl_URLs)
* [User/Group Representation](#OIDtoUPNConfiguration)
-* [Testing the hadoop-azure Module](#Testing_the_hadoop-azure_Module)
+* [Testing the `hadoop-azure` Module](#Testing_the_hadoop-azure_Module)
## <a name="Introduction" />Introduction
-The hadoop-azure-datalake module provides support for integration with
-[Azure Data Lake Store]( https://azure.microsoft.com/en-in/documentation/services/data-lake-store/).
-The jar file is named azure-datalake-store.jar.
+The `hadoop-azure-datalake` module provides support for integration with the
+[Azure Data Lake Store](https://azure.microsoft.com/en-in/documentation/services/data-lake-store/).
+This support comes via the JAR file `azure-datalake-store.jar`.
## <a name="Features" />Features
@@ -43,13 +43,14 @@ The jar file is named azure-datalake-store.jar.
* Can act as a source of data in a MapReduce job, or a sink.
* Tested on both Linux and Windows.
* Tested for scale.
-* API setOwner/setAcl/removeAclEntries/modifyAclEntries accepts UPN or OID
- (Object ID) as user and group name.
+* API `setOwner()`, `setAcl`, `removeAclEntries()`, `modifyAclEntries()` accepts UPN or OID
+ (Object ID) as user and group names.
## <a name="Limitations" />Limitations
+
Partial or no support for the following operations :
-* Operation on Symbolic Link
+* Operation on Symbolic Links
* Proxy Users
* File Truncate
* File Checksum
@@ -58,55 +59,71 @@ Partial or no support for the following operations :
* Extended Attributes(XAttrs) Operations
* Snapshot Operations
* Delegation Token Operations
-* User and group information returned as ListStatus and GetFileStatus is in form of GUID associated in Azure Active Directory.
+* User and group information returned as `listStatus()` and `getFileStatus()` is
+in the form of the GUID associated in Azure Active Directory.
## <a name="Usage" />Usage
### <a name="Concepts" />Concepts
-Azure Data Lake Storage access path syntax is
+Azure Data Lake Storage access path syntax is:
+
+```
+adl://<Account Name>.azuredatalakestore.net/
+```
- adl://<Account Name>.azuredatalakestore.net/
+For details on using the store, see
+[**Get started with Azure Data Lake Store using the Azure Portal**](https://azure.microsoft.com/en-in/documentation/articles/data-lake-store-get-started-portal/)
-Get started with azure data lake account with [https://azure.microsoft.com/en-in/documentation/articles/data-lake-store-get-started-portal/](https://azure.microsoft.com/en-in/documentation/articles/data-lake-store-get-started-portal/)
+### <a name="#OAuth2_Support" />OAuth2 Support
-#### <a name="#OAuth2_Support" />OAuth2 Support
-Usage of Azure Data Lake Storage requires OAuth2 bearer token to be present as part of the HTTPS header as per OAuth2 specification. Valid OAuth2 bearer token should be obtained from Azure Active Directory for valid users who have access to Azure Data Lake Storage Account.
+Usage of Azure Data Lake Storage requires an OAuth2 bearer token to be present as
+part of the HTTPS header as per the OAuth2 specification.
+A valid OAuth2 bearer token must be obtained from the Azure Active Directory service
+for those valid users who have access to Azure Data Lake Storage Account.
-Azure Active Directory (Azure AD) is Microsoft's multi-tenant cloud based directory and identity management service. See [https://azure.microsoft.com/en-in/documentation/articles/active-directory-whatis/](https://azure.microsoft.com/en-in/documentation/articles/active-directory-whatis/)
+Azure Active Directory (Azure AD) is Microsoft's multi-tenant cloud based directory
+and identity management service. See [*What is ActiveDirectory*](https://azure.microsoft.com/en-in/documentation/articles/active-directory-whatis/).
-Following sections describes on OAuth2 configuration in core-site.xml.
+Following sections describes theOAuth2 configuration in `core-site.xml`.
-## <a name="Configuring_Credentials" />Configuring Credentials & FileSystem
-Credentials can be configured using either a refresh token (associated with a user) or a client credential (analogous to a service principal).
+#### <a name="Configuring_Credentials" />Configuring Credentials & FileSystem
+Credentials can be configured using either a refresh token (associated with a user),
+or a client credential (analogous to a service principal).
-### <a name="Refresh_Token" />Using Refresh Token
+#### <a name="Refresh_Token" />Using Refresh Tokens
-Add the following properties to your core-site.xml
+Add the following properties to the cluster's `core-site.xml`
- <property>
- <name>dfs.adls.oauth2.access.token.provider.type</name>
- <value>RefreshToken</value>
- </property>
+```xml
+<property>
+ <name>dfs.adls.oauth2.access.token.provider.type</name>
+ <value>RefreshToken</value>
+</property>
+```
-Application require to set Client id and OAuth2 refresh token from Azure Active Directory associated with client id. See [https://github.com/AzureAD/azure-activedirectory-library-for-java](https://github.com/AzureAD/azure-activedirectory-library-for-java).
+Applications must set the Client id and OAuth2 refresh token from the Azure Active Directory
+service associated with the client id. See [*Active Directory Library For Java*](https://github.com/AzureAD/azure-activedirectory-library-for-java).
**Do not share client id and refresh token, it must be kept secret.**
- <property>
- <name>dfs.adls.oauth2.client.id</name>
- <value></value>
- </property>
+```xml
+<property>
+ <name>dfs.adls.oauth2.client.id</name>
+ <value></value>
+</property>
- <property>
- <name>dfs.adls.oauth2.refresh.token</name>
- <value></value>
- </property>
+<property>
+ <name>dfs.adls.oauth2.refresh.token</name>
+ <value></value>
+</property>
+```
### <a name="Client_Credential_Token" />Using Client Keys
#### Generating the Service Principal
-1. Go to the portal (https://portal.azure.com)
+
+1. Go to [the portal](https://portal.azure.com)
2. Under "Browse", look for Active Directory and click on it.
3. Create "Web Application". Remember the name you create here - that is what you will add to your ADL account as authorized user.
4. Go through the wizard
@@ -124,31 +141,31 @@ Application require to set Client id and OAuth2 refresh token from Azure Active
3. Add your user name you created in Step 6 above (note that it does not show up in the list, but will be found if you searched for the name)
4. Add "Owner" role
-#### Configure core-site.xml
-Add the following properties to your core-site.xml
-
- <property>
- <name>dfs.adls.oauth2.refresh.url</name>
- <value>TOKEN ENDPOINT FROM STEP 7 ABOVE</value>
- </property>
+### Configure core-site.xml
+Add the following properties to your `core-site.xml`
- <property>
- <name>dfs.adls.oauth2.client.id</name>
- <value>CLIENT ID FROM STEP 7 ABOVE</value>
- </property>
+```xml
+<property>
+ <name>dfs.adls.oauth2.refresh.url</name>
+ <value>TOKEN ENDPOINT FROM STEP 7 ABOVE</value>
+</property>
- <property>
- <name>dfs.adls.oauth2.credential</name>
- <value>PASSWORD FROM STEP 7 ABOVE</value>
- </property>
+<property>
+ <name>dfs.adls.oauth2.client.id</name>
+ <value>CLIENT ID FROM STEP 7 ABOVE</value>
+</property>
+<property>
+ <name>dfs.adls.oauth2.credential</name>
+ <value>PASSWORD FROM STEP 7 ABOVE</value>
+</property>
+```
### <a name="Credential_Provider" />Protecting the Credentials with Credential Providers
-In many Hadoop clusters, the core-site.xml file is world-readable. To protect
-these credentials from prying eyes, it is recommended that you use the
-credential provider framework to securely store them and access them through
-configuration.
+In many Hadoop clusters, the `core-site.xml` file is world-readable. To protect
+these credentials, it is recommended that you use the
+credential provider framework to securely store them and access them.
All ADLS credential properties can be protected by credential providers.
For additional reading on the credential provider API, see
@@ -156,16 +173,16 @@ For additional reading on the credential provider API, see
#### Provisioning
-```
-% hadoop credential create dfs.adls.oauth2.refresh.token -value 123
+```bash
+hadoop credential create dfs.adls.oauth2.refresh.token -value 123
-provider localjceks://file/home/foo/adls.jceks
-% hadoop credential create dfs.adls.oauth2.credential -value 123
+hadoop credential create dfs.adls.oauth2.credential -value 123
-provider localjceks://file/home/foo/adls.jceks
```
#### Configuring core-site.xml or command line property
-```
+```xml
<property>
<name>hadoop.security.credential.provider.path</name>
<value>localjceks://file/home/foo/adls.jceks</value>
@@ -175,42 +192,28 @@ For additional reading on the credential provider API, see
#### Running DistCp
-```
-% hadoop distcp
+```bash
+hadoop distcp
[-D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks]
hdfs://<NameNode Hostname>:9001/user/foo/007020615
adl://<Account Name>.azuredatalakestore.net/testDir/
```
-NOTE: You may optionally add the provider path property to the distcp command
-line instead of added job specific configuration to a generic core-site.xml.
-The square brackets above illustrate this capability.
-
-
-## <a name="Enabling_ADL" />Enabling ADL Filesystem
-
-For ADL FileSystem to take effect. Update core-site.xml with
-
- <property>
- <name>fs.adl.impl</name>
- <value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
- </property>
-
- <property>
- <name>fs.AbstractFileSystem.adl.impl</name>
- <value>org.apache.hadoop.fs.adl.Adl</value>
- </property>
-
+NOTE: You may optionally add the provider path property to the `distcp` command
+line instead of added job specific configuration to a generic `core-site.xml`.
+The square brackets above illustrate this capability.`
### <a name="Accessing_adl_URLs" />Accessing adl URLs
-After credentials are configured in core-site.xml, any Hadoop component may
+After credentials are configured in `core-site.xml`, any Hadoop component may
reference files in that Azure Data Lake Storage account by using URLs of the following
format:
- adl://<Account Name>.azuredatalakestore.net/<path>
+```
+adl://<Account Name>.azuredatalakestore.net/<path>
+```
-The schemes `adl` identify a URL on a file system backed by Azure
+The schemes `adl` identifies a URL on a Hadoop-compatible file system backed by Azure
Data Lake Storage. `adl` utilizes encrypted HTTPS access for all interaction with
the Azure Data Lake Storage API.
@@ -218,48 +221,56 @@ For example, the following
[FileSystem Shell](../hadoop-project-dist/hadoop-common/FileSystemShell.html)
commands demonstrate access to a storage account named `youraccount`.
- > hadoop fs -mkdir adl://yourcontainer.azuredatalakestore.net/testDir
- > hadoop fs -put testFile adl://yourcontainer.azuredatalakestore.net/testDir/testFile
+```bash
+hadoop fs -mkdir adl://yourcontainer.azuredatalakestore.net/testDir
- > hadoop fs -cat adl://yourcontainer.azuredatalakestore.net/testDir/testFile
- test file content
+hadoop fs -put testFile adl://yourcontainer.azuredatalakestore.net/testDir/testFile
+hadoop fs -cat adl://yourcontainer.azuredatalakestore.net/testDir/testFile
+test file content
+```
### <a name="OIDtoUPNConfiguration" />User/Group Representation
-The hadoop-azure-datalake module provides support for configuring how
-User/Group information is represented during
-getFileStatus/listStatus/getAclStatus.
-Add the following properties to your core-site.xml
-
- <property>
- <name>adl.feature.ownerandgroup.enableupn</name>
- <value>true</value>
- <description>
- When true : User and Group in FileStatus/AclStatus response is
- represented as user friendly name as per Azure AD profile.
-
- When false (default) : User and Group in FileStatus/AclStatus
- response is represented by the unique identifier from Azure AD
- profile (Object ID as GUID).
+The `hadoop-azure-datalake` module provides support for configuring how
+User/Group information is represented during
+`getFileStatus()`, `listStatus()`, and `getAclStatus()` calls..
- For performance optimization, Recommended default value.
- </description>
- </property>
+Add the following properties to `core-site.xml`
+```xml
+<property>
+ <name>adl.feature.ownerandgroup.enableupn</name>
+ <value>true</value>
+ <description>
+ When true : User and Group in FileStatus/AclStatus response is
+ represented as user friendly name as per Azure AD profile.
+
+ When false (default) : User and Group in FileStatus/AclStatus
+ response is represented by the unique identifier from Azure AD
+ profile (Object ID as GUID).
+
+ For performance optimization, Recommended default value.
+ </description>
+</property>
+```
## <a name="Testing_the_hadoop-azure_Module" />Testing the azure-datalake-store Module
-The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage.
+The `hadoop-azure` module includes a full suite of unit tests.
+Most of the tests will run without additional configuration by running mvn test.
+This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage.
A selection of tests can run against the Azure Data Lake Storage. To run these
tests, please create `src/test/resources/auth-keys.xml` with Adl account
information mentioned in the above sections and the following properties.
- <property>
- <name>dfs.adl.test.contract.enable</name>
- <value>true</value>
- </property>
+```xml
+<property>
+ <name>dfs.adl.test.contract.enable</name>
+ <value>true</value>
+</property>
- <property>
- <name>test.fs.adl.name</name>
- <value>adl://yourcontainer.azuredatalakestore.net</value>
- </property>
+<property>
+ <name>test.fs.adl.name</name>
+ <value>adl://yourcontainer.azuredatalakestore.net</value>
+</property>
+```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-1728. Regression test for
guice-undecoded pathInfo in YARN WebApp. (Yuanbo Liu via gera)
Posted by jh...@apache.org.
YARN-1728. Regression test for guice-undecoded pathInfo in YARN WebApp. (Yuanbo Liu via gera)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/480b4dd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/480b4dd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/480b4dd5
Branch: refs/heads/YARN-5734
Commit: 480b4dd574d0355bf6c976a38bb45cb86adb2714
Parents: d269b48
Author: Gera Shegalov <ge...@apache.org>
Authored: Tue Feb 28 09:22:29 2017 -0800
Committer: Gera Shegalov <ge...@apache.org>
Committed: Tue Feb 28 09:27:22 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/yarn/webapp/TestWebApp.java | 31 +++++++++++++++++++-
1 file changed, 30 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480b4dd5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
index 9454002..db50dd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
@@ -27,12 +27,12 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.net.URLEncoder;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
@@ -263,6 +263,35 @@ public class TestWebApp {
}
}
+ @Test public void testEncodedUrl() throws Exception {
+ WebApp app =
+ WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() {
+ @Override
+ public void setup() {
+ bind(MyTestJAXBContextResolver.class);
+ bind(MyTestWebService.class);
+
+ route("/:foo", FooController.class);
+ }
+ });
+ String baseUrl = baseUrl(app);
+
+ try {
+ // Test encoded url
+ String rawPath = "localhost:8080";
+ String encodedUrl = baseUrl + "test/" +
+ URLEncoder.encode(rawPath, "UTF-8");
+ assertEquals("foo" + rawPath, getContent(encodedUrl).trim());
+
+ rawPath = "@;%$";
+ encodedUrl = baseUrl + "test/" +
+ URLEncoder.encode(rawPath, "UTF-8");
+ assertEquals("foo" + rawPath, getContent(encodedUrl).trim());
+ } finally {
+ app.stop();
+ }
+ }
+
@Test public void testRobotsText() throws Exception {
WebApp app =
WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDFS-11479. Socket re-use address
option should be used in SimpleUdpServer. Contributed by Mukul Kumar Singh.
Posted by jh...@apache.org.
HDFS-11479. Socket re-use address option should be used in SimpleUdpServer. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/899d5c4d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/899d5c4d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/899d5c4d
Branch: refs/heads/YARN-5734
Commit: 899d5c4d49f00d90ddc3632efb2df92841867192
Parents: 82ef9ac
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Mar 1 12:59:02 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Mar 1 12:59:02 2017 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899d5c4d/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
index c9cb2b3..d691aba 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
@@ -63,6 +63,7 @@ public class SimpleUdpServer {
server.setOption("broadcast", "false");
server.setOption("sendBufferSize", SEND_BUFFER_SIZE);
server.setOption("receiveBufferSize", RECEIVE_BUFFER_SIZE);
+ server.setOption("reuseAddress", true);
// Listen to the UDP port
ch = server.bind(new InetSocketAddress(port));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDFS-11461. DataNode Disk Outlier
Detection. Contributed by Hanisha Koneru.
Posted by jh...@apache.org.
HDFS-11461. DataNode Disk Outlier Detection. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3ec531f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3ec531f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3ec531f
Branch: refs/heads/YARN-5734
Commit: b3ec531f400dd0a6506dc71233d38ae57b764a43
Parents: 747bafa
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Mar 2 12:45:48 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Mar 2 12:45:48 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +-
.../server/blockmanagement/SlowPeerTracker.java | 4 +-
.../hdfs/server/datanode/BPServiceActor.java | 2 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 16 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 10 +
.../datanode/metrics/DataNodeDiskMetrics.java | 181 +++++++++++++++++
.../datanode/metrics/DataNodePeerMetrics.java | 6 +-
.../datanode/metrics/OutlierDetector.java | 182 +++++++++++++++++
.../datanode/metrics/SlowNodeDetector.java | 194 -------------------
.../src/main/resources/hdfs-default.xml | 2 +-
.../TestDataNodeOutlierDetectionViaMetrics.java | 6 +-
.../datanode/metrics/TestSlowNodeDetector.java | 30 +--
12 files changed, 412 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 68cef36..be20829 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -677,9 +677,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT = "dfs.block.misreplication.processing.limit";
public static final int DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT = 10000;
- public static final String DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_KEY =
- "dfs.datanode.slow.peers.report.interval";
- public static final String DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_DEFAULT =
+ public static final String DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY =
+ "dfs.datanode.outliers.report.interval";
+ public static final String DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT =
"30m";
// property for fsimage compression
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
index cf3a20c..c72a621 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
@@ -94,8 +94,8 @@ public class SlowPeerTracker {
this.timer = timer;
this.allReports = new ConcurrentHashMap<>();
this.reportValidityMs = conf.getTimeDuration(
- DFSConfigKeys.DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_KEY,
- DFSConfigKeys.DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_DEFAULT,
+ DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS) * 3;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 644a8ab..a0ba627 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -129,7 +129,7 @@ class BPServiceActor implements Runnable {
prevBlockReportId = ThreadLocalRandom.current().nextLong();
scheduler = new Scheduler(dnConf.heartBeatInterval,
dnConf.getLifelineIntervalMs(), dnConf.blockReportInterval,
- dnConf.slowPeersReportIntervalMs);
+ dnConf.outliersReportIntervalMs);
// get the value of maxDataLength.
this.maxDataLength = dnConf.getMaxDataLength();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 2723677..3275ba8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -30,8 +30,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_LIFELINE_INTERVAL_SECONDS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_NON_LOCAL_LAZY_PERSIST;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_NON_LOCAL_LAZY_PERSIST_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY;
@@ -95,7 +95,8 @@ public class DNConf {
final long blockReportInterval;
final long blockReportSplitThreshold;
final boolean peerStatsEnabled;
- final long slowPeersReportIntervalMs;
+ final boolean diskStatsEnabled;
+ final long outliersReportIntervalMs;
final long ibrInterval;
final long initialBlockReportDelayMs;
final long cacheReportInterval;
@@ -173,9 +174,12 @@ public class DNConf {
this.peerStatsEnabled = getConf().getBoolean(
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
- this.slowPeersReportIntervalMs = getConf().getTimeDuration(
- DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_KEY,
- DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_DEFAULT,
+ this.diskStatsEnabled = getConf().getBoolean(
+ DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY,
+ DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
+ this.outliersReportIntervalMs = getConf().getTimeDuration(
+ DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
+ DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
this.ibrInterval = getConf().getLong(
DFSConfigKeys.DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 5db41bd..6f24858 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -164,6 +164,7 @@ import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResour
import org.apache.hadoop.hdfs.server.datanode.erasurecode.ErasureCodingWorker;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeDiskMetrics;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodePeerMetrics;
import org.apache.hadoop.hdfs.server.datanode.web.DatanodeHttpServer;
@@ -336,6 +337,7 @@ public class DataNode extends ReconfigurableBase
DataNodeMetrics metrics;
@Nullable
private DataNodePeerMetrics peerMetrics;
+ private DataNodeDiskMetrics diskMetrics;
private InetSocketAddress streamingAddr;
// See the note below in incrDatanodeNetworkErrors re: concurrency.
@@ -1390,6 +1392,11 @@ public class DataNode extends ReconfigurableBase
dnConf.saslPropsResolver, dnConf.trustedChannelResolver);
saslServer = new SaslDataTransferServer(dnConf, blockPoolTokenSecretManager);
startMetricsLogger();
+
+ if (dnConf.diskStatsEnabled) {
+ diskMetrics = new DataNodeDiskMetrics(this,
+ dnConf.outliersReportIntervalMs);
+ }
}
/**
@@ -2046,6 +2053,9 @@ public class DataNode extends ReconfigurableBase
if (metrics != null) {
metrics.shutdown();
}
+ if (diskMetrics != null) {
+ diskMetrics.shutdownAndWait();
+ }
if (dataNodeInfoBeanName != null) {
MBeans.unregister(dataNodeInfoBeanName);
dataNodeInfoBeanName = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
new file mode 100644
index 0000000..85e2bd9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.metrics;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.util.Daemon;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This class detects and maintains DataNode disk outliers and their
+ * latencies for different ops (metadata, read, write).
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class DataNodeDiskMetrics {
+
+ public static final Logger LOG = LoggerFactory.getLogger(
+ DataNodeDiskMetrics.class);
+
+ private DataNode dn;
+ private final long MIN_OUTLIER_DETECTION_DISKS = 5;
+ private final long SLOW_DISK_LOW_THRESHOLD_MS = 20;
+ private final long detectionInterval;
+ private volatile boolean shouldRun;
+ private OutlierDetector slowDiskDetector;
+ private Daemon slowDiskDetectionDaemon;
+ private volatile Map<String, Map<DiskOutlierDetectionOp, Double>> diskOutliersStats;
+
+ public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) {
+ this.dn = dn;
+ this.detectionInterval = diskOutlierDetectionIntervalMs;
+ slowDiskDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_DISKS,
+ SLOW_DISK_LOW_THRESHOLD_MS);
+ shouldRun = true;
+ startDiskOutlierDetectionThread();
+ }
+
+ private void startDiskOutlierDetectionThread() {
+ slowDiskDetectionDaemon = new Daemon(new Runnable() {
+ @Override
+ public void run() {
+ while (shouldRun) {
+ Map<String, Double> metadataOpStats = Maps.newHashMap();
+ Map<String, Double> readIoStats = Maps.newHashMap();
+ Map<String, Double> writeIoStats = Maps.newHashMap();
+ FsDatasetSpi.FsVolumeReferences fsVolumeReferences = null;
+ try {
+ fsVolumeReferences = dn.getFSDataset().getFsVolumeReferences();
+ Iterator<FsVolumeSpi> volumeIterator = fsVolumeReferences
+ .iterator();
+ while (volumeIterator.hasNext()) {
+ FsVolumeSpi volume = volumeIterator.next();
+ DataNodeVolumeMetrics metrics = volumeIterator.next().getMetrics();
+ String volumeName = volume.getBaseURI().getPath();
+
+ metadataOpStats.put(volumeName,
+ metrics.getMetadataOperationMean());
+ readIoStats.put(volumeName, metrics.getReadIoMean());
+ writeIoStats.put(volumeName, metrics.getWriteIoMean());
+ }
+ } finally {
+ if (fsVolumeReferences != null) {
+ try {
+ fsVolumeReferences.close();
+ } catch (IOException e) {
+ LOG.error("Error in releasing FS Volume references", e);
+ }
+ }
+ }
+ if (metadataOpStats.isEmpty() && readIoStats.isEmpty() &&
+ writeIoStats.isEmpty()) {
+ LOG.debug("No disk stats available for detecting outliers.");
+ return;
+ }
+
+ detectAndUpdateDiskOutliers(metadataOpStats, readIoStats,
+ writeIoStats);
+
+ try {
+ Thread.sleep(detectionInterval);
+ } catch (InterruptedException e) {
+ LOG.error("Disk Outlier Detection thread interrupted", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ });
+ slowDiskDetectionDaemon.start();
+ }
+
+ private void detectAndUpdateDiskOutliers(Map<String, Double> metadataOpStats,
+ Map<String, Double> readIoStats, Map<String, Double> writeIoStats) {
+ Set<String> diskOutliersSet = Sets.newHashSet();
+
+ // Get MetadataOp Outliers
+ Map<String, Double> metadataOpOutliers = slowDiskDetector
+ .getOutliers(metadataOpStats);
+ if (!metadataOpOutliers.isEmpty()) {
+ diskOutliersSet.addAll(metadataOpOutliers.keySet());
+ }
+
+ // Get ReadIo Outliers
+ Map<String, Double> readIoOutliers = slowDiskDetector
+ .getOutliers(readIoStats);
+ if (!readIoOutliers.isEmpty()) {
+ diskOutliersSet.addAll(readIoOutliers.keySet());
+ }
+
+ // Get WriteIo Outliers
+ Map<String, Double> writeIoOutliers = slowDiskDetector
+ .getOutliers(writeIoStats);
+ if (!readIoOutliers.isEmpty()) {
+ diskOutliersSet.addAll(writeIoOutliers.keySet());
+ }
+
+ Map<String, Map<DiskOutlierDetectionOp, Double>> diskStats =
+ Maps.newHashMap();
+ for (String disk : diskOutliersSet) {
+ Map<DiskOutlierDetectionOp, Double> diskStat = Maps.newHashMap();
+ diskStat.put(DiskOutlierDetectionOp.METADATA, metadataOpStats.get(disk));
+ diskStat.put(DiskOutlierDetectionOp.READ, readIoStats.get(disk));
+ diskStat.put(DiskOutlierDetectionOp.WRITE, writeIoStats.get(disk));
+ diskStats.put(disk, diskStat);
+ }
+
+ diskOutliersStats = diskStats;
+ LOG.debug("Updated disk outliers.");
+ }
+
+ /**
+ * Lists the types of operations on which disk latencies are measured.
+ */
+ public enum DiskOutlierDetectionOp {
+ METADATA,
+ READ,
+ WRITE
+ }
+
+ public Map<String,
+ Map<DiskOutlierDetectionOp, Double>> getDiskOutliersStats() {
+ return diskOutliersStats;
+ }
+
+ public void shutdownAndWait() {
+ shouldRun = false;
+ slowDiskDetectionDaemon.interrupt();
+ try {
+ slowDiskDetectionDaemon.join();
+ } catch (InterruptedException e) {
+ LOG.error("Disk Outlier Detection daemon did not shutdown", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
index 5241c78..827bdd2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
@@ -52,8 +52,9 @@ public class DataNodePeerMetrics {
* Threshold in milliseconds below which a DataNode is definitely not slow.
*/
private static final long LOW_THRESHOLD_MS = 5;
+ private static final long MIN_OUTLIER_DETECTION_NODES = 10;
- private final SlowNodeDetector slowNodeDetector;
+ private final OutlierDetector slowNodeDetector;
/**
* Minimum number of packet send samples which are required to qualify
@@ -68,7 +69,8 @@ public class DataNodePeerMetrics {
final long windowSizeMs,
final int numWindows) {
this.name = name;
- this.slowNodeDetector = new SlowNodeDetector(LOW_THRESHOLD_MS);
+ this.slowNodeDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_NODES,
+ LOW_THRESHOLD_MS);
sendPacketDownstreamRollingAvgerages = new RollingAverages(
windowSizeMs, numWindows);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
new file mode 100644
index 0000000..771a17b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.metrics;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * A utility class to help detect resources (nodes/ disks) whose aggregate
+ * latency is an outlier within a given set.
+ *
+ * We use the median absolute deviation for outlier detection as
+ * described in the following publication:
+ *
+ * Leys, C., et al., Detecting outliers: Do not use standard deviation
+ * around the mean, use absolute deviation around the median.
+ * http://dx.doi.org/10.1016/j.jesp.2013.03.013
+ *
+ * We augment the above scheme with the following heuristics to be even
+ * more conservative:
+ *
+ * 1. Skip outlier detection if the sample size is too small.
+ * 2. Never flag resources whose aggregate latency is below a low threshold.
+ * 3. Never flag resources whose aggregate latency is less than a small
+ * multiple of the median.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OutlierDetector {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(OutlierDetector.class);
+
+ /**
+ * Minimum number of resources to run outlier detection.
+ */
+ private final long minNumResources;
+
+ /**
+ * The multiplier is from Leys, C. et al.
+ */
+ private static final double MAD_MULTIPLIER = (double) 1.4826;
+
+ /**
+ * Threshold in milliseconds below which a node/ disk is definitely not slow.
+ */
+ private final long lowThresholdMs;
+
+ /**
+ * Deviation multiplier. A sample is considered to be an outlier if it
+ * exceeds the median by (multiplier * median abs. deviation). 3 is a
+ * conservative choice.
+ */
+ private static final int DEVIATION_MULTIPLIER = 3;
+
+ /**
+ * If most of the samples are clustered together, the MAD can be
+ * low. The median multiplier introduces another safeguard to avoid
+ * overaggressive outlier detection.
+ */
+ @VisibleForTesting
+ static final int MEDIAN_MULTIPLIER = 3;
+
+ public OutlierDetector(long minNumResources, long lowThresholdMs) {
+ this.minNumResources = minNumResources;
+ this.lowThresholdMs = lowThresholdMs;
+ }
+
+ /**
+ * Return a set of nodes/ disks whose latency is much higher than
+ * their counterparts. The input is a map of (resource -> aggregate latency)
+ * entries.
+ *
+ * The aggregate may be an arithmetic mean or a percentile e.g.
+ * 90th percentile. Percentiles are a better choice than median
+ * since latency is usually not a normal distribution.
+ *
+ * This method allocates temporary memory O(n) and
+ * has run time O(n.log(n)), where n = stats.size().
+ *
+ * @return
+ */
+ public Map<String, Double> getOutliers(Map<String, Double> stats) {
+ if (stats.size() < minNumResources) {
+ LOG.debug("Skipping statistical outlier detection as we don't have " +
+ "latency data for enough resources. Have {}, need at least {}",
+ stats.size(), minNumResources);
+ return ImmutableMap.of();
+ }
+ // Compute the median absolute deviation of the aggregates.
+ final List<Double> sorted = new ArrayList<>(stats.values());
+ Collections.sort(sorted);
+ final Double median = computeMedian(sorted);
+ final Double mad = computeMad(sorted);
+ Double upperLimitLatency = Math.max(
+ lowThresholdMs, median * MEDIAN_MULTIPLIER);
+ upperLimitLatency = Math.max(
+ upperLimitLatency, median + (DEVIATION_MULTIPLIER * mad));
+
+ final Map<String, Double> slowResources = new HashMap<>();
+
+ LOG.trace("getOutliers: List={}, MedianLatency={}, " +
+ "MedianAbsoluteDeviation={}, upperLimitLatency={}",
+ sorted, median, mad, upperLimitLatency);
+
+ // Find resources whose latency exceeds the threshold.
+ for (Map.Entry<String, Double> entry : stats.entrySet()) {
+ if (entry.getValue() > upperLimitLatency) {
+ slowResources.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ return slowResources;
+ }
+
+ /**
+ * Compute the Median Absolute Deviation of a sorted list.
+ */
+ public static Double computeMad(List<Double> sortedValues) {
+ if (sortedValues.size() == 0) {
+ throw new IllegalArgumentException(
+ "Cannot compute the Median Absolute Deviation " +
+ "of an empty list.");
+ }
+
+ // First get the median of the values.
+ Double median = computeMedian(sortedValues);
+ List<Double> deviations = new ArrayList<>(sortedValues);
+
+ // Then update the list to store deviation from the median.
+ for (int i = 0; i < sortedValues.size(); ++i) {
+ deviations.set(i, Math.abs(sortedValues.get(i) - median));
+ }
+
+ // Finally get the median absolute deviation.
+ Collections.sort(deviations);
+ return computeMedian(deviations) * MAD_MULTIPLIER;
+ }
+
+ /**
+ * Compute the median of a sorted list.
+ */
+ public static Double computeMedian(List<Double> sortedValues) {
+ if (sortedValues.size() == 0) {
+ throw new IllegalArgumentException(
+ "Cannot compute the median of an empty list.");
+ }
+
+ Double median = sortedValues.get(sortedValues.size() / 2);
+ if (sortedValues.size() % 2 == 0) {
+ median += sortedValues.get((sortedValues.size() / 2) - 1);
+ median /= 2;
+ }
+ return median;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/SlowNodeDetector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/SlowNodeDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/SlowNodeDetector.java
deleted file mode 100644
index b6278ce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/SlowNodeDetector.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.datanode.metrics;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-
-/**
- * A utility class to help detect nodes whose aggregate latency
- * is an outlier within a given set.
- *
- * We use the median absolute deviation for outlier detection as
- * described in the following publication:
- *
- * Leys, C., et al., Detecting outliers: Do not use standard deviation
- * around the mean, use absolute deviation around the median.
- * http://dx.doi.org/10.1016/j.jesp.2013.03.013
- *
- * We augment the above scheme with the following heuristics to be even
- * more conservative:
- *
- * 1. Skip outlier detection if the sample size is too small.
- * 2. Never flag nodes whose aggregate latency is below a low threshold.
- * 3. Never flag nodes whose aggregate latency is less than a small
- * multiple of the median.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class SlowNodeDetector {
- public static final Logger LOG =
- LoggerFactory.getLogger(SlowNodeDetector.class);
-
- /**
- * Minimum number of peers to run outlier detection.
- */
- private static long minOutlierDetectionPeers = 10;
-
- /**
- * The multiplier is from Leys, C. et al.
- */
- private static final double MAD_MULTIPLIER = (double) 1.4826;
-
- /**
- * Threshold in milliseconds below which a DataNode is definitely not slow.
- */
- private final long lowThresholdMs;
-
- /**
- * Deviation multiplier. A sample is considered to be an outlier if it
- * exceeds the median by (multiplier * median abs. deviation). 3 is a
- * conservative choice.
- */
- private static final int DEVIATION_MULTIPLIER = 3;
-
- /**
- * If most of the samples are clustered together, the MAD can be
- * low. The median multiplier introduces another safeguard to avoid
- * overaggressive outlier detection.
- */
- @VisibleForTesting
- static final int MEDIAN_MULTIPLIER = 3;
-
- public SlowNodeDetector(long lowThresholdMs) {
- this.lowThresholdMs = lowThresholdMs;
- }
-
- /**
- * Return a set of DataNodes whose latency is much higher than
- * their peers. The input is a map of (node -> aggregate latency)
- * entries.
- *
- * The aggregate may be an arithmetic mean or a percentile e.g.
- * 90th percentile. Percentiles are a better choice than median
- * since latency is usually not a normal distribution.
- *
- * This method allocates temporary memory O(n) and
- * has run time O(n.log(n)), where n = stats.size().
- *
- * @return
- */
- public Map<String, Double> getOutliers(Map<String, Double> stats) {
- if (stats.size() < minOutlierDetectionPeers) {
- LOG.debug("Skipping statistical outlier detection as we don't have " +
- "latency data for enough peers. Have {}, need at least {}",
- stats.size(), minOutlierDetectionPeers);
- return ImmutableMap.of();
- }
- // Compute the median absolute deviation of the aggregates.
- final List<Double> sorted = new ArrayList<>(stats.values());
- Collections.sort(sorted);
- final Double median = computeMedian(sorted);
- final Double mad = computeMad(sorted);
- Double upperLimitLatency = Math.max(
- lowThresholdMs, median * MEDIAN_MULTIPLIER);
- upperLimitLatency = Math.max(
- upperLimitLatency, median + (DEVIATION_MULTIPLIER * mad));
-
- final Map<String, Double> slowNodes = new HashMap<>();
-
- LOG.trace("getOutliers: List={}, MedianLatency={}, " +
- "MedianAbsoluteDeviation={}, upperLimitLatency={}",
- sorted, median, mad, upperLimitLatency);
-
- // Find nodes whose latency exceeds the threshold.
- for (Map.Entry<String, Double> entry : stats.entrySet()) {
- if (entry.getValue() > upperLimitLatency) {
- slowNodes.put(entry.getKey(), entry.getValue());
- }
- }
-
- return slowNodes;
- }
-
- /**
- * Compute the Median Absolute Deviation of a sorted list.
- */
- public static Double computeMad(List<Double> sortedValues) {
- if (sortedValues.size() == 0) {
- throw new IllegalArgumentException(
- "Cannot compute the Median Absolute Deviation " +
- "of an empty list.");
- }
-
- // First get the median of the values.
- Double median = computeMedian(sortedValues);
- List<Double> deviations = new ArrayList<>(sortedValues);
-
- // Then update the list to store deviation from the median.
- for (int i = 0; i < sortedValues.size(); ++i) {
- deviations.set(i, Math.abs(sortedValues.get(i) - median));
- }
-
- // Finally get the median absolute deviation.
- Collections.sort(deviations);
- return computeMedian(deviations) * MAD_MULTIPLIER;
- }
-
- /**
- * Compute the median of a sorted list.
- */
- public static Double computeMedian(List<Double> sortedValues) {
- if (sortedValues.size() == 0) {
- throw new IllegalArgumentException(
- "Cannot compute the median of an empty list.");
- }
-
- Double median = sortedValues.get(sortedValues.size() / 2);
- if (sortedValues.size() % 2 == 0) {
- median += sortedValues.get((sortedValues.size() / 2) - 1);
- median /= 2;
- }
- return median;
- }
-
- /**
- * This method *must not* be used outside of unit tests.
- */
- @VisibleForTesting
- static void setMinOutlierDetectionPeers(long minOutlierDetectionPeers) {
- SlowNodeDetector.minOutlierDetectionPeers = minOutlierDetectionPeers;
- }
-
- @VisibleForTesting
- static long getMinOutlierDetectionPeers() {
- return minOutlierDetectionPeers;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index c220025..36b93b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2009,7 +2009,7 @@
</property>
<property>
- <name>dfs.datanode.slow.peers.report.interval</name>
+ <name>dfs.datanode.outliers.report.interval</name>
<value>30m</value>
<description>
This setting controls how frequently DataNodes will report their peer
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
index 34e15e5..eb7769e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
@@ -54,13 +54,14 @@ public class TestDataNodeOutlierDetectionViaMetrics {
private static final int ROLLING_AVERAGE_WINDOWS = 10;
private static final int SLOW_NODE_LATENCY_MS = 20_000;
private static final int FAST_NODE_MAX_LATENCY_MS = 5;
+ private static final long MIN_OUTLIER_DETECTION_PEERS = 10;
private Random random = new Random(System.currentTimeMillis());
@Before
public void setup() {
GenericTestUtils.setLogLevel(DataNodePeerMetrics.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SlowNodeDetector.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(OutlierDetector.LOG, Level.ALL);
}
/**
@@ -111,8 +112,7 @@ public class TestDataNodeOutlierDetectionViaMetrics {
*/
public void injectFastNodesSamples(DataNodePeerMetrics peerMetrics) {
for (int nodeIndex = 0;
- nodeIndex < SlowNodeDetector.getMinOutlierDetectionPeers();
- ++nodeIndex) {
+ nodeIndex < MIN_OUTLIER_DETECTION_PEERS; ++nodeIndex) {
final String nodeName = "FastNode-" + nodeIndex;
LOG.info("Generating stats for node {}", nodeName);
for (int i = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestSlowNodeDetector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestSlowNodeDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestSlowNodeDetector.java
index 7b368c4..f06a87a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestSlowNodeDetector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestSlowNodeDetector.java
@@ -40,7 +40,7 @@ import java.util.Set;
import static org.junit.Assert.assertTrue;
/**
- * Unit tests for {@link SlowNodeDetector}.
+ * Unit tests for {@link OutlierDetector}.
*/
public class TestSlowNodeDetector {
public static final Logger LOG =
@@ -183,7 +183,7 @@ public class TestSlowNodeDetector {
.put(ImmutableMap.of(
"n1", LOW_THRESHOLD + 0.1,
"n2", LOW_THRESHOLD + 0.1,
- "n3", LOW_THRESHOLD * SlowNodeDetector.MEDIAN_MULTIPLIER - 0.1),
+ "n3", LOW_THRESHOLD * OutlierDetector.MEDIAN_MULTIPLIER - 0.1),
ImmutableSet.of())
// A statistical outlier must be returned if it is outside a
@@ -192,7 +192,7 @@ public class TestSlowNodeDetector {
"n1", LOW_THRESHOLD + 0.1,
"n2", LOW_THRESHOLD + 0.1,
"n3", (LOW_THRESHOLD + 0.1) *
- SlowNodeDetector.MEDIAN_MULTIPLIER + 0.1),
+ OutlierDetector.MEDIAN_MULTIPLIER + 0.1),
ImmutableSet.of("n3"))
// Only the statistical outliers n3 and n11 should be returned.
@@ -233,13 +233,13 @@ public class TestSlowNodeDetector {
.build();
- private SlowNodeDetector slowNodeDetector;
+ private OutlierDetector slowNodeDetector;
@Before
public void setup() {
- slowNodeDetector = new SlowNodeDetector((long) LOW_THRESHOLD);
- SlowNodeDetector.setMinOutlierDetectionPeers(MIN_OUTLIER_DETECTION_PEERS);
- GenericTestUtils.setLogLevel(SlowNodeDetector.LOG, Level.ALL);
+ slowNodeDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_PEERS,
+ (long) LOW_THRESHOLD);
+ GenericTestUtils.setLogLevel(OutlierDetector.LOG, Level.ALL);
}
@Test
@@ -259,7 +259,7 @@ public class TestSlowNodeDetector {
}
/**
- * Unit test for {@link SlowNodeDetector#computeMedian(List)}.
+ * Unit test for {@link OutlierDetector#computeMedian(List)}.
*/
@Test
public void testMediansFromTestMatrix() {
@@ -267,7 +267,7 @@ public class TestSlowNodeDetector {
medianTestMatrix.entrySet()) {
final List<Double> inputList = new ArrayList<>(entry.getKey());
Collections.sort(inputList);
- final Double median = SlowNodeDetector.computeMedian(inputList);
+ final Double median = OutlierDetector.computeMedian(inputList);
final Double expectedMedian = entry.getValue().getLeft();
// Ensure that the median is within 0.001% of expected.
@@ -283,7 +283,7 @@ public class TestSlowNodeDetector {
}
/**
- * Unit test for {@link SlowNodeDetector#computeMad(List)}.
+ * Unit test for {@link OutlierDetector#computeMad(List)}.
*/
@Test
public void testMadsFromTestMatrix() {
@@ -291,7 +291,7 @@ public class TestSlowNodeDetector {
medianTestMatrix.entrySet()) {
final List<Double> inputList = new ArrayList<>(entry.getKey());
Collections.sort(inputList);
- final Double mad = SlowNodeDetector.computeMad(inputList);
+ final Double mad = OutlierDetector.computeMad(inputList);
final Double expectedMad = entry.getValue().getRight();
// Ensure that the MAD is within 0.001% of expected.
@@ -316,20 +316,20 @@ public class TestSlowNodeDetector {
}
/**
- * Verify that {@link SlowNodeDetector#computeMedian(List)} throws when
+ * Verify that {@link OutlierDetector#computeMedian(List)} throws when
* passed an empty list.
*/
@Test(expected=IllegalArgumentException.class)
public void testMedianOfEmptyList() {
- SlowNodeDetector.computeMedian(Collections.emptyList());
+ OutlierDetector.computeMedian(Collections.emptyList());
}
/**
- * Verify that {@link SlowNodeDetector#computeMad(List)} throws when
+ * Verify that {@link OutlierDetector#computeMad(List)} throws when
* passed an empty list.
*/
@Test(expected=IllegalArgumentException.class)
public void testMadOfEmptyList() {
- SlowNodeDetector.computeMedian(Collections.emptyList());
+ OutlierDetector.computeMedian(Collections.emptyList());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit:
HADOOP-14116:FailoverOnNetworkExceptionRetry does not wait when failover on
certain exception. Contributed by Jian He
Posted by jh...@apache.org.
HADOOP-14116:FailoverOnNetworkExceptionRetry does not wait when failover
on certain exception. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/289bc50e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/289bc50e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/289bc50e
Branch: refs/heads/YARN-5734
Commit: 289bc50e663b882956878eeaefe0eaa1ef4ed39e
Parents: 53d372a
Author: Xuan <xg...@apache.org>
Authored: Fri Feb 24 11:42:23 2017 -0800
Committer: Xuan <xg...@apache.org>
Committed: Fri Feb 24 11:42:23 2017 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/289bc50e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
index 0c523a5..d6f3e04 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
@@ -683,7 +683,8 @@ public class RetryPolicies {
} else if (e instanceof SocketException
|| (e instanceof IOException && !(e instanceof RemoteException))) {
if (isIdempotentOrAtMostOnce) {
- return RetryAction.FAILOVER_AND_RETRY;
+ return new RetryAction(RetryAction.RetryDecision.FAILOVER_AND_RETRY,
+ getFailoverOrRetrySleepTime(retries));
} else {
return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
"the invoked method is not idempotent, and unable to determine "
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-6172. FSLeafQueue demand
update needs to be atomic. (Miklos Szegedi via kasha)
Posted by jh...@apache.org.
YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbfe86de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbfe86de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbfe86de
Branch: refs/heads/YARN-5734
Commit: fbfe86deea5f2aa857cd13fee913b7becee57f93
Parents: 815d535
Author: Karthik Kambatla <ka...@apache.org>
Authored: Sun Feb 26 20:36:33 2017 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Sun Feb 26 20:36:33 2017 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSLeafQueue.java | 21 +++++------------
.../scheduler/fair/TestFSAppStarvation.java | 24 ++++++++++++++++----
2 files changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index d0e0961..aad2916 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -331,20 +331,22 @@ public class FSLeafQueue extends FSQueue {
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
- demand = Resources.createResource(0);
+ Resource tmpDemand = Resources.createResource(0);
readLock.lock();
try {
for (FSAppAttempt sched : runnableApps) {
- updateDemandForApp(sched);
+ sched.updateDemand();
+ Resources.addTo(tmpDemand, sched.getDemand());
}
for (FSAppAttempt sched : nonRunnableApps) {
- updateDemandForApp(sched);
+ sched.updateDemand();
+ Resources.addTo(tmpDemand, sched.getDemand());
}
} finally {
readLock.unlock();
}
// Cap demand to maxShare to limit allocation to maxShare
- demand = Resources.componentwiseMin(demand, maxShare);
+ demand = Resources.componentwiseMin(tmpDemand, maxShare);
if (LOG.isDebugEnabled()) {
LOG.debug("The updated demand for " + getName() + " is " + demand
+ "; the max is " + maxShare);
@@ -352,17 +354,6 @@ public class FSLeafQueue extends FSQueue {
+ getFairShare());
}
}
-
- private void updateDemandForApp(FSAppAttempt sched) {
- sched.updateDemand();
- Resource toAdd = sched.getDemand();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
- + "; Total resource demand for " + getName() + " now "
- + demand);
- }
- demand = Resources.add(demand, toAdd);
- }
@Override
public Resource assignContainer(FSSchedulerNode node) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 2eacc9e..0712b4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -96,6 +96,14 @@ public class TestFSAppStarvation extends FairSchedulerTestBase {
public void testPreemptionEnabled() throws Exception {
setupClusterAndSubmitJobs();
+ // Wait for apps to be processed by MockPreemptionThread
+ for (int i = 0; i < 6000; ++i) {
+ if (preemptionThread.uniqueAppsAdded() >= 3) {
+ break;
+ }
+ Thread.sleep(10);
+ }
+
assertNotNull("FSContext does not have an FSStarvedApps instance",
scheduler.getContext().getStarvedApps());
assertEquals("Expecting 3 starved applications, one each for the "
@@ -113,8 +121,19 @@ public class TestFSAppStarvation extends FairSchedulerTestBase {
clock.tickMsec(
FairSchedulerWithMockPreemption.DELAY_FOR_NEXT_STARVATION_CHECK_MS);
scheduler.update();
+
+ // Wait for apps to be processed by MockPreemptionThread
+ for (int i = 0; i < 6000; ++i) {
+ if(preemptionThread.totalAppsAdded() >
+ preemptionThread.uniqueAppsAdded()) {
+ break;
+ }
+ Thread.sleep(10);
+ }
+
assertTrue("Each app is marked as starved exactly once",
- preemptionThread.totalAppsAdded() > preemptionThread.uniqueAppsAdded());
+ preemptionThread.totalAppsAdded() >
+ preemptionThread.uniqueAppsAdded());
}
/*
@@ -154,9 +173,6 @@ public class TestFSAppStarvation extends FairSchedulerTestBase {
// Scheduler update to populate starved apps
scheduler.update();
-
- // Wait for apps to be processed by MockPreemptionThread
- Thread.yield();
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-5280. Allow YARN containers
to run with Java Security Manager (gphillips via rkanter)
Posted by jh...@apache.org.
YARN-5280. Allow YARN containers to run with Java Security Manager (gphillips via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f6dfe02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f6dfe02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f6dfe02
Branch: refs/heads/YARN-5734
Commit: 6f6dfe0202249c129b36edfd145a2224140139cc
Parents: 899d5c4
Author: Robert Kanter <rk...@apache.org>
Authored: Wed Mar 1 14:53:47 2017 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Mar 1 14:53:47 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 20 +
.../src/main/resources/yarn-default.xml | 26 +
.../server/nodemanager/ContainerExecutor.java | 9 +
.../nodemanager/LinuxContainerExecutor.java | 23 +
.../launcher/ContainerLaunch.java | 34 +-
.../DelegatingLinuxContainerRuntime.java | 30 +-
.../JavaSandboxLinuxContainerRuntime.java | 495 +++++++++++++++++++
.../runtime/LinuxContainerRuntimeConstants.java | 2 +
.../executor/ContainerPrepareContext.java | 119 +++++
.../src/main/resources/java.policy | 63 +++
.../TestJavaSandboxLinuxContainerRuntime.java | 364 ++++++++++++++
11 files changed, 1155 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index cdccec6..b366855 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1465,6 +1465,26 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK =
"host";
+ /** The mode in which the Java Container Sandbox should run detailed by
+ * the JavaSandboxLinuxContainerRuntime. */
+ public static final String YARN_CONTAINER_SANDBOX =
+ LINUX_CONTAINER_RUNTIME_PREFIX + "sandbox-mode";
+ public static final String DEFAULT_YARN_CONTAINER_SANDBOX = "disabled";
+
+ /** Permissions for application local directories.*/
+ public static final String YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS =
+ YARN_CONTAINER_SANDBOX + ".local-dirs.permissions";
+ public static final String DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS =
+ "read";
+
+ /** Location for non-default java policy file.*/
+ public static final String YARN_CONTAINER_SANDBOX_POLICY =
+ YARN_CONTAINER_SANDBOX + ".policy";
+
+ /** The group which will run by default without the java security manager.*/
+ public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP =
+ YARN_CONTAINER_SANDBOX + ".whitelist-group";
+
/** The path to the Linux container executor.*/
public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH =
NM_PREFIX + "linux-container-executor.path";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 368946e..645a342 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1584,6 +1584,32 @@
</property>
<property>
+ <description>The mode in which the Java Container Sandbox should run detailed by
+ the JavaSandboxLinuxContainerRuntime.</description>
+ <name>yarn.nodemanager.runtime.linux.sandbox-mode</name>
+ <value>disabled</value>
+ </property>
+
+ <property>
+ <description>Permissions for application local directories.</description>
+ <name>yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions</name>
+ <value>read</value>
+ </property>
+
+ <property>
+ <description>Location for non-default java policy file.</description>
+ <name>yarn.nodemanager.runtime.linux.sandbox-mode.policy</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>The group which will run by default without the java security
+ manager.</description>
+ <name>yarn.nodemanager.runtime.linux.sandbox-mode.whitelist-group</name>
+ <value></value>
+ </property>
+
+ <property>
<description>This flag determines whether memory limit will be set for the Windows Job
Object of the containers launched by the default container executor.</description>
<name>yarn.nodemanager.windows-container.memory-limit.enabled</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 158585e..64d2af9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext;
@@ -148,6 +149,14 @@ public abstract class ContainerExecutor implements Configurable {
public abstract void startLocalizer(LocalizerStartContext ctx)
throws IOException, InterruptedException;
+ /**
+ * Prepare the container prior to the launch environment being written.
+ * @param ctx Encapsulates information necessary for launching containers.
+ * @throws IOException if errors occur during container preparation
+ */
+ public void prepareContainer(ContainerPrepareContext ctx) throws
+ IOException{
+ }
/**
* Launch the container on the node. This is a blocking call and returns only
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 323c443..46b6dfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
@@ -363,6 +364,28 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ public void prepareContainer(ContainerPrepareContext ctx) throws IOException {
+
+ ContainerRuntimeContext.Builder builder =
+ new ContainerRuntimeContext.Builder(ctx.getContainer());
+
+ builder.setExecutionAttribute(LOCALIZED_RESOURCES,
+ ctx.getLocalizedResources())
+ .setExecutionAttribute(USER, ctx.getUser())
+ .setExecutionAttribute(CONTAINER_LOCAL_DIRS,
+ ctx.getContainerLocalDirs())
+ .setExecutionAttribute(CONTAINER_RUN_CMDS, ctx.getCommands())
+ .setExecutionAttribute(CONTAINER_ID_STR,
+ ctx.getContainer().getContainerId().toString());
+
+ try {
+ linuxContainerRuntime.prepareContainer(builder.build());
+ } catch (ContainerExecutionException e) {
+ throw new IOException("Unable to prepare container: ", e);
+ }
+ }
+
+ @Override
public int launchContainer(ContainerStartContext ctx) throws IOException {
Container container = ctx.getContainer();
Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 823457f..a1c407f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
@@ -198,18 +199,14 @@ public class ContainerLaunch implements Callable<Integer> {
FileContext lfs = FileContext.getLocalFSFileContext();
- Path nmPrivateContainerScriptPath =
- dirsHandler.getLocalPathForWrite(
+ Path nmPrivateContainerScriptPath = dirsHandler.getLocalPathForWrite(
getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ CONTAINER_SCRIPT);
- Path nmPrivateTokensPath =
- dirsHandler.getLocalPathForWrite(
- getContainerPrivateDir(appIdStr, containerIdStr)
- + Path.SEPARATOR
+ Path nmPrivateTokensPath = dirsHandler.getLocalPathForWrite(
+ getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
containerIdStr));
- Path nmPrivateClasspathJarDir =
- dirsHandler.getLocalPathForWrite(
+ Path nmPrivateClasspathJarDir = dirsHandler.getLocalPathForWrite(
getContainerPrivateDir(appIdStr, containerIdStr));
DataOutputStream containerScriptOutStream = null;
DataOutputStream tokensOutStream = null;
@@ -224,7 +221,6 @@ public class ContainerLaunch implements Callable<Integer> {
recordContainerWorkDir(containerID, containerWorkDir.toString());
String pidFileSubpath = getPidFileSubpath(appIdStr, containerIdStr);
-
// pid file should be in nm private dir so that it is not
// accessible by users
pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
@@ -240,11 +236,9 @@ public class ContainerLaunch implements Callable<Integer> {
throw new IOException("Most of the disks failed. "
+ dirsHandler.getDisksHealthReport(false));
}
-
try {
// /////////// Write out the container-script in the nmPrivate space.
List<Path> appDirs = new ArrayList<Path>(localDirs.size());
-
for (String localDir : localDirs) {
Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
Path userdir = new Path(usersdir, user);
@@ -258,18 +252,22 @@ public class ContainerLaunch implements Callable<Integer> {
// Set the token location too.
environment.put(
ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME,
- new Path(containerWorkDir,
+ new Path(containerWorkDir,
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
// Sanitize the container's environment
sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs,
- containerLogDirs,
- localResources, nmPrivateClasspathJarDir);
-
+ containerLogDirs, localResources, nmPrivateClasspathJarDir);
+
+ exec.prepareContainer(new ContainerPrepareContext.Builder()
+ .setContainer(container)
+ .setLocalizedResources(localResources)
+ .setUser(user)
+ .setContainerLocalDirs(containerLocalDirs)
+ .setCommands(launchContext.getCommands()).build());
// Write out the environment
exec.writeLaunchEnv(containerScriptOutStream, environment,
localResources, launchContext.getCommands(),
new Path(containerLogDirs.get(0)), user);
-
// /////////// End of writing out container-script
// /////////// Write out the container-tokens in the nmPrivate space.
@@ -295,8 +293,7 @@ public class ContainerLaunch implements Callable<Integer> {
.setFilecacheDirs(filecacheDirs)
.setUserLocalDirs(userLocalDirs)
.setContainerLocalDirs(containerLocalDirs)
- .setContainerLogDirs(containerLogDirs)
- .build());
+ .setContainerLogDirs(containerLogDirs).build());
} catch (Throwable e) {
LOG.warn("Failed to launch container.", e);
dispatcher.getEventHandler().handle(new ContainerExitEvent(
@@ -308,7 +305,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
handleContainerExitCode(ret, containerLogDir);
-
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index c12858e..896ffd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -35,10 +35,10 @@ import java.util.Map;
/**
* This class is a {@link ContainerRuntime} implementation that delegates all
- * operations to either a {@link DefaultLinuxContainerRuntime} instance or a
- * {@link DockerLinuxContainerRuntime} instance, depending on whether the
- * {@link DockerLinuxContainerRuntime} instance believes the operation to be
- * requesting a Docker container.
+ * operations to a {@link DefaultLinuxContainerRuntime} instance, a
+ * {@link DockerLinuxContainerRuntime} instance, or a
+ * {@link JavaSandboxLinuxContainerRuntime} instance depending on whether
+ * each instance believes the operation to be within its scope.
*
* @see DockerLinuxContainerRuntime#isDockerContainerRequested
*/
@@ -49,6 +49,7 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
.getLog(DelegatingLinuxContainerRuntime.class);
private DefaultLinuxContainerRuntime defaultLinuxContainerRuntime;
private DockerLinuxContainerRuntime dockerLinuxContainerRuntime;
+ private JavaSandboxLinuxContainerRuntime javaSandboxLinuxContainerRuntime;
@Override
public void initialize(Configuration conf)
@@ -61,15 +62,20 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
dockerLinuxContainerRuntime = new DockerLinuxContainerRuntime(
privilegedOperationExecutor);
dockerLinuxContainerRuntime.initialize(conf);
+ javaSandboxLinuxContainerRuntime = new JavaSandboxLinuxContainerRuntime(
+ privilegedOperationExecutor);
+ javaSandboxLinuxContainerRuntime.initialize(conf);
}
- private LinuxContainerRuntime pickContainerRuntime(Container container) {
- Map<String, String> env = container.getLaunchContext().getEnvironment();
+ private LinuxContainerRuntime pickContainerRuntime(
+ Map<String, String> environment){
LinuxContainerRuntime runtime;
- if (DockerLinuxContainerRuntime.isDockerContainerRequested(env)){
+ if (DockerLinuxContainerRuntime.isDockerContainerRequested(environment)){
runtime = dockerLinuxContainerRuntime;
- } else {
+ } else if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()) {
+ runtime = javaSandboxLinuxContainerRuntime;
+ } else {
runtime = defaultLinuxContainerRuntime;
}
@@ -81,12 +87,14 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
return runtime;
}
+ private LinuxContainerRuntime pickContainerRuntime(Container container) {
+ return pickContainerRuntime(container.getLaunchContext().getEnvironment());
+ }
+
@Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
- Container container = ctx.getContainer();
- LinuxContainerRuntime runtime = pickContainerRuntime(container);
-
+ LinuxContainerRuntime runtime = pickContainerRuntime(ctx.getContainer());
runtime.prepareContainer(ctx);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
new file mode 100644
index 0000000..6dc627b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.linux.runtime;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+import org.apache.log4j.Logger;
+
+import java.io.FilePermission;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.security.AllPermission;
+import java.util.Formatter;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.fs.Path.SEPARATOR;
+import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR;
+import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCALIZED_RESOURCES;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER;
+/**
+ * <p>This class extends the {@link DefaultLinuxContainerRuntime} specifically
+ * for containers which run Java commands. It generates a new java security
+ * policy file per container and modifies the java command to enable the
+ * Java Security Manager with the generated policy.</p>
+ *
+ * The behavior of the {@link JavaSandboxLinuxContainerRuntime} can be modified
+ * using the following settings:
+ *
+ * <ul>
+ * <li>
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX} :
+ * This yarn-site.xml setting has three options:
+ * <ul>
+ * <li>disabled - Default behavior. {@link LinuxContainerRuntime}
+ * is disabled</li>
+ * <li>permissive - JVM containers will run with Java Security Manager
+ * enabled. Non-JVM containers will run normally</li>
+ * <li>enforcing - JVM containers will run with Java Security Manager
+ * enabled. Non-JVM containers will be prevented from executing and an
+ * {@link ContainerExecutionException} will be thrown.</li>
+ * </ul>
+ * </li>
+ * <li>
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS} :
+ * Determines the file permissions for the application directories. The
+ * permissions come in the form of comma separated values
+ * (e.g. read,write,execute,delete). Defaults to {@code read} for read-only.
+ * </li>
+ * <li>
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY} :
+ * Accepts canonical path to a java policy file on the local filesystem.
+ * This file will be loaded as the base policy, any additional container
+ * grants will be appended to this base file. If not specified, the default
+ * java.policy file provided with hadoop resources will be used.
+ * </li>
+ * <li>
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} :
+ * Optional setting to specify a YARN queue which will be exempt from the
+ * sand-boxing process.
+ * </li>
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class JavaSandboxLinuxContainerRuntime
+ extends DefaultLinuxContainerRuntime {
+ private static final Log LOG =
+ LogFactory.getLog(DefaultLinuxContainerRuntime.class);
+ private Configuration configuration;
+ private SandboxMode sandboxMode;
+
+ public static final String POLICY_FILE_DIR = "nm-sandbox-policies";
+
+ private static Path policyFileDir;
+ private static final FileAttribute<Set<PosixFilePermission>> POLICY_ATTR =
+ PosixFilePermissions.asFileAttribute(
+ PosixFilePermissions.fromString("rwxr-xr-x"));
+
+ private Map<String, Path> containerPolicies = new HashMap<>();
+
+ /**
+ * Create an instance using the given {@link PrivilegedOperationExecutor}
+ * instance for performing operations.
+ *
+ * @param privilegedOperationExecutor the {@link PrivilegedOperationExecutor}
+ * instance
+ */
+ public JavaSandboxLinuxContainerRuntime(
+ PrivilegedOperationExecutor privilegedOperationExecutor) {
+ super(privilegedOperationExecutor);
+ }
+
+ @Override
+ public void initialize(Configuration conf)
+ throws ContainerExecutionException {
+ this.configuration = conf;
+ this.sandboxMode =
+ SandboxMode.get(
+ this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
+ YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
+
+ initializePolicyDir();
+
+ super.initialize(conf);
+ }
+
+ /**
+ * Initialize the Java Security Policy directory. Either creates the
+ * directory if it doesn't exist, or clears the contents of the directory if
+ * already created.
+ * @throws ContainerExecutionException If unable to resolve policy directory
+ */
+ private void initializePolicyDir() throws ContainerExecutionException {
+ String hadoopTempDir = configuration.get("hadoop.tmp.dir");
+ if (hadoopTempDir == null) {
+ throw new ContainerExecutionException("hadoop.tmp.dir not set!");
+ }
+ policyFileDir = Paths.get(hadoopTempDir, POLICY_FILE_DIR);
+ //Delete any existing policy files if the directory has already been created
+ if(Files.exists(policyFileDir)){
+ try (DirectoryStream<Path> stream =
+ Files.newDirectoryStream(policyFileDir)){
+ for(Path policyFile : stream){
+ Files.delete(policyFile);
+ }
+ }catch(IOException e){
+ throw new ContainerExecutionException("Unable to initialize policy "
+ + "directory: " + e);
+ }
+ } else {
+ try {
+ policyFileDir = Files.createDirectories(
+ Paths.get(hadoopTempDir, POLICY_FILE_DIR), POLICY_ATTR);
+ } catch (IOException e) {
+ throw new ContainerExecutionException("Unable to create policy file " +
+ "directory: " + e);
+ }
+ }
+ }
+
+ /**
+ * Prior to environment from being written locally need to generate
+ * policy file which limits container access to a small set of directories.
+ * Additionally the container run command needs to be modified to include
+ * flags to enable the java security manager with the generated policy.
+ * <br>
+ * The Java Sandbox will be circumvented if the user is a member of the
+ * group specified in:
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} and if
+ * they do not include the JVM flag:
+ * {@value NMContainerPolicyUtils#SECURITY_FLAG}
+ *
+ * @param ctx The {@link ContainerRuntimeContext} containing container
+ * setup properties.
+ * @throws ContainerExecutionException Exception thrown if temporary policy
+ * file directory can't be created, or if any exceptions occur during policy
+ * file parsing and generation.
+ */
+ @Override
+ public void prepareContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+
+ @SuppressWarnings("unchecked")
+ List<String> localDirs =
+ ctx.getExecutionAttribute(CONTAINER_LOCAL_DIRS);
+ @SuppressWarnings("unchecked")
+ Map<org.apache.hadoop.fs.Path, List<String>> resources =
+ ctx.getExecutionAttribute(LOCALIZED_RESOURCES);
+ @SuppressWarnings("unchecked")
+ List<String> commands =
+ ctx.getExecutionAttribute(CONTAINER_RUN_CMDS);
+ Map<String, String> env =
+ ctx.getContainer().getLaunchContext().getEnvironment();
+
+ if(!isSandboxContainerWhitelisted(ctx, commands)) {
+ String tmpDirBase = configuration.get("hadoop.tmp.dir");
+ if (tmpDirBase == null) {
+ throw new ContainerExecutionException("hadoop.tmp.dir not set!");
+ }
+
+ OutputStream policyOutputStream = null;
+ try {
+ String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
+
+ Path policyFilePath = Files.createFile(
+ Paths.get(policyFileDir.toString(),
+ containerID + "-" + NMContainerPolicyUtils.POLICY_FILE),
+ POLICY_ATTR);
+ policyOutputStream = Files.newOutputStream(policyFilePath);
+
+ containerPolicies.put(containerID, policyFilePath);
+
+ NMContainerPolicyUtils.generatePolicyFile(
+ policyOutputStream, localDirs, resources, configuration);
+ NMContainerPolicyUtils.appendSecurityFlags(
+ commands, env, policyFilePath, sandboxMode);
+
+ } catch (Exception e) {
+ throw new ContainerExecutionException(e);
+ } finally {
+ IOUtils.cleanup(LOG, policyOutputStream);
+ }
+ }
+ }
+
+ @Override
+ public void launchContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ try {
+ super.launchContainer(ctx);
+ } finally {
+ deletePolicyFiles(ctx);
+ }
+ }
+
+ /**
+ * Determine if JVMSandboxLinuxContainerRuntime should be used. This is
+ * decided based on the value of
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX}
+ * @return true if Sandbox is requested, false otherwise
+ */
+ boolean isSandboxContainerRequested() {
+ return sandboxMode != SandboxMode.disabled;
+ }
+
+ /**
+ * Determine if the container should be whitelisted (i.e. exempt from the
+ * Java Security Manager).
+ * @param ctx The container runtime context for the requested container
+ * @param commands The list of run commands for the container
+ * @return boolean value denoting whether the container should be whitelisted.
+ * @throws ContainerExecutionException If container user can not be resolved
+ */
+ private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx,
+ List<String> commands) throws ContainerExecutionException {
+ String whitelistGroup = configuration.get(
+ YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
+ Groups groups = Groups.getUserToGroupsMappingService(configuration);
+ List<String> userGroups;
+ boolean isWhitelisted = false;
+
+ try {
+ userGroups = groups.getGroups(ctx.getExecutionAttribute(USER));
+ } catch (IOException e) {
+ throw new ContainerExecutionException("Container user does not exist");
+ }
+
+ if(whitelistGroup != null && userGroups.contains(whitelistGroup)) {
+ // If any command has security flag, whitelisting is disabled
+ for(String cmd : commands) {
+ if(cmd.contains(NMContainerPolicyUtils.SECURITY_FLAG)){
+ isWhitelisted = false;
+ break;
+ } else {
+ isWhitelisted = true;
+ }
+ }
+ }
+ return isWhitelisted;
+ }
+
+ /**
+ * Deletes policy files for container specified by parameter. Additionally
+ * this method will age off any stale policy files generated by
+ * {@link JavaSandboxLinuxContainerRuntime}
+ * @param ctx Container context for files to be deleted
+ * @throws ContainerExecutionException if unable to access or delete policy
+ * files or generated policy file directory
+ */
+ private void deletePolicyFiles(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ try {
+ Files.delete(containerPolicies.remove(
+ ctx.getExecutionAttribute(CONTAINER_ID_STR)));
+ } catch (IOException e) {
+ throw new ContainerExecutionException("Unable to delete policy file: "
+ + e);
+ }
+ }
+
+ /**
+ * Enumeration of the modes the JavaSandboxLinuxContainerRuntime can use.
+ * See {@link JavaSandboxLinuxContainerRuntime} for details on the
+ * behavior of each setting.
+ */
+ public enum SandboxMode {
+ enforcing("enforcing"),
+ permissive("permissive"),
+ disabled("disabled");
+
+ private final String mode;
+ SandboxMode(String mode){
+ this.mode = mode;
+ }
+
+ public static SandboxMode get(String mode) {
+
+ if(enforcing.mode.equals(mode)) {
+ return enforcing;
+ } else if(permissive.mode.equals(mode)) {
+ return permissive;
+ } else {
+ return disabled;
+ }
+ }
+
+ public String toString(){
+ return mode;
+ }
+ }
+
+ /**
+ * Static utility class defining String constants and static methods for the
+ * use of the {@link JavaSandboxLinuxContainerRuntime}.
+ */
+ static final class NMContainerPolicyUtils{
+
+ static final String POLICY_FILE = "java.policy";
+ static final String SECURITY_DEBUG = " -Djava.security.debug=all";
+ static final String SECURITY_FLAG = "-Djava.security.manager";
+ static final String POLICY_APPEND_FLAG = "-Djava.security.policy=";
+ static final String POLICY_FLAG = POLICY_APPEND_FLAG + "=";
+ static final String JAVA_CMD = "/bin/java ";
+ static final String JVM_SECURITY_CMD =
+ JAVA_CMD + SECURITY_FLAG + " " + POLICY_FLAG;
+
+ static final String STRIP_POLICY_FLAG = POLICY_APPEND_FLAG + "[^ ]+";
+ static final String CONTAINS_JAVA_CMD = "\\$" + JAVA_HOME + JAVA_CMD + ".*";
+ static final String CHAINED_COMMAND_REGEX =
+ "^.*(&&.+$)|(\\|\\|.+$).*$"; //Matches any occurrences of '||' or '&&'
+ static final String CLEAN_CMD_REGEX =
+ "(" + SECURITY_FLAG + ")|" +
+ "(" + STRIP_POLICY_FLAG + ")";
+
+ static final String FILE_PERMISSION_FORMAT = " permission "
+ + FilePermission.class.getCanonicalName()
+ + " \"%1$s" + SEPARATOR + "-\", \"%2$s\";%n";
+ static final String HADOOP_HOME_PERMISSION = "%ngrant codeBase \"file:"
+ + Paths.get(System.getProperty(SYSPROP_HADOOP_HOME_DIR))
+ + SEPARATOR + "-\" {%n" +
+ " permission " + AllPermission.class.getCanonicalName() + ";%n};%n";
+ static final Logger LOG =
+ Logger.getLogger(NMContainerPolicyUtils.class);
+
+ /**
+ * Write new policy file to policyOutStream which will include read access
+ * to localize resources. Optionally a default policyFilePath can be
+ * specified to append a custom policy implementation to the new policy file
+ * @param policyOutStream OutputStream pointing to java.policy file
+ * @param localDirs Container local directories
+ * @param resources List of local container resources
+ * @param conf YARN configuration
+ * @throws IOException - If policy file generation is unable to read the
+ * base policy file or if it is unable to create a new policy file.
+ */
+ static void generatePolicyFile(OutputStream policyOutStream,
+ List<String> localDirs, Map<org.apache.hadoop.fs.Path,
+ List<String>> resources, Configuration conf)
+ throws IOException {
+
+ String policyFilePath =
+ conf.get(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY);
+ String filePermissions =
+ conf.get(YarnConfiguration.YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS,
+ YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS);
+
+ Set<String> cacheDirs = new HashSet<>();
+ for(org.apache.hadoop.fs.Path path : resources.keySet()) {
+ cacheDirs.add(path.getParent().toString());
+ }
+
+ if(policyFilePath == null) {
+ IOUtils.copyBytes(
+ NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE),
+ policyOutStream, conf, false);
+ } else {
+ Files.copy(Paths.get(policyFilePath), policyOutStream);
+ policyOutStream.flush();
+ }
+
+ Formatter filePermissionFormat = new Formatter(policyOutStream,
+ StandardCharsets.UTF_8.name());
+ filePermissionFormat.format(HADOOP_HOME_PERMISSION);
+ filePermissionFormat.format("grant {%n");
+ for(String localDir : localDirs) {
+ filePermissionFormat.format(
+ FILE_PERMISSION_FORMAT, localDir, filePermissions);
+ }
+ for(String cacheDir : cacheDirs) {
+ filePermissionFormat.format(
+ FILE_PERMISSION_FORMAT, cacheDir, filePermissions);
+ }
+ filePermissionFormat.format("};%n");
+ filePermissionFormat.flush();
+ }
+
+ /**
+ * Modify command to enable the Java Security Manager and specify
+ * java.policy file. Will modify the passed commands to strip any
+ * existing java security configurations. Expects a java command to be the
+ * first and only executable provided in enforcing mode. In passive mode
+ * any commands with '||' or '&&' will not be modified.
+ * @param commands List of container commands
+ * @param env Container environment variables
+ * @param policyPath Path to the container specific policy file
+ * @param sandboxMode (enforcing, permissive, disabled) Determines
+ * whether non-java containers will be launched
+ * @throws ContainerExecutionException - Exception thrown if
+ * JVM Sandbox enabled in 'enforcing' mode and a non-java command is
+ * provided in the list of commands
+ */
+ static void appendSecurityFlags(List<String> commands,
+ Map<String, String> env, Path policyPath, SandboxMode sandboxMode)
+ throws ContainerExecutionException {
+
+ for(int i = 0; i < commands.size(); i++){
+ String command = commands.get(i);
+ if(validateJavaHome(env.get(JAVA_HOME.name()))
+ && command.matches(CONTAINS_JAVA_CMD)
+ && !command.matches(CHAINED_COMMAND_REGEX)){
+ command = command.replaceAll(CLEAN_CMD_REGEX, "");
+ String securityString = JVM_SECURITY_CMD + policyPath + " ";
+ if(LOG.isDebugEnabled()) {
+ securityString += SECURITY_DEBUG;
+ }
+ commands.set(i, command.replaceFirst(JAVA_CMD, securityString));
+ } else if (sandboxMode == SandboxMode.enforcing){
+ throw new ContainerExecutionException(
+ "Only JVM containers permitted in YARN sandbox mode (enforcing). "
+ + "The following command can not be executed securely: " + command);
+ } else if (sandboxMode == SandboxMode.permissive){
+ LOG.warn("The container will run without the java security manager"
+ + " due to an unsupported container command. The command"
+ + " will be permitted to run in Sandbox permissive mode: "
+ + command);
+ }
+ }
+ }
+
+ private static boolean validateJavaHome(String containerJavaHome)
+ throws ContainerExecutionException{
+ if (System.getenv(JAVA_HOME.name()) == null){
+ throw new ContainerExecutionException(
+ "JAVA_HOME is not set for NodeManager");
+ }
+ if (containerJavaHome == null) {
+ throw new ContainerExecutionException(
+ "JAVA_HOME is not set for container");
+ }
+ return System.getenv(JAVA_HOME.name()).equals(containerJavaHome);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
index 0c1ec3e..2e632fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
@@ -67,6 +67,8 @@ public final class LinuxContainerRuntimeConstants {
String.class, "resources_options");
public static final Attribute<String> TC_COMMAND_FILE = Attribute.attribute(
String.class, "tc_command_file");
+ public static final Attribute<List> CONTAINER_RUN_CMDS = Attribute.attribute(
+ List.class, "container_run_cmds");
public static final Attribute<String> CGROUP_RELATIVE_PATH = Attribute
.attribute(String.class, "cgroup_relative_path");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerPrepareContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerPrepareContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerPrepareContext.java
new file mode 100644
index 0000000..f711da2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerPrepareContext.java
@@ -0,0 +1,119 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.executor;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Encapsulates information required for preparing containers.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class ContainerPrepareContext {
+ private final Container container;
+ private final Map<Path, List<String>> localizedResources;
+ private final String user;
+ private final List<String> containerLocalDirs;
+ private final List<String> commands;
+
+ /**
+ * Builder for ContainerPrepareContext.
+ */
+ public static final class Builder {
+ private Container container;
+ private Map<Path, List<String>> localizedResources;
+ private String user;
+ private List<String> containerLocalDirs;
+ private List<String> commands;
+
+ public Builder() {
+ }
+
+ public ContainerPrepareContext.Builder setContainer(Container container) {
+ this.container = container;
+ return this;
+ }
+
+ public ContainerPrepareContext.Builder setLocalizedResources(Map<Path,
+ List<String>> localizedResources) {
+ this.localizedResources = localizedResources;
+ return this;
+ }
+
+ public ContainerPrepareContext.Builder setUser(String user) {
+ this.user = user;
+ return this;
+ }
+ public ContainerPrepareContext.Builder setContainerLocalDirs(
+ List<String> containerLocalDirs) {
+ this.containerLocalDirs = containerLocalDirs;
+ return this;
+ }
+
+ public ContainerPrepareContext build() {
+ return new ContainerPrepareContext(this);
+ }
+
+ public ContainerPrepareContext.Builder setCommands(List<String> commands) {
+ this.commands = commands;
+ return this;
+ }
+ }
+
+ private ContainerPrepareContext(ContainerPrepareContext.Builder builder) {
+ this.container = builder.container;
+ this.localizedResources = builder.localizedResources;
+ this.user = builder.user;
+ this.containerLocalDirs = builder.containerLocalDirs;
+ this.commands = builder.commands;
+ }
+
+ public Container getContainer() {
+ return this.container;
+ }
+
+ public Map<Path, List<String>> getLocalizedResources() {
+ if (this.localizedResources != null) {
+ return Collections.unmodifiableMap(this.localizedResources);
+ } else {
+ return null;
+ }
+ }
+
+ public String getUser() {
+ return this.user;
+ }
+
+ public List<String> getContainerLocalDirs() {
+ return Collections.unmodifiableList(this.containerLocalDirs);
+ }
+
+ public List<String> getCommands(){
+ return this.commands;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/java.policy
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/java.policy b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/java.policy
new file mode 100644
index 0000000..d9320c2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/java.policy
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Standard extensions get all permissions by default
+grant codeBase "file:${{java.ext.dirs}}/*" {
+ permission java.security.AllPermission;
+};
+
+// default permissions granted to all domains
+grant {
+ permission java.lang.RuntimePermission "accessDeclaredMembers";
+
+ permission java.util.PropertyPermission "java.version", "read";
+ permission java.util.PropertyPermission "java.vendor", "read";
+ permission java.util.PropertyPermission "java.vendor.url", "read";
+ permission java.util.PropertyPermission "java.class.version", "read";
+ permission java.util.PropertyPermission "os.name", "read";
+ permission java.util.PropertyPermission "os.version", "read";
+ permission java.util.PropertyPermission "os.arch", "read";
+ permission java.util.PropertyPermission "file.separator", "read";
+ permission java.util.PropertyPermission "path.separator", "read";
+ permission java.util.PropertyPermission "line.separator", "read";
+
+ permission java.util.PropertyPermission "java.specification.version", "read";
+ permission java.util.PropertyPermission "java.specification.vendor", "read";
+ permission java.util.PropertyPermission "java.specification.name", "read";
+
+ permission java.util.PropertyPermission "java.vm.specification.version", "read";
+ permission java.util.PropertyPermission "java.vm.specification.vendor", "read";
+ permission java.util.PropertyPermission "java.vm.specification.name", "read";
+ permission java.util.PropertyPermission "java.vm.version", "read";
+ permission java.util.PropertyPermission "java.vm.vendor", "read";
+ permission java.util.PropertyPermission "java.vm.name", "read";
+
+ //additional hadoop permissions
+ permission java.util.PropertyPermission "awt.Toolkit", "read";
+ permission java.util.PropertyPermission "file.encoding", "read";
+ permission java.util.PropertyPermission "file.encoding.pkg", "read";
+ permission java.util.PropertyPermission "hadoop.metrics.log.level", "read";
+ permission java.util.PropertyPermission "hadoop.root.logger", "read";
+ permission java.util.PropertyPermission "java.awt.graphicsenv" ,"read";
+ permission java.util.PropertyPermission "java.awt.printerjob", "read";
+ permission java.util.PropertyPermission "java.class.path", "read";
+ permission java.util.PropertyPermission "yarn.app.container.log.dir", "read";
+ permission java.util.PropertyPermission "yarn.app.container.log.filesize", "read";
+ permission java.lang.RuntimePermission "loadLibrary.gplcompression";
+ permission javax.security.auth.AuthPermission "getSubject";
+};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
new file mode 100644
index 0000000..e482c8d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.linux.runtime;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilePermission;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CHAINED_COMMAND_REGEX;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FLAG;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.SECURITY_FLAG;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.POLICY_FILE_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPID;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_WORK_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.FILECACHE_DIRS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCALIZED_RESOURCES;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCAL_DIRS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOG_DIRS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.RUN_AS_USER;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER_LOCAL_DIRS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test policy file generation and policy enforcement for the
+ * {@link JavaSandboxLinuxContainerRuntime}.
+ */
+public class TestJavaSandboxLinuxContainerRuntime {
+
+ private static final String HADOOP_HOME = "hadoop.home.dir";
+ private static String hadoopHomeDir = System.getProperty(HADOOP_HOME);
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ private static File grantFile, denyFile, policyFile,
+ grantDir, denyDir, containerDir;
+ private static java.nio.file.Path policyFilePath;
+ private static SecurityManager securityManager;
+ private Map<Path, List<String>> resources;
+ private Map<String, String> env;
+ private List<String> whitelistGroup;
+
+ private PrivilegedOperationExecutor mockExecutor;
+ private JavaSandboxLinuxContainerRuntime runtime;
+ private ContainerRuntimeContext.Builder runtimeContextBuilder;
+ private Configuration conf;
+
+ private final static String NORMAL_USER = System.getProperty("user.name");
+ private final static String NORMAL_GROUP = "normalGroup";
+ private final static String WHITELIST_USER = "picard";
+ private final static String WHITELIST_GROUP = "captains";
+ private final static String CONTAINER_ID = "container_1234567890";
+ private final static String APPLICATION_ID = "application_1234567890";
+
+ @Before
+ public void setup() throws Exception {
+
+ File baseTestDirectory = new File(System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir", "target")),
+ TestJavaSandboxLinuxContainerRuntime.class.getName());
+
+ whitelistGroup = new ArrayList<>();
+ whitelistGroup.add(WHITELIST_GROUP);
+
+ conf = new Configuration();
+ conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
+ WHITELIST_USER + "=" + WHITELIST_GROUP + ";"
+ + NORMAL_USER + "=" + NORMAL_GROUP + ";");
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+ WHITELIST_GROUP);
+ conf.set("hadoop.tmp.dir", baseTestDirectory.getAbsolutePath());
+
+ Files.deleteIfExists(Paths.get(baseTestDirectory.getAbsolutePath(),
+ POLICY_FILE_DIR, CONTAINER_ID + "-" + POLICY_FILE));
+
+ mockExecutor = mock(PrivilegedOperationExecutor.class);
+ runtime = new JavaSandboxLinuxContainerRuntime(mockExecutor);
+ runtime.initialize(conf);
+
+ resources = new HashMap<>();
+ grantDir = new File(baseTestDirectory, "grantDir");
+ denyDir = new File(baseTestDirectory, "denyDir");
+ containerDir = new File(baseTestDirectory,
+ APPLICATION_ID + Path.SEPARATOR + CONTAINER_ID);
+ grantDir.mkdirs();
+ denyDir.mkdirs();
+ containerDir.mkdirs();
+
+ grantFile = File.createTempFile("grantFile", "tmp", grantDir);
+ denyFile = File.createTempFile("denyFile", "tmp", denyDir);
+
+ List<String> symLinks = new ArrayList<>();
+ symLinks.add(grantFile.getName());
+ resources.put(new Path(grantFile.getCanonicalPath()), symLinks);
+
+ env = new HashMap();
+ env.put(JAVA_HOME.name(), System.getenv(JAVA_HOME.name()));
+
+ policyFile = File.createTempFile("java", "policy", containerDir);
+ policyFilePath = Paths.get(policyFile.getAbsolutePath());
+
+ runtimeContextBuilder = createRuntimeContext();
+
+ if (hadoopHomeDir == null) {
+ System.setProperty(HADOOP_HOME, policyFile.getParent());
+ }
+
+ OutputStream outStream = new FileOutputStream(policyFile);
+ JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
+ .generatePolicyFile(outStream, symLinks, resources, conf);
+ outStream.close();
+
+ System.setProperty("java.security.policy", policyFile.getCanonicalPath());
+ securityManager = new SecurityManager();
+
+ }
+
+ public ContainerRuntimeContext.Builder createRuntimeContext(){
+
+ Container container = mock(Container.class);
+ ContainerLaunchContext ctx = mock(ContainerLaunchContext.class);
+
+ when(container.getLaunchContext()).thenReturn(ctx);
+ when(ctx.getEnvironment()).thenReturn(env);
+
+ ContainerRuntimeContext.Builder builder =
+ new ContainerRuntimeContext.Builder(container);
+
+ List<String> localDirs = new ArrayList<>();
+
+ builder.setExecutionAttribute(LOCALIZED_RESOURCES, resources)
+ .setExecutionAttribute(RUN_AS_USER, NORMAL_USER)
+ .setExecutionAttribute(CONTAINER_ID_STR, CONTAINER_ID)
+ .setExecutionAttribute(APPID, APPLICATION_ID)
+ .setExecutionAttribute(CONTAINER_WORK_DIR,
+ new Path(containerDir.toString()))
+ .setExecutionAttribute(LOCAL_DIRS, localDirs)
+ .setExecutionAttribute(LOG_DIRS, localDirs)
+ .setExecutionAttribute(FILECACHE_DIRS, localDirs)
+ .setExecutionAttribute(USER_LOCAL_DIRS, localDirs)
+ .setExecutionAttribute(CONTAINER_LOCAL_DIRS, localDirs)
+ .setExecutionAttribute(CONTAINER_RUN_CMDS, localDirs);
+
+ return builder;
+ }
+
+ @Test
+ public void testGrant() throws Exception {
+ FilePermission grantPermission =
+ new FilePermission(grantFile.getAbsolutePath(), "read");
+ securityManager.checkPermission(grantPermission);
+ }
+
+ @Test
+ public void testDeny() throws Exception {
+ FilePermission denyPermission =
+ new FilePermission(denyFile.getAbsolutePath(), "read");
+ exception.expect(java.security.AccessControlException.class);
+ securityManager.checkPermission(denyPermission);
+ }
+
+ @Test
+ public void testEnforcingMode() throws ContainerExecutionException {
+ String[] nonJavaCommands = {
+ "bash malicious_script.sh",
+ "python malicious_script.py"
+ };
+
+ List<String> commands = Arrays.asList(nonJavaCommands);
+ exception.expect(ContainerExecutionException.class);
+ JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
+ .appendSecurityFlags(commands, env, policyFilePath,
+ JavaSandboxLinuxContainerRuntime.SandboxMode.enforcing);
+
+ }
+
+ @Test
+ public void testPermissiveMode() throws ContainerExecutionException {
+ String[] nonJavaCommands = {
+ "bash non-java-script.sh",
+ "python non-java-script.py"
+ };
+
+ List<String> commands = Arrays.asList(nonJavaCommands);
+ JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
+ .appendSecurityFlags(commands, env, policyFilePath,
+ JavaSandboxLinuxContainerRuntime.SandboxMode.permissive);
+
+ }
+
+ @Test
+ public void testDisabledSandboxWithWhitelist()
+ throws ContainerExecutionException {
+
+ String[] inputCommand = {
+ "java jar MyJob.jar"
+ };
+ List<String> commands = Arrays.asList(inputCommand);
+
+ runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
+ runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
+ runtime.prepareContainer(runtimeContextBuilder.build());
+
+ Assert.assertTrue("Command should not be modified when user is " +
+ "member of whitelisted group",
+ inputCommand[0].equals(commands.get(0)));
+
+ }
+
+ @Test
+ public void testEnabledSandboxWithWhitelist()
+ throws ContainerExecutionException{
+ String[] inputCommand = {
+ "$JAVA_HOME/bin/java jar -Djava.security.manager MyJob.jar"
+ };
+ List<String> commands = Arrays.asList(inputCommand);
+
+ runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
+ runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
+ runtime.prepareContainer(runtimeContextBuilder.build());
+
+ Assert.assertTrue("Command should be modified to include " +
+ "policy file in whitelisted Sandbox mode",
+ commands.get(0).contains(SECURITY_FLAG)
+ && commands.get(0).contains(POLICY_FLAG));
+ }
+
+ @Test
+ public void testDeniedWhitelistGroup() throws ContainerExecutionException {
+
+ String[] inputCommand = {
+ "$JAVA_HOME/bin/java jar MyJob.jar"
+ };
+ List<String> commands = Arrays.asList(inputCommand);
+
+ runtimeContextBuilder.setExecutionAttribute(USER, NORMAL_USER);
+ runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
+ runtime.prepareContainer(runtimeContextBuilder.build());
+
+ Assert.assertTrue("Java security manager must be enabled for "
+ + "unauthorized users",
+ commands.get(0).contains(SECURITY_FLAG));
+ }
+
+ @Test
+ public void testChainedCmdRegex(){
+ Assert.assertTrue("cmd1 && cmd2 || cmd3".matches(CHAINED_COMMAND_REGEX));
+ Assert.assertFalse("cmd1 &> logfile".matches(CHAINED_COMMAND_REGEX));
+ }
+
+ @Test
+ public void testContainsJavaRegex(){
+ String[] javaCmds = {
+ "$JAVA_HOME/bin/java -cp App.jar AppClass",
+ "$JAVA_HOME/bin/java -jar App.jar AppClass &> logfile"
+ };
+ String[] nonJavaCmds = {
+ "$JAVA_HOME/bin/jajavava -cp App.jar AppClass",
+ "/nm/app/container/usercache/badjava -cp Bad.jar ChaosClass"
+ };
+ for(String javaCmd : javaCmds) {
+ Assert.assertTrue(javaCmd.matches(CONTAINS_JAVA_CMD));
+ }
+ for(String nonJavaCmd : nonJavaCmds) {
+ Assert.assertFalse(nonJavaCmd.matches(CONTAINS_JAVA_CMD));
+ }
+ }
+
+ @Test
+ public void testCleanCmdRegex(){
+ String[] securityManagerCmds = {
+ "/usr/bin/java -Djava.security.manager -cp $CLASSPATH $MainClass",
+ "-Djava.security.manager -Djava.security.policy==testpolicy keepThis"
+ };
+ String[] cleanedCmdsResult = {
+ "/usr/bin/java -cp $CLASSPATH $MainClass",
+ "keepThis"
+ };
+ for(int i = 0; i < securityManagerCmds.length; i++){
+ Assert.assertEquals(
+ securityManagerCmds[i].replaceAll(CLEAN_CMD_REGEX, "").trim(),
+ cleanedCmdsResult[i]);
+ }
+ }
+
+ @Test
+ public void testAppendSecurityFlags() throws ContainerExecutionException {
+ String securityString = "-Djava.security.manager -Djava.security.policy=="
+ + policyFile.getAbsolutePath();
+ String[] badCommands = {
+ "$JAVA_HOME/bin/java -Djava.security.manager "
+ + "-Djava.security.policy=/home/user/java.policy",
+ "$JAVA_HOME/bin/java -cp MyApp.jar MrAppMaster"
+ };
+ String[] cleanCommands = {
+ "$JAVA_HOME/bin/java " + securityString,
+ "$JAVA_HOME/bin/java " + securityString
+ + " -cp MyApp.jar MrAppMaster"
+ };
+
+ List<String> commands = Arrays.asList(badCommands);
+ JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
+ .appendSecurityFlags(commands, env, policyFilePath,
+ JavaSandboxLinuxContainerRuntime.SandboxMode.enforcing);
+
+ for(int i = 0; i < commands.size(); i++) {
+ Assert.assertTrue(commands.get(i).trim().equals(cleanCommands[i].trim()));
+ }
+ }
+
+ @After
+ public void cleanup(){
+ System.setSecurityManager(null);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: YARN-6189: Improve application
status log message when RM restarted when app is in NEW state. Contributed by
Junping Du
Posted by jh...@apache.org.
YARN-6189: Improve application status log message when RM restarted when
app is in NEW state. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0bb867c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0bb867c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0bb867c
Branch: refs/heads/YARN-5734
Commit: e0bb867c3fa638c9f689ee0b044b400481cf02b5
Parents: eac6b4c
Author: Xuan <xg...@apache.org>
Authored: Tue Feb 28 11:04:56 2017 -0800
Committer: Xuan <xg...@apache.org>
Committed: Tue Feb 28 11:04:56 2017 -0800
----------------------------------------------------------------------
.../yarn/server/resourcemanager/ClientRMService.java | 15 ++++++++++-----
.../server/resourcemanager/TestClientRMService.java | 3 ++-
2 files changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0bb867c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 48bccfb..929a9e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -359,7 +359,8 @@ public class ClientRMService extends AbstractService implements
// If the RM doesn't have the application, throw
// ApplicationNotFoundException and let client to handle.
throw new ApplicationNotFoundException("Application with id '"
- + applicationId + "' doesn't exist in RM.");
+ + applicationId + "' doesn't exist in RM. Please check "
+ + "that the job submission was successful.");
}
boolean allowAccess = checkAccess(callerUGI, application.getUser(),
@@ -393,7 +394,8 @@ public class ClientRMService extends AbstractService implements
// ApplicationNotFoundException and let client to handle.
throw new ApplicationNotFoundException("Application with id '"
+ request.getApplicationAttemptId().getApplicationId()
- + "' doesn't exist in RM.");
+ + "' doesn't exist in RM. Please check that the job "
+ + "submission was successful.");
}
boolean allowAccess = checkAccess(callerUGI, application.getUser(),
@@ -432,7 +434,8 @@ public class ClientRMService extends AbstractService implements
// If the RM doesn't have the application, throw
// ApplicationNotFoundException and let client to handle.
throw new ApplicationNotFoundException("Application with id '" + appId
- + "' doesn't exist in RM.");
+ + "' doesn't exist in RM. Please check that the job submission "
+ + "was successful.");
}
boolean allowAccess = checkAccess(callerUGI, application.getUser(),
ApplicationAccessType.VIEW_APP, application);
@@ -480,7 +483,8 @@ public class ClientRMService extends AbstractService implements
// If the RM doesn't have the application, throw
// ApplicationNotFoundException and let client to handle.
throw new ApplicationNotFoundException("Application with id '" + appId
- + "' doesn't exist in RM.");
+ + "' doesn't exist in RM. Please check that the job submission "
+ + "was successful.");
}
boolean allowAccess = checkAccess(callerUGI, application.getUser(),
ApplicationAccessType.VIEW_APP, application);
@@ -530,7 +534,8 @@ public class ClientRMService extends AbstractService implements
// If the RM doesn't have the application, throw
// ApplicationNotFoundException and let client to handle.
throw new ApplicationNotFoundException("Application with id '" + appId
- + "' doesn't exist in RM.");
+ + "' doesn't exist in RM. Please check that the job submission "
+ + "was successful.");
}
boolean allowAccess = checkAccess(callerUGI, application.getUser(),
ApplicationAccessType.VIEW_APP, application);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0bb867c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index d5f9acc..9076ac3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -283,7 +283,8 @@ public class TestClientRMService {
} catch (ApplicationNotFoundException ex) {
Assert.assertEquals(ex.getMessage(),
"Application with id '" + request.getApplicationId()
- + "' doesn't exist in RM.");
+ + "' doesn't exist in RM. Please check that the "
+ + "job submission was successful.");
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-5951. Changes to allow
CapacityScheduler to use configuration store
Posted by jh...@apache.org.
YARN-5951. Changes to allow CapacityScheduler to use configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5d40ae0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5d40ae0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5d40ae0
Branch: refs/heads/YARN-5734
Commit: d5d40ae00db226c78316ff122d17a3ad7ae4189a
Parents: 8f4817f
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Mon Jan 30 19:03:48 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Thu Mar 2 18:47:45 2017 -0800
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 36 +++++------
.../CapacitySchedulerConfiguration.java | 10 +++
.../capacity/conf/CSConfigurationProvider.java | 46 ++++++++++++++
.../conf/FileBasedCSConfigurationProvider.java | 67 ++++++++++++++++++++
.../scheduler/capacity/conf/package-info.java | 29 +++++++++
.../capacity/TestCapacityScheduler.java | 4 +-
6 files changed, 170 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d40ae0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 20ea607..9cec3e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
@@ -104,6 +103,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation;
@@ -162,6 +163,7 @@ public class CapacityScheduler extends
private int offswitchPerHeartbeatLimit;
+ private CSConfigurationProvider csConfProvider;
@Override
public void setConf(Configuration conf) {
@@ -285,7 +287,18 @@ public class CapacityScheduler extends
IOException {
try {
writeLock.lock();
- this.conf = loadCapacitySchedulerConfiguration(configuration);
+ String confProviderStr = configuration.get(
+ CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
+ if (confProviderStr.equals(
+ CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) {
+ this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext);
+ } else {
+ throw new IOException("Invalid CS configuration provider: " +
+ confProviderStr);
+ }
+ this.csConfProvider.init(configuration);
+ this.conf = this.csConfProvider.loadConfiguration(configuration);
validateConf(this.conf);
this.minimumAllocation = this.conf.getMinimumAllocation();
initMaximumResourceCapability(this.conf.getMaximumAllocation());
@@ -392,7 +405,7 @@ public class CapacityScheduler extends
writeLock.lock();
Configuration configuration = new Configuration(newConf);
CapacitySchedulerConfiguration oldConf = this.conf;
- this.conf = loadCapacitySchedulerConfiguration(configuration);
+ this.conf = csConfProvider.loadConfiguration(configuration);
validateConf(this.conf);
try {
LOG.info("Re-initializing queues...");
@@ -1754,23 +1767,6 @@ public class CapacityScheduler extends
return true;
}
- private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration(
- Configuration configuration) throws IOException {
- try {
- InputStream CSInputStream =
- this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(configuration,
- YarnConfiguration.CS_CONFIGURATION_FILE);
- if (CSInputStream != null) {
- configuration.addResource(CSInputStream);
- return new CapacitySchedulerConfiguration(configuration, false);
- }
- return new CapacitySchedulerConfiguration(configuration, true);
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
private String getDefaultReservationQueueName(String planQueueName) {
return planQueueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d40ae0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 43ec390..d11b5ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -285,6 +285,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final boolean DEFAULT_LAZY_PREEMPTION_ENABLED = false;
+ @Private
+ public static final String CS_CONF_PROVIDER = PREFIX
+ + "configuration.provider";
+
+ @Private
+ public static final String FILE_CS_CONF_PROVIDER = "file";
+
+ @Private
+ public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
+
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
public CapacitySchedulerConfiguration() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d40ae0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
new file mode 100644
index 0000000..c9984ac
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import java.io.IOException;
+
+/**
+ * Configuration provider for {@link CapacityScheduler}.
+ */
+public interface CSConfigurationProvider {
+
+ /**
+ * Initialize the configuration provider with given conf.
+ * @param conf configuration to initialize with
+ */
+ void init(Configuration conf);
+
+ /**
+ * Loads capacity scheduler configuration object.
+ * @param conf initial bootstrap configuration
+ * @return CS configuration
+ * @throws IOException if fail to retrieve configuration
+ */
+ CapacitySchedulerConfiguration loadConfiguration(Configuration conf)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d40ae0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
new file mode 100644
index 0000000..51c64fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * {@link CapacityScheduler} configuration provider based on local
+ * {@code capacity-scheduler.xml} file.
+ */
+public class FileBasedCSConfigurationProvider implements
+ CSConfigurationProvider {
+
+ private RMContext rmContext;
+
+ /**
+ * Construct file based CS configuration provider with given context.
+ * @param rmContext the RM context
+ */
+ public FileBasedCSConfigurationProvider(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void init(Configuration conf) {}
+
+ @Override
+ public CapacitySchedulerConfiguration loadConfiguration(Configuration conf)
+ throws IOException {
+ try {
+ InputStream csInputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.CS_CONFIGURATION_FILE);
+ if (csInputStream != null) {
+ conf.addResource(csInputStream);
+ return new CapacitySchedulerConfiguration(conf, false);
+ }
+ return new CapacitySchedulerConfiguration(conf, true);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d40ae0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
new file mode 100644
index 0000000..08d0522
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package
+ * org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf
+ * contains classes related to capacity scheduler configuration management.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d40ae0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 2b60ecf..9a99cab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -223,13 +223,13 @@ public class TestCapacityScheduler {
@Test (timeout = 30000)
public void testConfValidation() throws Exception {
- ResourceScheduler scheduler = new CapacityScheduler();
+ CapacityScheduler scheduler = new CapacityScheduler();
scheduler.setRMContext(resourceManager.getRMContext());
Configuration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
try {
- scheduler.reinitialize(conf, mockContext);
+ scheduler.init(conf);
fail("Exception is expected because the min memory allocation is" +
" larger than the max memory allocation.");
} catch (YarnRuntimeException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HADOOP-13817. Add a finite shell
command timeout to ShellBasedUnixGroupsMapping. (harsh)
Posted by jh...@apache.org.
HADOOP-13817. Add a finite shell command timeout to ShellBasedUnixGroupsMapping. (harsh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8694deb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8694deb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8694deb
Branch: refs/heads/YARN-5734
Commit: e8694deb6ad180449f8ce6c1c8b4f84873c0587a
Parents: 50decd3
Author: Harsh J <ha...@cloudera.com>
Authored: Mon Nov 14 15:59:58 2016 +0530
Committer: Harsh J <ha...@cloudera.com>
Committed: Fri Feb 24 21:34:00 2017 +0530
----------------------------------------------------------------------
.../fs/CommonConfigurationKeysPublic.java | 15 +++
.../security/ShellBasedUnixGroupsMapping.java | 114 +++++++++++++---
.../main/java/org/apache/hadoop/util/Shell.java | 19 ++-
.../src/main/resources/core-default.xml | 13 ++
.../hadoop/security/TestGroupsCaching.java | 19 +--
.../TestShellBasedUnixGroupsMapping.java | 135 ++++++++++++++++++-
6 files changed, 277 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index f23dd51..e1feda1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -517,6 +517,21 @@ public class CommonConfigurationKeysPublic {
* <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
* core-default.xml</a>
*/
+ public static final String HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS =
+ "hadoop.security.groups.shell.command.timeout";
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final long
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT =
+ 0L;
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_AUTHENTICATION =
"hadoop.security.authentication";
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
index 9b80be9..4146e7b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
@@ -18,17 +18,25 @@
package org.apache.hadoop.security;
import java.io.IOException;
+import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.StringTokenizer;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A simple shell-based implementation of {@link GroupMappingServiceProvider}
@@ -37,11 +45,28 @@ import org.apache.hadoop.util.Shell.ShellCommandExecutor;
*/
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Evolving
-public class ShellBasedUnixGroupsMapping
+public class ShellBasedUnixGroupsMapping extends Configured
implements GroupMappingServiceProvider {
-
- private static final Log LOG =
- LogFactory.getLog(ShellBasedUnixGroupsMapping.class);
+
+ @VisibleForTesting
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ShellBasedUnixGroupsMapping.class);
+
+ private long timeout = 0L;
+ private static final List<String> EMPTY_GROUPS = new LinkedList<>();
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ if (conf != null) {
+ timeout = conf.getTimeDuration(
+ CommonConfigurationKeys.
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS,
+ CommonConfigurationKeys.
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT,
+ TimeUnit.SECONDS);
+ }
+ }
@SuppressWarnings("serial")
private static class PartialGroupNameException extends IOException {
@@ -98,7 +123,17 @@ public class ShellBasedUnixGroupsMapping
*/
protected ShellCommandExecutor createGroupExecutor(String userName) {
return new ShellCommandExecutor(
- Shell.getGroupsForUserCommand(userName), null, null, 0L);
+ getGroupsForUserCommand(userName), null, null, timeout);
+ }
+
+ /**
+ * Returns just the shell command to be used to fetch a user's groups list.
+ * This is mainly separate to make some tests easier.
+ * @param userName The username that needs to be passed into the command built
+ * @return An appropriate shell command with arguments
+ */
+ protected String[] getGroupsForUserCommand(String userName) {
+ return Shell.getGroupsForUserCommand(userName);
}
/**
@@ -109,7 +144,17 @@ public class ShellBasedUnixGroupsMapping
*/
protected ShellCommandExecutor createGroupIDExecutor(String userName) {
return new ShellCommandExecutor(
- Shell.getGroupsIDForUserCommand(userName), null, null, 0L);
+ getGroupsIDForUserCommand(userName), null, null, timeout);
+ }
+
+ /**
+ * Returns just the shell command to be used to fetch a user's group IDs list.
+ * This is mainly separate to make some tests easier.
+ * @param userName The username that needs to be passed into the command built
+ * @return An appropriate shell command with arguments
+ */
+ protected String[] getGroupsIDForUserCommand(String userName) {
+ return Shell.getGroupsIDForUserCommand(userName);
}
/**
@@ -133,8 +178,26 @@ public class ShellBasedUnixGroupsMapping
groups = resolvePartialGroupNames(user, e.getMessage(),
executor.getOutput());
} catch (PartialGroupNameException pge) {
- LOG.warn("unable to return groups for user " + user, pge);
- return new LinkedList<>();
+ LOG.warn("unable to return groups for user {}", user, pge);
+ return EMPTY_GROUPS;
+ }
+ } catch (IOException ioe) {
+ // If its a shell executor timeout, indicate so in the message
+ // but treat the result as empty instead of throwing it up,
+ // similar to how partial resolution failures are handled above
+ if (executor.isTimedOut()) {
+ LOG.warn(
+ "Unable to return groups for user '{}' as shell group lookup " +
+ "command '{}' ran longer than the configured timeout limit of " +
+ "{} seconds.",
+ user,
+ Joiner.on(' ').join(executor.getExecString()),
+ timeout
+ );
+ return EMPTY_GROUPS;
+ } else {
+ // If its not an executor timeout, we should let the caller handle it
+ throw ioe;
}
}
@@ -196,7 +259,7 @@ public class ShellBasedUnixGroupsMapping
* @param errMessage error message from the shell command
* @param groupNames the incomplete list of group names
* @return a list of resolved group names
- * @throws PartialGroupNameException
+ * @throws PartialGroupNameException if the resolution fails or times out
*/
private List<String> resolvePartialGroupNames(String userName,
String errMessage, String groupNames) throws PartialGroupNameException {
@@ -212,21 +275,29 @@ public class ShellBasedUnixGroupsMapping
throw new PartialGroupNameException("The user name '" + userName
+ "' is not found. " + errMessage);
} else {
- LOG.warn("Some group names for '" + userName + "' are not resolvable. "
- + errMessage);
+ LOG.warn("Some group names for '{}' are not resolvable. {}",
+ userName, errMessage);
// attempt to partially resolve group names
+ ShellCommandExecutor partialResolver = createGroupIDExecutor(userName);
try {
- ShellCommandExecutor exec2 = createGroupIDExecutor(userName);
- exec2.execute();
- return parsePartialGroupNames(groupNames, exec2.getOutput());
+ partialResolver.execute();
+ return parsePartialGroupNames(
+ groupNames, partialResolver.getOutput());
} catch (ExitCodeException ece) {
// If exception is thrown trying to get group id list,
// something is terribly wrong, so give up.
- throw new PartialGroupNameException("failed to get group id list for " +
- "user '" + userName + "'", ece);
+ throw new PartialGroupNameException(
+ "failed to get group id list for user '" + userName + "'", ece);
} catch (IOException ioe) {
- throw new PartialGroupNameException("can't execute the shell command to"
- + " get the list of group id for user '" + userName + "'", ioe);
+ String message =
+ "Can't execute the shell command to " +
+ "get the list of group id for user '" + userName + "'";
+ if (partialResolver.isTimedOut()) {
+ message +=
+ " because of the command taking longer than " +
+ "the configured timeout: " + timeout + " seconds";
+ }
+ throw new PartialGroupNameException(message, ioe);
}
}
}
@@ -237,7 +308,8 @@ public class ShellBasedUnixGroupsMapping
* @param groupNames a string representing the user's group names
* @return a linked list of group names
*/
- private List<String> resolveFullGroupNames(String groupNames) {
+ @VisibleForTesting
+ protected List<String> resolveFullGroupNames(String groupNames) {
StringTokenizer tokenizer =
new StringTokenizer(groupNames, Shell.TOKEN_SEPARATOR_REGEX);
List<String> groups = new LinkedList<String>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index ca59b0e..bfb8183 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -955,7 +955,15 @@ public abstract class Shell {
line = errReader.readLine();
}
} catch(IOException ioe) {
- LOG.warn("Error reading the error stream", ioe);
+ // Its normal to observe a "Stream closed" I/O error on
+ // command timeouts destroying the underlying process
+ // so only log a WARN if the command didn't time out
+ if (!isTimedOut()) {
+ LOG.warn("Error reading the error stream", ioe);
+ } else {
+ LOG.debug("Error reading the error stream due to shell "
+ + "command timeout", ioe);
+ }
}
}
};
@@ -1181,6 +1189,15 @@ public abstract class Shell {
}
/**
+ * Returns the timeout value set for the executor's sub-commands.
+ * @return The timeout value in seconds
+ */
+ @VisibleForTesting
+ public long getTimeoutInterval() {
+ return timeOutInterval;
+ }
+
+ /**
* Execute the shell command.
* @throws IOException if the command fails, or if the command is
* not well constructed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 1941bec..d8136ee 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -188,6 +188,19 @@
</property>
<property>
+ <name>hadoop.security.groups.shell.command.timeout</name>
+ <value>0s</value>
+ <description>
+ Used by the ShellBasedUnixGroupsMapping class, this property controls how
+ long to wait for the underlying shell command that is run to fetch groups.
+ Expressed in seconds (e.g. 10s, 1m, etc.), if the running command takes
+ longer than the value configured, the command is aborted and the groups
+ resolver would return a result of no groups found. A value of 0s (default)
+ would mean an infinite wait (i.e. wait until the command exits on its own).
+ </description>
+</property>
+
+<property>
<name>hadoop.security.group.mapping.ldap.connection.timeout.ms</name>
<value>60000</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
index 58c2d1a..930c45e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
@@ -50,7 +50,7 @@ import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
public class TestGroupsCaching {
- public static final Log LOG = LogFactory.getLog(TestGroupsCaching.class);
+ public static final Log TESTLOG = LogFactory.getLog(TestGroupsCaching.class);
private static String[] myGroups = {"grp1", "grp2"};
private Configuration conf;
@@ -76,7 +76,7 @@ public class TestGroupsCaching {
@Override
public List<String> getGroups(String user) throws IOException {
- LOG.info("Getting groups for " + user);
+ TESTLOG.info("Getting groups for " + user);
delayIfNecessary();
requestCount++;
@@ -115,18 +115,18 @@ public class TestGroupsCaching {
@Override
public void cacheGroupsRefresh() throws IOException {
- LOG.info("Cache is being refreshed.");
+ TESTLOG.info("Cache is being refreshed.");
clearBlackList();
return;
}
public static void clearBlackList() throws IOException {
- LOG.info("Clearing the blacklist");
+ TESTLOG.info("Clearing the blacklist");
blackList.clear();
}
public static void clearAll() throws IOException {
- LOG.info("Resetting FakeGroupMapping");
+ TESTLOG.info("Resetting FakeGroupMapping");
blackList.clear();
allGroups.clear();
requestCount = 0;
@@ -137,12 +137,12 @@ public class TestGroupsCaching {
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
- LOG.info("Adding " + groups + " to groups.");
+ TESTLOG.info("Adding " + groups + " to groups.");
allGroups.addAll(groups);
}
public static void addToBlackList(String user) throws IOException {
- LOG.info("Adding " + user + " to the blacklist");
+ TESTLOG.info("Adding " + user + " to the blacklist");
blackList.add(user);
}
@@ -226,11 +226,12 @@ public class TestGroupsCaching {
// ask for a negative entry
try {
- LOG.error("We are not supposed to get here." + groups.getGroups("user1").toString());
+ TESTLOG.error("We are not supposed to get here."
+ + groups.getGroups("user1").toString());
fail();
} catch (IOException ioe) {
if(!ioe.getMessage().startsWith("No groups found")) {
- LOG.error("Got unexpected exception: " + ioe.getMessage());
+ TESTLOG.error("Got unexpected exception: " + ioe.getMessage());
fail();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
index f28cc62..6d9ea08 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
@@ -22,9 +22,15 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.junit.Test;
+
import static org.junit.Assert.*;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
@@ -32,9 +38,13 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class TestShellBasedUnixGroupsMapping {
- private static final Log LOG =
+ private static final Log TESTLOG =
LogFactory.getLog(TestShellBasedUnixGroupsMapping.class);
+ private final GenericTestUtils.LogCapturer shellMappingLog =
+ GenericTestUtils.LogCapturer.captureLogs(
+ ShellBasedUnixGroupsMapping.LOG);
+
private class TestGroupUserNotExist
extends ShellBasedUnixGroupsMapping {
/**
@@ -55,7 +65,7 @@ public class TestShellBasedUnixGroupsMapping {
when(executor.getOutput()).thenReturn("");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -90,7 +100,7 @@ public class TestShellBasedUnixGroupsMapping {
when(executor.getOutput()).thenReturn("9999\n9999 abc def");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -133,7 +143,7 @@ public class TestShellBasedUnixGroupsMapping {
doNothing().when(executor).execute();
when(executor.getOutput()).thenReturn("23\n23 groupname zzz");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -146,7 +156,7 @@ public class TestShellBasedUnixGroupsMapping {
doNothing().when(executor).execute();
when(executor.getOutput()).thenReturn("111\n111 112 113");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -179,7 +189,7 @@ public class TestShellBasedUnixGroupsMapping {
doNothing().when(executor).execute();
when(executor.getOutput()).thenReturn("abc\ndef abc hij");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -192,7 +202,7 @@ public class TestShellBasedUnixGroupsMapping {
doNothing().when(executor).execute();
when(executor.getOutput()).thenReturn("1\n1 2 3");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -208,6 +218,117 @@ public class TestShellBasedUnixGroupsMapping {
assertTrue(groups.contains("def"));
assertTrue(groups.contains("hij"));
}
+
+ private static class TestDelayedGroupCommand
+ extends ShellBasedUnixGroupsMapping {
+
+ private Long timeoutSecs = 2L;
+
+ TestDelayedGroupCommand() {
+ super();
+ }
+
+ @Override
+ protected String[] getGroupsForUserCommand(String userName) {
+ // Sleeps 2 seconds when executed and writes no output
+ if (Shell.WINDOWS) {
+ return new String[]{"timeout", timeoutSecs.toString()};
+ }
+ return new String[]{"sleep", timeoutSecs.toString()};
+ }
+
+ @Override
+ protected String[] getGroupsIDForUserCommand(String userName) {
+ return getGroupsForUserCommand(userName);
+ }
+ }
+
+ @Test(timeout=4000)
+ public void testFiniteGroupResolutionTime() throws Exception {
+ Configuration conf = new Configuration();
+ String userName = "foobarnonexistinguser";
+ String commandTimeoutMessage =
+ "ran longer than the configured timeout limit";
+ long testTimeout = 1L;
+
+ // Test a 1 second max-runtime timeout
+ conf.setLong(
+ CommonConfigurationKeys.
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS,
+ testTimeout);
+
+ TestDelayedGroupCommand mapping =
+ ReflectionUtils.newInstance(TestDelayedGroupCommand.class, conf);
+
+ ShellCommandExecutor executor = mapping.createGroupExecutor(userName);
+ assertEquals(
+ "Expected the group names executor to carry the configured timeout",
+ testTimeout,
+ executor.getTimeoutInterval());
+
+ executor = mapping.createGroupIDExecutor(userName);
+ assertEquals(
+ "Expected the group ID executor to carry the configured timeout",
+ testTimeout,
+ executor.getTimeoutInterval());
+
+ assertEquals(
+ "Expected no groups to be returned given a shell command timeout",
+ 0,
+ mapping.getGroups(userName).size());
+ assertTrue(
+ "Expected the logs to carry " +
+ "a message about command timeout but was: " +
+ shellMappingLog.getOutput(),
+ shellMappingLog.getOutput().contains(commandTimeoutMessage));
+ shellMappingLog.clearOutput();
+
+ // Test also the parent Groups framework for expected behaviour
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ TestDelayedGroupCommand.class,
+ GroupMappingServiceProvider.class);
+ Groups groups = new Groups(conf);
+ try {
+ groups.getGroups(userName);
+ fail(
+ "The groups framework call should " +
+ "have failed with a command timeout");
+ } catch (IOException e) {
+ assertTrue(
+ "Expected the logs to carry " +
+ "a message about command timeout but was: " +
+ shellMappingLog.getOutput(),
+ shellMappingLog.getOutput().contains(commandTimeoutMessage));
+ }
+ shellMappingLog.clearOutput();
+
+ // Test the no-timeout (default) configuration
+ conf = new Configuration();
+ long defaultTimeout =
+ CommonConfigurationKeys.
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT;
+
+ mapping =
+ ReflectionUtils.newInstance(TestDelayedGroupCommand.class, conf);
+
+ executor = mapping.createGroupExecutor(userName);
+ assertEquals(
+ "Expected the group names executor to carry the default timeout",
+ defaultTimeout,
+ executor.getTimeoutInterval());
+
+ executor = mapping.createGroupIDExecutor(userName);
+ assertEquals(
+ "Expected the group ID executor to carry the default timeout",
+ defaultTimeout,
+ executor.getTimeoutInterval());
+
+ mapping.getGroups(userName);
+ assertFalse(
+ "Didn't expect a timeout of command in execution but logs carry it: " +
+ shellMappingLog.getOutput(),
+ shellMappingLog.getOutput().contains(commandTimeoutMessage));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-6222.
TestFairScheduler.testReservationMetrics is flaky. (Yufei Gu via kasha)
Posted by jh...@apache.org.
YARN-6222. TestFairScheduler.testReservationMetrics is flaky. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/694e680d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/694e680d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/694e680d
Branch: refs/heads/YARN-5734
Commit: 694e680d20dc07f634b539537021b09d9316601c
Parents: 159d6c5
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Thu Feb 23 15:21:52 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Thu Feb 23 15:21:52 2017 -0800
----------------------------------------------------------------------
.../yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java | 3 +--
.../server/resourcemanager/scheduler/fair/TestFairScheduler.java | 1 +
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/694e680d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 59bde5b..d0e0961 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -407,8 +407,7 @@ public class FSLeafQueue extends FSQueue {
readLock.lock();
try {
for (FSAppAttempt app : runnableApps) {
- Resource pending = app.getAppAttemptResourceUsage().getPending();
- if (!Resources.isNone(pending) &&
+ if (!Resources.isNone(app.getPendingDemand()) &&
(assignment || app.shouldCheckForStarvation())) {
pendingForResourceApps.add(app);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/694e680d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 62430bf..31dd7fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -5079,6 +5079,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
createSchedulingRequestExistingApplication(1024, 1, 1, appAttemptId);
+ scheduler.update();
scheduler.handle(updateEvent);
// no reservation yet
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDFS-11382. Persist Erasure Coding
Policy ID in a new optional field in INodeFile in FSImage. Contributed by
Manoj Govindassamy.
Posted by jh...@apache.org.
HDFS-11382. Persist Erasure Coding Policy ID in a new optional field in INodeFile in FSImage. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55c07bbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55c07bbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55c07bbe
Branch: refs/heads/YARN-5734
Commit: 55c07bbed2f475f7b584a86112ee1b6fe0221e98
Parents: 5f5b031
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Feb 27 17:07:45 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Feb 27 17:07:45 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 2 +-
.../dev-support/findbugsExcludeFile.xml | 6 ++
.../hdfs/server/namenode/FSDirWriteFileOp.java | 36 ++++++----
.../hdfs/server/namenode/FSImageFormat.java | 6 +-
.../server/namenode/FSImageFormatPBINode.java | 25 ++++---
.../hadoop/hdfs/server/namenode/INodeFile.java | 69 ++++++++++++++------
.../server/namenode/INodeFileAttributes.java | 6 +-
.../snapshot/FSImageFormatPBSnapshot.java | 9 ++-
.../tools/offlineImageViewer/FSImageLoader.java | 7 +-
.../offlineImageViewer/PBImageXmlWriter.java | 4 +-
.../hadoop-hdfs/src/main/proto/fsimage.proto | 1 +
.../hadoop/hdfs/TestErasureCodingPolicies.java | 5 +-
.../hdfs/server/namenode/TestFSImage.java | 25 ++++---
.../hdfs/server/namenode/TestINodeFile.java | 64 ++++++++++++++++--
.../server/namenode/TestStripedINodeFile.java | 62 +++++++++++++++++-
.../TestOfflineImageViewer.java | 56 +++++++++++++++-
...TestOfflineImageViewerWithStripedBlocks.java | 2 +
17 files changed, 313 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
index 2a04800..17926a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
@@ -79,7 +79,7 @@ public class HdfsFileStatus {
byte storagePolicy, ErasureCodingPolicy ecPolicy) {
this.length = length;
this.isdir = isdir;
- this.block_replication = ecPolicy == null ? (short) block_replication : 0;
+ this.block_replication = (short) block_replication;
this.blocksize = blocksize;
this.modification_time = modification_time;
this.access_time = access_time;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index 3fa4e8d..598f4fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -234,4 +234,10 @@
<Method name="assertAllResultsEqual" />
<Bug pattern="NP_LOAD_OF_KNOWN_NULL_VALUE" />
</Match>
+ <!-- Manually verified that signed byte value involved in bitwise OR is not negative -->
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.server.namenode.INodeFile$HeaderFormat" />
+ <Method name="getBlockLayoutRedundancy" />
+ <Bug pattern="BIT_IOR_OF_SIGNED_BYTE" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index fbd9ad7..0a4d3b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -416,21 +416,25 @@ class FSDirWriteFileOp {
assert fsd.hasWriteLock();
try {
// check if the file has an EC policy
+ boolean isStriped = false;
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
getErasureCodingPolicy(fsd.getFSNamesystem(), existing);
if (ecPolicy != null) {
- replication = ecPolicy.getId();
+ isStriped = true;
}
- final BlockType blockType = ecPolicy != null?
+ final BlockType blockType = isStriped ?
BlockType.STRIPED : BlockType.CONTIGUOUS;
+ final Short replicationFactor = (!isStriped ? replication : null);
+ final Byte ecPolicyID = (isStriped ? ecPolicy.getId() : null);
if (underConstruction) {
newNode = newINodeFile(id, permissions, modificationTime,
- modificationTime, replication, preferredBlockSize, storagePolicyId,
- blockType);
+ modificationTime, replicationFactor, ecPolicyID, preferredBlockSize,
+ storagePolicyId, blockType);
newNode.toUnderConstruction(clientName, clientMachine);
} else {
newNode = newINodeFile(id, permissions, modificationTime, atime,
- replication, preferredBlockSize, storagePolicyId, blockType);
+ replicationFactor, ecPolicyID, preferredBlockSize,
+ storagePolicyId, blockType);
}
newNode.setLocalName(localName);
INodesInPath iip = fsd.addINode(existing, newNode,
@@ -523,15 +527,19 @@ class FSDirWriteFileOp {
INodesInPath newiip;
fsd.writeLock();
try {
+ boolean isStriped = false;
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
getErasureCodingPolicy(fsd.getFSNamesystem(), existing);
if (ecPolicy != null) {
- replication = ecPolicy.getId();
+ isStriped = true;
}
- final BlockType blockType = ecPolicy != null?
+ final BlockType blockType = isStriped ?
BlockType.STRIPED : BlockType.CONTIGUOUS;
+ final Short replicationFactor = (!isStriped ? replication : null);
+ final Byte ecPolicyID = (isStriped ? ecPolicy.getId() : null);
INodeFile newNode = newINodeFile(fsd.allocateNewInodeId(), permissions,
- modTime, modTime, replication, preferredBlockSize, blockType);
+ modTime, modTime, replicationFactor, ecPolicyID, preferredBlockSize,
+ blockType);
newNode.setLocalName(localName);
newNode.toUnderConstruction(clientName, clientMachine);
newiip = fsd.addINode(existing, newNode, permissions.getPermission());
@@ -702,17 +710,17 @@ class FSDirWriteFileOp {
private static INodeFile newINodeFile(
long id, PermissionStatus permissions, long mtime, long atime,
- short replication, long preferredBlockSize, byte storagePolicyId,
- BlockType blockType) {
+ Short replication, Byte ecPolicyID, long preferredBlockSize,
+ byte storagePolicyId, BlockType blockType) {
return new INodeFile(id, null, permissions, mtime, atime,
- BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize,
+ BlockInfo.EMPTY_ARRAY, replication, ecPolicyID, preferredBlockSize,
storagePolicyId, blockType);
}
private static INodeFile newINodeFile(long id, PermissionStatus permissions,
- long mtime, long atime, short replication, long preferredBlockSize,
- BlockType blockType) {
- return newINodeFile(id, permissions, mtime, atime, replication,
+ long mtime, long atime, Short replication, Byte ecPolicyID,
+ long preferredBlockSize, BlockType blockType) {
+ return newINodeFile(id, permissions, mtime, atime, replication, ecPolicyID,
preferredBlockSize, (byte)0, blockType);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index 9d57604..83f9c93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -896,9 +896,9 @@ public class FSImageFormat {
in.readShort());
final long preferredBlockSize = in.readLong();
- return new INodeFileAttributes.SnapshotCopy(name, permissions, null, modificationTime,
- accessTime, replication, preferredBlockSize, (byte) 0, null,
- BlockType.CONTIGUOUS);
+ return new INodeFileAttributes.SnapshotCopy(name, permissions, null,
+ modificationTime, accessTime, replication, null, preferredBlockSize,
+ (byte) 0, null, BlockType.CONTIGUOUS);
}
public INodeDirectoryAttributes loadINodeDirectoryAttributes(DataInput in)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index fcc7012..0ceae78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -329,17 +329,19 @@ public final class FSImageFormatPBINode {
assert n.getType() == INodeSection.INode.Type.FILE;
INodeSection.INodeFile f = n.getFile();
List<BlockProto> bp = f.getBlocksList();
- short replication = (short) f.getReplication();
BlockType blockType = PBHelperClient.convert(f.getBlockType());
LoaderContext state = parent.getLoaderContext();
- ErasureCodingPolicy ecPolicy = (blockType == BlockType.STRIPED) ?
- ErasureCodingPolicyManager.getPolicyByPolicyID((byte) replication) :
- null;
+ boolean isStriped = f.hasErasureCodingPolicyID();
+ Short replication = (!isStriped ? (short) f.getReplication() : null);
+ ErasureCodingPolicy ecPolicy = isStriped ?
+ ErasureCodingPolicyManager.getPolicyByPolicyID(
+ (byte) f.getErasureCodingPolicyID()) : null;
+ Byte ecPolicyID = (isStriped ? ecPolicy.getId() : null);
BlockInfo[] blocks = new BlockInfo[bp.size()];
for (int i = 0; i < bp.size(); ++i) {
BlockProto b = bp.get(i);
- if (blockType == BlockType.STRIPED) {
+ if (isStriped) {
blocks[i] = new BlockInfoStriped(PBHelperClient.convert(b), ecPolicy);
} else {
blocks[i] = new BlockInfoContiguous(PBHelperClient.convert(b),
@@ -352,8 +354,8 @@ public final class FSImageFormatPBINode {
final INodeFile file = new INodeFile(n.getId(),
n.getName().toByteArray(), permissions, f.getModificationTime(),
- f.getAccessTime(), blocks, replication, f.getPreferredBlockSize(),
- (byte)f.getStoragePolicyID(), blockType);
+ f.getAccessTime(), blocks, replication, ecPolicyID,
+ f.getPreferredBlockSize(), (byte)f.getStoragePolicyID(), blockType);
if (f.hasAcl()) {
int[] entries = AclEntryStatusFormat.toInt(loadAclEntries(
@@ -376,7 +378,7 @@ public final class FSImageFormatPBINode {
BlockInfo lastBlk = file.getLastBlock();
// replace the last block of file
final BlockInfo ucBlk;
- if (blockType == BlockType.STRIPED) {
+ if (isStriped) {
BlockInfoStriped striped = (BlockInfoStriped) lastBlk;
ucBlk = new BlockInfoStriped(striped, ecPolicy);
} else {
@@ -503,10 +505,15 @@ public final class FSImageFormatPBINode {
.setModificationTime(file.getModificationTime())
.setPermission(buildPermissionStatus(file, state.getStringMap()))
.setPreferredBlockSize(file.getPreferredBlockSize())
- .setReplication(file.getFileReplication())
.setStoragePolicyID(file.getLocalStoragePolicyID())
.setBlockType(PBHelperClient.convert(file.getBlockType()));
+ if (file.isStriped()) {
+ b.setErasureCodingPolicyID(file.getErasureCodingPolicyID());
+ } else {
+ b.setReplication(file.getFileReplication());
+ }
+
AclFeature f = file.getAclFeature();
if (f != null) {
b.setAcl(buildAclEntries(f, state.getStringMap()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index c3c0ab2..0315150 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -62,6 +62,11 @@ import com.google.common.base.Preconditions;
public class INodeFile extends INodeWithAdditionalFields
implements INodeFileAttributes, BlockCollection {
+ /**
+ * Erasure Coded striped blocks have replication factor of 1.
+ */
+ public static final short DEFAULT_REPL_FOR_STRIPED_BLOCKS = 1;
+
/** The same as valueOf(inode, path, false). */
public static INodeFile valueOf(INode inode, String path
) throws FileNotFoundException {
@@ -126,7 +131,6 @@ public class INodeFile extends INodeWithAdditionalFields
* Different types can be replica or EC
*/
private static final int LAYOUT_BIT_WIDTH = 1;
-
private static final int MAX_REDUNDANCY = (1 << 11) - 1;
HeaderFormat(LongBitFormat previous, int length, long min) {
@@ -134,8 +138,13 @@ public class INodeFile extends INodeWithAdditionalFields
}
static short getReplication(long header) {
- long layoutRedundancy = BLOCK_LAYOUT_AND_REDUNDANCY.BITS.retrieve(header);
- return (short) (layoutRedundancy & MAX_REDUNDANCY);
+ if (isStriped(header)) {
+ return DEFAULT_REPL_FOR_STRIPED_BLOCKS;
+ } else {
+ long layoutRedundancy =
+ BLOCK_LAYOUT_AND_REDUNDANCY.BITS.retrieve(header);
+ return (short) (layoutRedundancy & MAX_REDUNDANCY);
+ }
}
static byte getECPolicyID(long header) {
@@ -158,8 +167,7 @@ public class INodeFile extends INodeWithAdditionalFields
static final long BLOCK_TYPE_MASK_STRIPED = 1 << 11;
static boolean isStriped(long header) {
- long layoutRedundancy = BLOCK_LAYOUT_AND_REDUNDANCY.BITS.retrieve(header);
- return (layoutRedundancy & BLOCK_TYPE_MASK) != 0;
+ return getBlockType(header) == STRIPED;
}
static BlockType getBlockType(long header) {
@@ -172,22 +180,40 @@ public class INodeFile extends INodeWithAdditionalFields
}
}
- static long toLong(long preferredBlockSize, short replication,
- BlockType blockType, byte storagePolicyID) {
- Preconditions.checkArgument(replication >= 0 &&
- replication <= MAX_REDUNDANCY);
+ /**
+ * Construct block layout redundancy based on the given BlockType,
+ * replication factor and EC PolicyID.
+ */
+ static long getBlockLayoutRedundancy(final BlockType blockType,
+ final Short replication, final Byte erasureCodingPolicyID) {
+ long layoutRedundancy = 0;
+ if (blockType == STRIPED) {
+ Preconditions.checkArgument(replication == null &&
+ erasureCodingPolicyID != null);
+ Preconditions.checkArgument(
+ ErasureCodingPolicyManager.getPolicyByPolicyID(
+ erasureCodingPolicyID) != null);
+ layoutRedundancy |= BLOCK_TYPE_MASK_STRIPED;
+ // Following bitwise OR with signed byte erasureCodingPolicyID is safe
+ // as the PolicyID can never be in negative.
+ layoutRedundancy |= erasureCodingPolicyID;
+ } else {
+ Preconditions.checkArgument(replication != null &&
+ erasureCodingPolicyID == null);
+ Preconditions.checkArgument(replication >= 0 &&
+ replication <= MAX_REDUNDANCY);
+ layoutRedundancy |= replication;
+ }
+ return layoutRedundancy;
+ }
+
+ static long toLong(long preferredBlockSize, long layoutRedundancy,
+ byte storagePolicyID) {
long h = 0;
if (preferredBlockSize == 0) {
preferredBlockSize = PREFERRED_BLOCK_SIZE.BITS.getMin();
}
h = PREFERRED_BLOCK_SIZE.BITS.combine(preferredBlockSize, h);
- // For erasure coded files, replication is used to store ec policy id
- // TODO: this is hacky. Add some utility to generate the layoutRedundancy
- long layoutRedundancy = 0;
- if (blockType == STRIPED) {
- layoutRedundancy |= BLOCK_TYPE_MASK_STRIPED;
- }
- layoutRedundancy |= replication;
h = BLOCK_LAYOUT_AND_REDUNDANCY.BITS.combine(layoutRedundancy, h);
h = STORAGE_POLICY_ID.BITS.combine(storagePolicyID, h);
return h;
@@ -202,15 +228,17 @@ public class INodeFile extends INodeWithAdditionalFields
INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
long atime, BlockInfo[] blklist, short replication,
long preferredBlockSize) {
- this(id, name, permissions, mtime, atime, blklist, replication,
+ this(id, name, permissions, mtime, atime, blklist, replication, null,
preferredBlockSize, (byte) 0, CONTIGUOUS);
}
INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
- long atime, BlockInfo[] blklist, short replication,
+ long atime, BlockInfo[] blklist, Short replication, Byte ecPolicyID,
long preferredBlockSize, byte storagePolicyID, BlockType blockType) {
super(id, name, permissions, mtime, atime);
- header = HeaderFormat.toLong(preferredBlockSize, replication, blockType,
+ final long layoutRedundancy = HeaderFormat.getBlockLayoutRedundancy(
+ blockType, replication, ecPolicyID);
+ header = HeaderFormat.toLong(preferredBlockSize, layoutRedundancy,
storagePolicyID);
if (blklist != null && blklist.length > 0) {
for (BlockInfo b : blklist) {
@@ -462,6 +490,9 @@ public class INodeFile extends INodeWithAdditionalFields
* */
@Override // INodeFileAttributes
public final short getFileReplication() {
+ if (isStriped()) {
+ return DEFAULT_REPL_FOR_STRIPED_BLOCKS;
+ }
return getFileReplication(CURRENT_STATE_ID);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
index 59dc924..4e02bb0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
@@ -56,11 +56,13 @@ public interface INodeFileAttributes extends INodeAttributes {
public SnapshotCopy(byte[] name, PermissionStatus permissions,
AclFeature aclFeature, long modificationTime, long accessTime,
- short replication, long preferredBlockSize,
+ Short replication, Byte ecPolicyID, long preferredBlockSize,
byte storagePolicyID, XAttrFeature xAttrsFeature, BlockType blockType) {
super(name, permissions, aclFeature, modificationTime, accessTime,
xAttrsFeature);
- header = HeaderFormat.toLong(preferredBlockSize, replication, blockType,
+ final long layoutRedundancy = HeaderFormat.getBlockLayoutRedundancy(
+ blockType, replication, ecPolicyID);
+ header = HeaderFormat.toLong(preferredBlockSize, layoutRedundancy,
storagePolicyID);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index 9a49498..bc7b051 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTypeProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
@@ -232,9 +233,15 @@ public class FSImageFormatPBSnapshot {
fileInPb.getXAttrs(), state.getStringTable()));
}
+ boolean isStriped =
+ (fileInPb.getBlockType() == BlockTypeProto .STRIPED);
+ Short replication =
+ (!isStriped ? (short)fileInPb.getReplication() : null);
+ Byte ecPolicyID =
+ (isStriped ? (byte)fileInPb.getErasureCodingPolicyID() : null);
copy = new INodeFileAttributes.SnapshotCopy(pbf.getName()
.toByteArray(), permission, acl, fileInPb.getModificationTime(),
- fileInPb.getAccessTime(), (short) fileInPb.getReplication(),
+ fileInPb.getAccessTime(), replication, ecPolicyID,
fileInPb.getPreferredBlockSize(),
(byte)fileInPb.getStoragePolicyID(), xAttrs,
PBHelperClient.convert(fileInPb.getBlockType()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index 21e9d2e..61494f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf;
import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.resources.XAttrEncodingParam;
@@ -589,7 +590,11 @@ class FSImageLoader {
map.put("pathSuffix",
printSuffix ? inode.getName().toStringUtf8() : "");
map.put("permission", toString(p.getPermission()));
- map.put("replication", f.getReplication());
+ if (f.hasErasureCodingPolicyID()) {
+ map.put("replication", INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS);
+ } else {
+ map.put("replication", f.getReplication());
+ }
map.put("type", inode.getType());
map.put("fileId", inode.getId());
map.put("childrenNum", 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
index 7f0bf38..f8734cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
@@ -496,9 +496,7 @@ public final class PBImageXmlWriter {
o(INODE_SECTION_STORAGE_POLICY_ID, f.getStoragePolicyID());
}
if (f.getBlockType() != BlockTypeProto.CONTIGUOUS) {
- out.print("<" + INODE_SECTION_BLOCK_TYPE + ">");
- o(SECTION_NAME, f.getBlockType().name());
- out.print("</" + INODE_SECTION_BLOCK_TYPE + ">\n");
+ o(INODE_SECTION_BLOCK_TYPE, f.getBlockType().name());
}
if (f.hasFileUC()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
index 4b8c93b..4e21310 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
@@ -141,6 +141,7 @@ message INodeSection {
optional XAttrFeatureProto xAttrs = 9;
optional uint32 storagePolicyID = 10;
optional BlockTypeProto blockType = 11;
+ optional uint32 erasureCodingPolicyID = 12;
}
message QuotaByStorageTypeEntryProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 27fbf18..0e431fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.junit.After;
import org.junit.Before;
@@ -343,7 +344,9 @@ public class TestErasureCodingPolicies {
assertEquals(policy, fs.getErasureCodingPolicy(file));
assertEquals(policy, fs.getErasureCodingPolicy(dir));
INode iNode = namesystem.getFSDirectory().getINode(file.toString());
- assertEquals(policy.getId(), iNode.asFile().getFileReplication());
+ assertEquals(policy.getId(), iNode.asFile().getErasureCodingPolicyID());
+ assertEquals(INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS,
+ iNode.asFile().getFileReplication());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 339c591..179e297 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -155,7 +155,7 @@ public class TestFSImage {
private void testSaveAndLoadStripedINodeFile(FSNamesystem fsn, Configuration conf,
boolean isUC) throws IOException{
- // contruct a INode with StripedBlock for saving and loading
+ // Construct an INode with StripedBlock for saving and loading
fsn.setErasureCodingPolicy("/", testECPolicy, false);
long id = 123456789;
byte[] name = "testSaveAndLoadInodeFile_testfile".getBytes();
@@ -163,29 +163,30 @@ public class TestFSImage {
"testuser_groups", new FsPermission((short)0x755));
long mtime = 1426222916-3600;
long atime = 1426222916;
- BlockInfoContiguous[] blks = new BlockInfoContiguous[0];
- short replication = testECPolicy.getId();
+ BlockInfoContiguous[] blocks = new BlockInfoContiguous[0];
+ byte erasureCodingPolicyID = testECPolicy.getId();
long preferredBlockSize = 128*1024*1024;
INodeFile file = new INodeFile(id, name, permissionStatus, mtime, atime,
- blks, replication, preferredBlockSize, (byte) 0, BlockType.STRIPED);
+ blocks, null, erasureCodingPolicyID, preferredBlockSize,
+ (byte) 0, BlockType.STRIPED);
ByteArrayOutputStream bs = new ByteArrayOutputStream();
- //construct StripedBlocks for the INode
- BlockInfoStriped[] stripedBlks = new BlockInfoStriped[3];
+ // Construct StripedBlocks for the INode
+ BlockInfoStriped[] stripedBlocks = new BlockInfoStriped[3];
long stripedBlkId = 10000001;
long timestamp = mtime+3600;
- for (int i = 0; i < stripedBlks.length; i++) {
- stripedBlks[i] = new BlockInfoStriped(
+ for (int i = 0; i < stripedBlocks.length; i++) {
+ stripedBlocks[i] = new BlockInfoStriped(
new Block(stripedBlkId + i, preferredBlockSize, timestamp),
testECPolicy);
- file.addBlock(stripedBlks[i]);
+ file.addBlock(stripedBlocks[i]);
}
final String client = "testClient";
final String clientMachine = "testClientMachine";
final String path = "testUnderConstructionPath";
- //save the INode to byte array
+ // Save the INode to byte array
DataOutput out = new DataOutputStream(bs);
if (isUC) {
file.toUnderConstruction(client, clientMachine);
@@ -495,6 +496,7 @@ public class TestFSImage {
FSNamesystem fsn = cluster.getNamesystem();
INodeFile inode = fsn.dir.getINode(file_10_4.toString()).asFile();
assertTrue(inode.isStriped());
+ assertEquals(testECPolicy.getId(), inode.getErasureCodingPolicyID());
BlockInfo[] blks = inode.getBlocks();
assertEquals(1, blks.length);
assertTrue(blks[0].isStriped());
@@ -513,6 +515,9 @@ public class TestFSImage {
// check the information of file_3_2
inode = fsn.dir.getINode(file_3_2.toString()).asFile();
assertTrue(inode.isStriped());
+ assertEquals(ErasureCodingPolicyManager.getPolicyByPolicyID(
+ HdfsConstants.RS_3_2_POLICY_ID).getId(),
+ inode.getErasureCodingPolicyID());
blks = inode.getBlocks();
assertEquals(1, blks.length);
assertTrue(blks[0].isStriped());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index a0768a5..ced6c7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Time;
+import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
@@ -104,15 +105,15 @@ public class TestINodeFile {
null, replication, preferredBlockSize);
}
- INodeFile createStripedINodeFile(short replication, long preferredBlockSize) {
+ INodeFile createStripedINodeFile(long preferredBlockSize) {
return new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
- null, replication, preferredBlockSize,
+ null, null, HdfsConstants.RS_6_3_POLICY_ID, preferredBlockSize,
HdfsConstants.WARM_STORAGE_POLICY_ID, STRIPED);
}
private static INodeFile createINodeFile(byte storagePolicyID) {
return new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
- null, (short)3, 1024L, storagePolicyID, CONTIGUOUS);
+ null, (short)3, null, 1024L, storagePolicyID, CONTIGUOUS);
}
@Test
@@ -133,6 +134,61 @@ public class TestINodeFile {
createINodeFile((byte)16);
}
+ @Test
+ public void testContiguousLayoutRedundancy() {
+ INodeFile inodeFile;
+ try {
+ new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID,
+ null, perm, 0L, 0L, null, new Short((short) 3) /*replication*/,
+ HdfsConstants.RS_6_3_POLICY_ID /*ec policy*/,
+ preferredBlockSize, HdfsConstants.WARM_STORAGE_POLICY_ID, CONTIGUOUS);
+ fail("INodeFile construction should fail when both replication and " +
+ "ECPolicy requested!");
+ } catch (IllegalArgumentException iae) {
+ LOG.info("Expected exception: ", iae);
+ }
+
+ try {
+ new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID,
+ null, perm, 0L, 0L, null, null /*replication*/, null /*ec policy*/,
+ preferredBlockSize, HdfsConstants.WARM_STORAGE_POLICY_ID, CONTIGUOUS);
+ fail("INodeFile construction should fail when replication param not " +
+ "provided for contiguous layout!");
+ } catch (IllegalArgumentException iae) {
+ LOG.info("Expected exception: ", iae);
+ }
+
+ try {
+ new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID,
+ null, perm, 0L, 0L, null, Short.MAX_VALUE /*replication*/,
+ null /*ec policy*/, preferredBlockSize,
+ HdfsConstants.WARM_STORAGE_POLICY_ID, CONTIGUOUS);
+ fail("INodeFile construction should fail when replication param is " +
+ "beyond the range supported!");
+ } catch (IllegalArgumentException iae) {
+ LOG.info("Expected exception: ", iae);
+ }
+
+ final Short replication = new Short((short) 3);
+ try {
+ new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID,
+ null, perm, 0L, 0L, null, replication, null /*ec policy*/,
+ preferredBlockSize, HdfsConstants.WARM_STORAGE_POLICY_ID, STRIPED);
+ fail("INodeFile construction should fail when replication param is " +
+ "provided for striped layout!");
+ } catch (IllegalArgumentException iae) {
+ LOG.info("Expected exception: ", iae);
+ }
+
+ inodeFile = new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID,
+ null, perm, 0L, 0L, null, replication, null /*ec policy*/,
+ preferredBlockSize, HdfsConstants.WARM_STORAGE_POLICY_ID, CONTIGUOUS);
+
+ Assert.assertTrue(!inodeFile.isStriped());
+ Assert.assertEquals(replication.shortValue(),
+ inodeFile.getFileReplication());
+ }
+
/**
* Test for the Replication value. Sets a value and checks if it was set
* correct.
@@ -237,7 +293,7 @@ public class TestINodeFile {
preferredBlockSize = 128*1024*1024;
INodeFile inf = createINodeFile(replication, preferredBlockSize);
assertEquals(inf.getBlockType(), CONTIGUOUS);
- INodeFile striped = createStripedINodeFile(replication, preferredBlockSize);
+ INodeFile striped = createStripedINodeFile(preferredBlockSize);
assertEquals(striped.getBlockType(), STRIPED);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
index 14b4500..03bd124 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -17,9 +17,12 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.hdfs.protocol.BlockType.CONTIGUOUS;
+import static org.apache.hadoop.hdfs.protocol.BlockType.STRIPED;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
import java.io.IOException;
@@ -76,8 +79,8 @@ public class TestStripedINodeFile {
private static INodeFile createStripedINodeFile() {
return new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
- null, (short)0, 1024L, HdfsConstants.COLD_STORAGE_POLICY_ID,
- BlockType.STRIPED);
+ null, null, HdfsConstants.RS_6_3_POLICY_ID, 1024L,
+ HdfsConstants.COLD_STORAGE_POLICY_ID, BlockType.STRIPED);
}
@Test
@@ -96,6 +99,61 @@ public class TestStripedINodeFile {
}
@Test
+ public void testStripedLayoutRedundancy() {
+ INodeFile inodeFile;
+ try {
+ new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID,
+ null, perm, 0L, 0L, null, new Short((short) 3) /*replication*/,
+ HdfsConstants.RS_6_3_POLICY_ID /*ec policy*/,
+ 1024L, HdfsConstants.WARM_STORAGE_POLICY_ID, STRIPED);
+ fail("INodeFile construction should fail when both replication and " +
+ "ECPolicy requested!");
+ } catch (IllegalArgumentException iae) {
+ LOG.info("Expected exception: ", iae);
+ }
+
+ try {
+ new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID,
+ null, perm, 0L, 0L, null, null /*replication*/, null /*ec policy*/,
+ 1024L, HdfsConstants.WARM_STORAGE_POLICY_ID, STRIPED);
+ fail("INodeFile construction should fail when EC Policy param not " +
+ "provided for striped layout!");
+ } catch (IllegalArgumentException iae) {
+ LOG.info("Expected exception: ", iae);
+ }
+
+ try {
+ new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID,
+ null, perm, 0L, 0L, null, null /*replication*/,
+ Byte.MAX_VALUE /*ec policy*/, 1024L,
+ HdfsConstants.WARM_STORAGE_POLICY_ID, STRIPED);
+ fail("INodeFile construction should fail when EC Policy is " +
+ "not in the supported list!");
+ } catch (IllegalArgumentException iae) {
+ LOG.info("Expected exception: ", iae);
+ }
+
+ final Byte ecPolicyID = HdfsConstants.RS_6_3_POLICY_ID;
+ try {
+ new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID,
+ null, perm, 0L, 0L, null, null /*replication*/, ecPolicyID,
+ 1024L, HdfsConstants.WARM_STORAGE_POLICY_ID, CONTIGUOUS);
+ fail("INodeFile construction should fail when replication param is " +
+ "provided for striped layout!");
+ } catch (IllegalArgumentException iae) {
+ LOG.info("Expected exception: ", iae);
+ }
+
+ inodeFile = new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID,
+ null, perm, 0L, 0L, null, null /*replication*/, ecPolicyID,
+ 1024L, HdfsConstants.WARM_STORAGE_POLICY_ID, STRIPED);
+
+ Assert.assertTrue(inodeFile.isStriped());
+ Assert.assertEquals(ecPolicyID.byteValue(),
+ inodeFile.getErasureCodingPolicyID());
+ }
+
+ @Test
public void testBlockStripedLength()
throws IOException, InterruptedException {
INodeFile inf = createStripedINodeFile();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index f77911d..053b856 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -48,6 +48,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.Random;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -75,7 +76,10 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
@@ -103,6 +107,7 @@ public class TestOfflineImageViewer {
private static final int FILES_PER_DIR = 4;
private static final String TEST_RENEWER = "JobTracker";
private static File originalFsimage = null;
+ private static int filesECCount = 0;
// namespace as written to dfs, to be compared with viewer's output
final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
@@ -129,7 +134,7 @@ public class TestOfflineImageViewer {
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL,
"RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
cluster.waitActive();
DistributedFileSystem hdfs = cluster.getFileSystem();
@@ -224,9 +229,38 @@ public class TestOfflineImageViewer {
aclEntry(ACCESS, GROUP, "bar", READ_EXECUTE),
aclEntry(ACCESS, OTHER, EXECUTE)));
+ // Create an Erasure Coded dir
+ Path ecDir = new Path("/ec");
+ hdfs.mkdirs(ecDir);
+ dirCount++;
+ ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getPolicyByPolicyID(
+ HdfsConstants.XOR_2_1_POLICY_ID);
+ hdfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy);
+ writtenFiles.put(ecDir.toString(), hdfs.getFileStatus(ecDir));
+
+ // Create an empty Erasure Coded file
+ Path emptyECFile = new Path(ecDir, "EmptyECFile.txt");
+ hdfs.create(emptyECFile).close();
+ writtenFiles.put(emptyECFile.toString(),
+ pathToFileEntry(hdfs, emptyECFile.toString()));
+ filesECCount++;
+
+ // Create a small Erasure Coded file
+ Path smallECFile = new Path(ecDir, "SmallECFile.txt");
+ FSDataOutputStream out = hdfs.create(smallECFile);
+ Random r = new Random();
+ byte[] bytes = new byte[1024 * 10];
+ r.nextBytes(bytes);
+ out.write(bytes);
+ writtenFiles.put(smallECFile.toString(),
+ pathToFileEntry(hdfs, smallECFile.toString()));
+ filesECCount++;
+
// Write results to the fsimage file
hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
hdfs.saveNamespace();
+ hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE, false);
// Determine location of fsimage file
originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
@@ -292,7 +326,7 @@ public class TestOfflineImageViewer {
Matcher matcher = p.matcher(outputString);
assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalFiles = Integer.parseInt(matcher.group(1));
- assertEquals(NUM_DIRS * FILES_PER_DIR + 1, totalFiles);
+ assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles);
p = Pattern.compile("totalDirectories = (\\d+)\n");
matcher = p.matcher(outputString);
@@ -375,6 +409,24 @@ public class TestOfflineImageViewer {
url = new URL("http://localhost:" + port + "/foo");
verifyHttpResponseCode(HttpURLConnection.HTTP_NOT_FOUND, url);
+ // Verify the Erasure Coded empty file status
+ Path emptyECFilePath = new Path("/ec/EmptyECFile.txt");
+ FileStatus actualEmptyECFileStatus =
+ webhdfs.getFileStatus(new Path(emptyECFilePath.toString()));
+ FileStatus expectedEmptyECFileStatus = writtenFiles.get(
+ emptyECFilePath.toString());
+ System.out.println(webhdfs.getFileStatus(new Path(emptyECFilePath
+ .toString())));
+ compareFile(expectedEmptyECFileStatus, actualEmptyECFileStatus);
+
+ // Verify the Erasure Coded small file status
+ Path smallECFilePath = new Path("/ec/SmallECFile.txt");
+ FileStatus actualSmallECFileStatus =
+ webhdfs.getFileStatus(new Path(smallECFilePath.toString()));
+ FileStatus expectedSmallECFileStatus = writtenFiles.get(
+ smallECFilePath.toString());
+ compareFile(expectedSmallECFileStatus, actualSmallECFileStatus);
+
// GETFILESTATUS operation
status = webhdfs.getFileStatus(new Path("/dir0/file0"));
compareFile(expected, status);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c07bbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
index 6fd0a1a..db21217 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
@@ -143,6 +143,8 @@ public class TestOfflineImageViewerWithStripedBlocks {
// Verify space consumed present in BlockInfoStriped
FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
+ assertEquals(ErasureCodingPolicyManager.getSystemDefaultPolicy().getId(),
+ fileNode.getErasureCodingPolicyID());
assertTrue("Invalid block size", fileNode.getBlocks().length > 0);
long actualFileSize = 0;
for (BlockInfo blockInfo : fileNode.getBlocks()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-6239. Fix javadoc warnings in
YARN that caused by deprecated FileSystem APIs. Contributed by Yiqun Lin.
Posted by jh...@apache.org.
YARN-6239. Fix javadoc warnings in YARN that caused by deprecated FileSystem APIs. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f35443b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f35443b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f35443b
Branch: refs/heads/YARN-5734
Commit: 0f35443bf262bf2b92ca01cc76178aeb32533ae2
Parents: ef48804
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Feb 28 18:51:56 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Tue Feb 28 18:51:56 2017 +0800
----------------------------------------------------------------------
.../TestFileSystemApplicationHistoryStore.java | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f35443b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
index bd6bea3..15a00d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
@@ -35,6 +35,7 @@ import static org.mockito.Mockito.verify;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
@@ -49,6 +50,7 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.records.Container
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestFileSystemApplicationHistoryStore extends
ApplicationHistoryStoreTestUtils {
@@ -271,7 +273,9 @@ public class TestFileSystemApplicationHistoryStore extends
// Setup file system to inject startup conditions
FileSystem fs = spy(new RawLocalFileSystem());
- doReturn(true).when(fs).isDirectory(any(Path.class));
+ FileStatus fileStatus = Mockito.mock(FileStatus.class);
+ doReturn(true).when(fileStatus).isDirectory();
+ doReturn(fileStatus).when(fs).getFileStatus(any(Path.class));
try {
initAndStartStore(fs);
@@ -280,7 +284,7 @@ public class TestFileSystemApplicationHistoryStore extends
}
// Make sure that directory creation was not attempted
- verify(fs, never()).isDirectory(any(Path.class));
+ verify(fileStatus, never()).isDirectory();
verify(fs, times(1)).mkdirs(any(Path.class));
}
@@ -291,7 +295,9 @@ public class TestFileSystemApplicationHistoryStore extends
// Setup file system to inject startup conditions
FileSystem fs = spy(new RawLocalFileSystem());
- doReturn(false).when(fs).isDirectory(any(Path.class));
+ FileStatus fileStatus = Mockito.mock(FileStatus.class);
+ doReturn(false).when(fileStatus).isDirectory();
+ doReturn(fileStatus).when(fs).getFileStatus(any(Path.class));
doThrow(new IOException()).when(fs).mkdirs(any(Path.class));
try {
@@ -302,7 +308,7 @@ public class TestFileSystemApplicationHistoryStore extends
}
// Make sure that directory creation was attempted
- verify(fs, never()).isDirectory(any(Path.class));
+ verify(fileStatus, never()).isDirectory();
verify(fs, times(1)).mkdirs(any(Path.class));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: Merge branch 'trunk' of
https://git-wip-us.apache.org/repos/asf/hadoop into trunk
Posted by jh...@apache.org.
Merge branch 'trunk' of https://git-wip-us.apache.org/repos/asf/hadoop into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e14eade
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e14eade
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e14eade
Branch: refs/heads/YARN-5734
Commit: 4e14eaded21f244d280b1ffe90a89959fd753047
Parents: 25c84d2 555d0c3
Author: Ming Ma <mi...@apache.org>
Authored: Wed Mar 1 20:24:49 2017 -0800
Committer: Ming Ma <mi...@apache.org>
Committed: Wed Mar 1 20:24:49 2017 -0800
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-6216. Unify Container
Resizing code paths with Container Updates making it scheduler agnostic.
(Arun Suresh via wangda)
Posted by jh...@apache.org.
YARN-6216. Unify Container Resizing code paths with Container Updates making it scheduler agnostic. (Arun Suresh via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eac6b4c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eac6b4c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eac6b4c3
Branch: refs/heads/YARN-5734
Commit: eac6b4c35c50e555c2f1b5f913bb2c4d839f1ff4
Parents: 480b4dd
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Feb 28 10:35:50 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Feb 28 10:35:50 2017 -0800
----------------------------------------------------------------------
.../sls/scheduler/ResourceSchedulerWrapper.java | 8 -
.../server/scheduler/SchedulerRequestKey.java | 12 +-
.../server/resourcemanager/RMServerUtils.java | 27 +-
.../rmcontainer/RMContainer.java | 4 -
.../RMContainerChangeResourceEvent.java | 44 ---
.../rmcontainer/RMContainerImpl.java | 46 ---
.../scheduler/AbstractYarnScheduler.java | 171 +++++++---
.../scheduler/AppSchedulingInfo.java | 283 +---------------
.../scheduler/ContainerUpdateContext.java | 193 ++++++++---
.../scheduler/SchedulerApplicationAttempt.java | 212 ++++--------
.../scheduler/SchedulerNode.java | 44 ---
.../scheduler/capacity/AbstractCSQueue.java | 13 +-
.../scheduler/capacity/CSQueue.java | 15 -
.../scheduler/capacity/CapacityScheduler.java | 121 +------
.../scheduler/capacity/LeafQueue.java | 152 +--------
.../scheduler/capacity/ParentQueue.java | 53 +--
.../capacity/allocator/ContainerAllocator.java | 31 +-
.../allocator/IncreaseContainerAllocator.java | 337 -------------------
.../common/ContainerAllocationProposal.java | 9 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 245 +++-----------
.../common/fica/FiCaSchedulerNode.java | 14 -
.../scheduler/fair/FairScheduler.java | 11 +-
.../scheduler/fifo/FifoScheduler.java | 8 -
.../scheduler/capacity/TestChildQueueOrder.java | 4 +-
.../capacity/TestContainerResizing.java | 134 +-------
.../capacity/TestIncreaseAllocationExpirer.java | 12 +-
.../scheduler/capacity/TestLeafQueue.java | 4 +-
.../scheduler/capacity/TestParentQueue.java | 4 +-
28 files changed, 482 insertions(+), 1729 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
index 5517362..df8323a 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
@@ -969,12 +969,4 @@ final public class ResourceSchedulerWrapper
return Priority.newInstance(0);
}
- @Override
- protected void decreaseContainer(
- SchedContainerChangeRequest decreaseRequest,
- SchedulerApplicationAttempt attempt) {
- // TODO Auto-generated method stub
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
index 02539ba..c4f37f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
@@ -116,7 +116,17 @@ public final class SchedulerRequestKey implements
if (priorityCompare != 0) {
return priorityCompare;
}
- return Long.compare(allocationRequestId, o.getAllocationRequestId());
+ int allocReqCompare = Long.compare(
+ allocationRequestId, o.getAllocationRequestId());
+
+ if (allocReqCompare != 0) {
+ return allocReqCompare;
+ }
+
+ if (this.containerToUpdate != null && o.containerToUpdate != null) {
+ return (this.containerToUpdate.compareTo(o.containerToUpdate));
+ }
+ return 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index e98141b..0aa7a2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -152,26 +152,16 @@ public class RMServerUtils {
if (msg == null) {
if ((updateType != ContainerUpdateType.PROMOTE_EXECUTION_TYPE) &&
(updateType !=ContainerUpdateType.DEMOTE_EXECUTION_TYPE)) {
- Resource original = rmContainer.getContainer().getResource();
- Resource target = updateReq.getCapability();
- if (Resources.fitsIn(target, original)) {
- // This is a decrease request
- if (validateIncreaseDecreaseRequest(rmContext, updateReq,
- maximumAllocation, false)) {
- updateRequests.getDecreaseRequests().add(updateReq);
- outstandingUpdate.add(updateReq.getContainerId());
- } else {
- msg = RESOURCE_OUTSIDE_ALLOWED_RANGE;
- }
- } else {
- // This is an increase request
- if (validateIncreaseDecreaseRequest(rmContext, updateReq,
- maximumAllocation, true)) {
+ if (validateIncreaseDecreaseRequest(
+ rmContext, updateReq, maximumAllocation)) {
+ if (ContainerUpdateType.INCREASE_RESOURCE == updateType) {
updateRequests.getIncreaseRequests().add(updateReq);
- outstandingUpdate.add(updateReq.getContainerId());
} else {
- msg = RESOURCE_OUTSIDE_ALLOWED_RANGE;
+ updateRequests.getDecreaseRequests().add(updateReq);
}
+ outstandingUpdate.add(updateReq.getContainerId());
+ } else {
+ msg = RESOURCE_OUTSIDE_ALLOWED_RANGE;
}
} else {
ExecutionType original = rmContainer.getExecutionType();
@@ -329,8 +319,7 @@ public class RMServerUtils {
// Sanity check and normalize target resource
private static boolean validateIncreaseDecreaseRequest(RMContext rmContext,
- UpdateContainerRequest request, Resource maximumAllocation,
- boolean increase) {
+ UpdateContainerRequest request, Resource maximumAllocation) {
if (request.getCapability().getMemorySize() < 0
|| request.getCapability().getMemorySize() > maximumAllocation
.getMemorySize()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index 020764b..7ad381e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -91,10 +91,6 @@ public interface RMContainer extends EventHandler<RMContainerEvent> {
String getNodeHttpAddress();
String getNodeLabelExpression();
-
- boolean hasIncreaseReservation();
-
- void cancelIncreaseReservation();
String getQueueName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerChangeResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerChangeResourceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerChangeResourceEvent.java
deleted file mode 100644
index 920cfdb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerChangeResourceEvent.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Resource;
-
-public class RMContainerChangeResourceEvent extends RMContainerEvent {
-
- final Resource targetResource;
- final boolean increase;
-
- public RMContainerChangeResourceEvent(ContainerId containerId,
- Resource targetResource, boolean increase) {
- super(containerId, RMContainerEventType.CHANGE_RESOURCE);
-
- this.targetResource = targetResource;
- this.increase = increase;
- }
-
- public Resource getTargetResource() {
- return targetResource;
- }
-
- public boolean isIncrease() {
- return increase;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 72ce1a0..12fbbea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -131,8 +131,6 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
.addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
RMContainerEventType.RESERVED, new ContainerReservedTransition())
.addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
- RMContainerEventType.CHANGE_RESOURCE, new ChangeResourceTransition())
- .addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
RMContainerEventType.ACQUIRE_UPDATED_CONTAINER,
new ContainerAcquiredWhileRunningTransition())
.addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
@@ -183,7 +181,6 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
private boolean isAMContainer;
private List<ResourceRequest> resourceRequests;
- private volatile boolean hasIncreaseReservation = false;
// Only used for container resource increase and decrease. This is the
// resource to rollback to should container resource increase token expires.
private Resource lastConfirmedResource;
@@ -561,12 +558,6 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
if (c != null) {
c.setNodeId(container.reservedNode);
}
-
- if (!EnumSet.of(RMContainerState.NEW, RMContainerState.RESERVED)
- .contains(container.getState())) {
- // When container's state != NEW/RESERVED, it is an increase reservation
- container.hasIncreaseReservation = true;
- }
}
}
@@ -681,33 +672,6 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
}
}
}
-
- private static final class ChangeResourceTransition extends BaseTransition {
-
- @Override
- public void transition(RMContainerImpl container, RMContainerEvent event) {
- RMContainerChangeResourceEvent changeEvent = (RMContainerChangeResourceEvent)event;
-
- Resource targetResource = changeEvent.getTargetResource();
- Resource lastConfirmedResource = container.lastConfirmedResource;
-
- if (!changeEvent.isIncrease()) {
- // Only unregister from the containerAllocationExpirer when target
- // resource is less than or equal to the last confirmed resource.
- if (Resources.fitsIn(targetResource, lastConfirmedResource)) {
- container.lastConfirmedResource = targetResource;
- container.containerAllocationExpirer.unregister(
- new AllocationExpirationInfo(event.getContainerId()));
- }
- }
-
- container.container.setResource(targetResource);
-
- // We reach here means we either allocated increase reservation OR
- // decreased container, reservation will be cancelled anyway.
- container.hasIncreaseReservation = false;
- }
- }
private static class FinishedTransition extends BaseTransition {
@@ -857,16 +821,6 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
return -1;
}
- @Override
- public boolean hasIncreaseReservation() {
- return hasIncreaseReservation;
- }
-
- @Override
- public void cancelIncreaseReservation() {
- hasIncreaseReservation = false;
- }
-
public void setQueueName(String queueName) {
this.queueName = queueName;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index ce6d2a2..213839d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
@@ -44,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -85,6 +87,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+
import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -597,6 +600,8 @@ public abstract class AbstractYarnScheduler
if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
completedContainerInternal(rmContainer, containerStatus, event);
+ completeOustandingUpdatesWhichAreReserved(
+ rmContainer, containerStatus, event);
} else {
ContainerId containerId = rmContainer.getContainerId();
// Inform the container
@@ -622,6 +627,33 @@ public abstract class AbstractYarnScheduler
recoverResourceRequestForContainer(rmContainer);
}
+ // Optimization:
+ // Check if there are in-flight container updates and complete the
+ // associated temp containers. These are removed when the app completes,
+ // but removing them when the actual container completes would allow the
+ // scheduler to reallocate those resources sooner.
+ private void completeOustandingUpdatesWhichAreReserved(
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event) {
+ N schedulerNode = getSchedulerNode(rmContainer.getNodeId());
+ if (schedulerNode != null &&
+ schedulerNode.getReservedContainer() != null) {
+ RMContainer resContainer = schedulerNode.getReservedContainer();
+ if (resContainer.getReservedSchedulerKey() != null) {
+ ContainerId containerToUpdate = resContainer
+ .getReservedSchedulerKey().getContainerToUpdate();
+ if (containerToUpdate != null &&
+ containerToUpdate.equals(containerStatus.getContainerId())) {
+ completedContainerInternal(resContainer,
+ ContainerStatus.newInstance(resContainer.getContainerId(),
+ containerStatus.getState(), containerStatus
+ .getDiagnostics(),
+ containerStatus.getExitStatus()), event);
+ }
+ }
+ }
+ }
+
// clean up a completed container
protected abstract void completedContainerInternal(RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event);
@@ -650,28 +682,6 @@ public abstract class AbstractYarnScheduler
}
}
- protected void decreaseContainers(
- List<UpdateContainerRequest> decreaseRequests,
- SchedulerApplicationAttempt attempt) {
- if (null == decreaseRequests || decreaseRequests.isEmpty()) {
- return;
- }
- // Pre-process decrease requests
- List<SchedContainerChangeRequest> schedDecreaseRequests =
- createSchedContainerChangeRequests(decreaseRequests, false);
- for (SchedContainerChangeRequest request : schedDecreaseRequests) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Processing decrease request:" + request);
- }
- // handle decrease request
- decreaseContainer(request, attempt);
- }
- }
-
- protected abstract void decreaseContainer(
- SchedContainerChangeRequest decreaseRequest,
- SchedulerApplicationAttempt attempt);
-
@Override
public N getSchedulerNode(NodeId nodeId) {
return nodeTracker.getNode(nodeId);
@@ -1074,21 +1084,39 @@ public abstract class AbstractYarnScheduler
}
}
- protected void handleExecutionTypeUpdates(
- SchedulerApplicationAttempt appAttempt,
- List<UpdateContainerRequest> promotionRequests,
- List<UpdateContainerRequest> demotionRequests) {
+ protected void handleContainerUpdates(
+ SchedulerApplicationAttempt appAttempt, ContainerUpdates updates) {
+ List<UpdateContainerRequest> promotionRequests =
+ updates.getPromotionRequests();
if (promotionRequests != null && !promotionRequests.isEmpty()) {
LOG.info("Promotion Update requests : " + promotionRequests);
- handlePromotionRequests(appAttempt, promotionRequests);
+ // Promotion is technically an increase request from
+ // 0 resources to target resources.
+ handleIncreaseRequests(appAttempt, promotionRequests);
}
+ List<UpdateContainerRequest> increaseRequests =
+ updates.getIncreaseRequests();
+ if (increaseRequests != null && !increaseRequests.isEmpty()) {
+ LOG.info("Resource increase requests : " + increaseRequests);
+ handleIncreaseRequests(appAttempt, increaseRequests);
+ }
+ List<UpdateContainerRequest> demotionRequests =
+ updates.getDemotionRequests();
if (demotionRequests != null && !demotionRequests.isEmpty()) {
LOG.info("Demotion Update requests : " + demotionRequests);
- handleDemotionRequests(appAttempt, demotionRequests);
+ // Demotion is technically a decrease request from initial
+ // to 0 resources
+ handleDecreaseRequests(appAttempt, demotionRequests);
+ }
+ List<UpdateContainerRequest> decreaseRequests =
+ updates.getDecreaseRequests();
+ if (decreaseRequests != null && !decreaseRequests.isEmpty()) {
+ LOG.info("Resource decrease requests : " + decreaseRequests);
+ handleDecreaseRequests(appAttempt, decreaseRequests);
}
}
- private void handlePromotionRequests(
+ private void handleIncreaseRequests(
SchedulerApplicationAttempt applicationAttempt,
List<UpdateContainerRequest> updateContainerRequests) {
for (UpdateContainerRequest uReq : updateContainerRequests) {
@@ -1118,7 +1146,7 @@ public abstract class AbstractYarnScheduler
}
}
- private void handleDemotionRequests(SchedulerApplicationAttempt appAttempt,
+ private void handleDecreaseRequests(SchedulerApplicationAttempt appAttempt,
List<UpdateContainerRequest> demotionRequests) {
OpportunisticContainerContext oppCntxt =
appAttempt.getOpportunisticContainerContext();
@@ -1126,24 +1154,59 @@ public abstract class AbstractYarnScheduler
RMContainer rmContainer =
rmContext.getScheduler().getRMContainer(uReq.getContainerId());
if (rmContainer != null) {
- if (appAttempt.getUpdateContext().checkAndAddToOutstandingDecreases(
- rmContainer.getContainer())) {
- RMContainer demotedRMContainer =
- createDemotedRMContainer(appAttempt, oppCntxt, rmContainer);
- appAttempt.addToNewlyDemotedContainers(
- uReq.getContainerId(), demotedRMContainer);
+ SchedulerNode schedulerNode = rmContext.getScheduler()
+ .getSchedulerNode(rmContainer.getContainer().getNodeId());
+ if (appAttempt.getUpdateContext()
+ .checkAndAddToOutstandingDecreases(uReq, schedulerNode,
+ rmContainer.getContainer())) {
+ if (ContainerUpdateType.DEMOTE_EXECUTION_TYPE ==
+ uReq.getContainerUpdateType()) {
+ RMContainer demotedRMContainer =
+ createDemotedRMContainer(appAttempt, oppCntxt, rmContainer);
+ appAttempt.addToNewlyDemotedContainers(
+ uReq.getContainerId(), demotedRMContainer);
+ } else {
+ RMContainer demotedRMContainer = createDecreasedRMContainer(
+ appAttempt, uReq, rmContainer);
+ appAttempt.addToNewlyDecreasedContainers(
+ uReq.getContainerId(), demotedRMContainer);
+ }
} else {
appAttempt.addToUpdateContainerErrors(
UpdateContainerError.newInstance(
RMServerUtils.UPDATE_OUTSTANDING_ERROR, uReq));
}
} else {
- LOG.warn("Cannot demote non-existent (or completed) Container ["
- + uReq.getContainerId() + "]");
+ LOG.warn("Cannot demote/decrease non-existent (or completed) " +
+ "Container [" + uReq.getContainerId() + "]");
}
}
}
+ private RMContainer createDecreasedRMContainer(
+ SchedulerApplicationAttempt appAttempt, UpdateContainerRequest uReq,
+ RMContainer rmContainer) {
+ SchedulerRequestKey sk =
+ SchedulerRequestKey.extractFrom(rmContainer.getContainer());
+ Container decreasedContainer = BuilderUtils.newContainer(
+ ContainerId.newContainerId(appAttempt.getApplicationAttemptId(),
+ appAttempt.getNewContainerId()),
+ rmContainer.getContainer().getNodeId(),
+ rmContainer.getContainer().getNodeHttpAddress(),
+ Resources.none(),
+ sk.getPriority(), null, rmContainer.getExecutionType(),
+ sk.getAllocationRequestId());
+ decreasedContainer.setVersion(rmContainer.getContainer().getVersion());
+ RMContainer newRmContainer = new RMContainerImpl(decreasedContainer,
+ sk, appAttempt.getApplicationAttemptId(),
+ decreasedContainer.getNodeId(), appAttempt.getUser(), rmContext,
+ rmContainer.isRemotelyAllocated());
+ appAttempt.addRMContainer(decreasedContainer.getId(), rmContainer);
+ ((AbstractYarnScheduler) rmContext.getScheduler()).getNode(
+ decreasedContainer.getNodeId()).allocateContainer(newRmContainer);
+ return newRmContainer;
+ }
+
private RMContainer createDemotedRMContainer(
SchedulerApplicationAttempt appAttempt,
OpportunisticContainerContext oppCntxt,
@@ -1162,4 +1225,36 @@ public abstract class AbstractYarnScheduler
return SchedulerUtils.createOpportunisticRmContainer(
rmContext, demotedContainer, false);
}
+
+ /**
+ * Rollback container update after expiry.
+ * @param containerId ContainerId.
+ */
+ protected void rollbackContainerUpdate(
+ ContainerId containerId) {
+ RMContainer rmContainer = getRMContainer(containerId);
+ if (rmContainer == null) {
+ LOG.info("Cannot rollback resource for container " + containerId
+ + ". The container does not exist.");
+ return;
+ }
+ T app = getCurrentAttemptForContainer(containerId);
+ if (getCurrentAttemptForContainer(containerId) == null) {
+ LOG.info("Cannot rollback resource for container " + containerId
+ + ". The application that the container "
+ + "belongs to does not exist.");
+ return;
+ }
+
+ if (Resources.fitsIn(rmContainer.getLastConfirmedResource(),
+ rmContainer.getContainer().getResource())) {
+ LOG.info("Roll back resource for container " + containerId);
+ handleDecreaseRequests(app, Arrays.asList(
+ UpdateContainerRequest.newInstance(
+ rmContainer.getContainer().getVersion(),
+ rmContainer.getContainerId(),
+ ContainerUpdateType.DECREASE_RESOURCE,
+ rmContainer.getLastConfirmedResource(), null)));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 48ecd2e..bff9c41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -90,9 +90,7 @@ public class AppSchedulingInfo {
schedulerKeys = new ConcurrentSkipListMap<>();
final Map<SchedulerRequestKey, SchedulingPlacementSet<SchedulerNode>>
schedulerKeyToPlacementSets = new ConcurrentHashMap<>();
- final Map<NodeId, Map<SchedulerRequestKey, Map<ContainerId,
- SchedContainerChangeRequest>>> containerIncreaseRequestMap =
- new ConcurrentHashMap<>();
+
private final ReentrantReadWriteLock.ReadLock readLock;
private final ReentrantReadWriteLock.WriteLock writeLock;
@@ -158,137 +156,6 @@ public class AppSchedulingInfo {
LOG.info("Application " + applicationId + " requests cleared");
}
- public boolean hasIncreaseRequest(NodeId nodeId) {
- try {
- this.readLock.lock();
- Map<SchedulerRequestKey, Map<ContainerId, SchedContainerChangeRequest>>
- requestsOnNode = containerIncreaseRequestMap.get(nodeId);
- return requestsOnNode == null ? false : requestsOnNode.size() > 0;
- } finally {
- this.readLock.unlock();
- }
- }
-
- public Map<ContainerId, SchedContainerChangeRequest>
- getIncreaseRequests(NodeId nodeId, SchedulerRequestKey schedulerKey) {
- try {
- this.readLock.lock();
- Map<SchedulerRequestKey, Map<ContainerId, SchedContainerChangeRequest>>
- requestsOnNode = containerIncreaseRequestMap.get(nodeId);
- return requestsOnNode == null ? null : requestsOnNode.get(
- schedulerKey);
- } finally {
- this.readLock.unlock();
- }
- }
-
- /**
- * return true if any of the existing increase requests are updated,
- * false if none of them are updated
- */
- public boolean updateIncreaseRequests(
- List<SchedContainerChangeRequest> increaseRequests) {
- boolean resourceUpdated = false;
-
- try {
- this.writeLock.lock();
- for (SchedContainerChangeRequest r : increaseRequests) {
- if (r.getRMContainer().getState() != RMContainerState.RUNNING) {
- LOG.warn("rmContainer's state is not RUNNING, for increase request"
- + " with container-id=" + r.getContainerId());
- continue;
- }
- try {
- RMServerUtils.checkSchedContainerChangeRequest(r, true);
- } catch (YarnException e) {
- LOG.warn("Error happens when checking increase request, Ignoring.."
- + " exception=", e);
- continue;
- }
- NodeId nodeId = r.getRMContainer().getAllocatedNode();
-
- Map<SchedulerRequestKey, Map<ContainerId, SchedContainerChangeRequest>>
- requestsOnNode = containerIncreaseRequestMap.get(nodeId);
- if (null == requestsOnNode) {
- requestsOnNode = new TreeMap<>();
- containerIncreaseRequestMap.put(nodeId, requestsOnNode);
- }
-
- SchedContainerChangeRequest prevChangeRequest =
- getIncreaseRequest(nodeId,
- r.getRMContainer().getAllocatedSchedulerKey(),
- r.getContainerId());
- if (null != prevChangeRequest) {
- if (Resources.equals(prevChangeRequest.getTargetCapacity(),
- r.getTargetCapacity())) {
- // increase request hasn't changed
- continue;
- }
-
- // remove the old one, as we will use the new one going forward
- removeIncreaseRequest(nodeId,
- prevChangeRequest.getRMContainer().getAllocatedSchedulerKey(),
- prevChangeRequest.getContainerId());
- }
-
- if (Resources.equals(r.getTargetCapacity(),
- r.getRMContainer().getAllocatedResource())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Trying to increase container " + r.getContainerId()
- + ", target capacity = previous capacity = " + prevChangeRequest
- + ". Will ignore this increase request.");
- }
- continue;
- }
-
- // add the new one
- resourceUpdated = true;
- insertIncreaseRequest(r);
- }
- return resourceUpdated;
- } finally {
- this.writeLock.unlock();
- }
- }
-
- /**
- * Insert increase request, adding any missing items in the data-structure
- * hierarchy.
- */
- private void insertIncreaseRequest(SchedContainerChangeRequest request) {
- NodeId nodeId = request.getNodeId();
- SchedulerRequestKey schedulerKey =
- request.getRMContainer().getAllocatedSchedulerKey();
- ContainerId containerId = request.getContainerId();
-
- Map<SchedulerRequestKey, Map<ContainerId, SchedContainerChangeRequest>>
- requestsOnNode = containerIncreaseRequestMap.get(nodeId);
- if (null == requestsOnNode) {
- requestsOnNode = new HashMap<>();
- containerIncreaseRequestMap.put(nodeId, requestsOnNode);
- }
-
- Map<ContainerId, SchedContainerChangeRequest> requestsOnNodeWithPriority =
- requestsOnNode.get(schedulerKey);
- if (null == requestsOnNodeWithPriority) {
- requestsOnNodeWithPriority = new TreeMap<>();
- requestsOnNode.put(schedulerKey, requestsOnNodeWithPriority);
- incrementSchedulerKeyReference(schedulerKey);
- }
-
- requestsOnNodeWithPriority.put(containerId, request);
-
- // update resources
- String partition = request.getRMContainer().getNodeLabelExpression();
- Resource delta = request.getDeltaCapacity();
- appResourceUsage.incPending(partition, delta);
- queue.incPendingResource(partition, delta);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Added increase request:" + request.getContainerId()
- + " delta=" + delta);
- }
- }
private void incrementSchedulerKeyReference(
SchedulerRequestKey schedulerKey) {
@@ -312,73 +179,6 @@ public class AppSchedulingInfo {
}
}
- public boolean removeIncreaseRequest(NodeId nodeId,
- SchedulerRequestKey schedulerKey, ContainerId containerId) {
- try {
- this.writeLock.lock();
- Map<SchedulerRequestKey, Map<ContainerId, SchedContainerChangeRequest>>
- requestsOnNode = containerIncreaseRequestMap.get(nodeId);
- if (null == requestsOnNode) {
- return false;
- }
-
- Map<ContainerId, SchedContainerChangeRequest> requestsOnNodeWithPriority =
- requestsOnNode.get(schedulerKey);
- if (null == requestsOnNodeWithPriority) {
- return false;
- }
-
- SchedContainerChangeRequest request =
- requestsOnNodeWithPriority.remove(containerId);
-
- // remove hierarchies if it becomes empty
- if (requestsOnNodeWithPriority.isEmpty()) {
- requestsOnNode.remove(schedulerKey);
- decrementSchedulerKeyReference(schedulerKey);
- }
- if (requestsOnNode.isEmpty()) {
- containerIncreaseRequestMap.remove(nodeId);
- }
-
- if (request == null) {
- return false;
- }
-
- // update queue's pending resource if request exists
- String partition = request.getRMContainer().getNodeLabelExpression();
- Resource delta = request.getDeltaCapacity();
- appResourceUsage.decPending(partition, delta);
- queue.decPendingResource(partition, delta);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("remove increase request:" + request);
- }
-
- return true;
- } finally {
- this.writeLock.unlock();
- }
- }
-
- public SchedContainerChangeRequest getIncreaseRequest(NodeId nodeId,
- SchedulerRequestKey schedulerKey, ContainerId containerId) {
- try {
- this.readLock.lock();
- Map<SchedulerRequestKey, Map<ContainerId, SchedContainerChangeRequest>>
- requestsOnNode = containerIncreaseRequestMap.get(nodeId);
- if (null == requestsOnNode) {
- return null;
- }
-
- Map<ContainerId, SchedContainerChangeRequest> requestsOnNodeWithPriority =
- requestsOnNode.get(schedulerKey);
- return requestsOnNodeWithPriority == null ? null
- : requestsOnNodeWithPriority.get(containerId);
- } finally {
- this.readLock.unlock();
- }
- }
-
public ContainerUpdateContext getUpdateContext() {
return updateContext;
}
@@ -514,21 +314,6 @@ public class AppSchedulingInfo {
appResourceUsage.decPending(partition, toDecrease);
}
- private boolean hasRequestLabelChanged(ResourceRequest requestOne,
- ResourceRequest requestTwo) {
- String requestOneLabelExp = requestOne.getNodeLabelExpression();
- String requestTwoLabelExp = requestTwo.getNodeLabelExpression();
- // First request label expression can be null and second request
- // is not null then we have to consider it as changed.
- if ((null == requestOneLabelExp) && (null != requestTwoLabelExp)) {
- return true;
- }
- // If the label is not matching between both request when
- // requestOneLabelExp is not null.
- return ((null != requestOneLabelExp) && !(requestOneLabelExp
- .equals(requestTwoLabelExp)));
- }
-
/**
* The ApplicationMaster is updating the placesBlacklistedByApp used for
* containers other than AMs.
@@ -601,22 +386,6 @@ public class AppSchedulingInfo {
return ret;
}
- public SchedulingPlacementSet getFirstSchedulingPlacementSet() {
- try {
- readLock.lock();
- for (SchedulerRequestKey key : schedulerKeys.keySet()) {
- SchedulingPlacementSet ps = schedulerKeyToPlacementSets.get(key);
- if (null != ps) {
- return ps;
- }
- }
- return null;
- } finally {
- readLock.unlock();
- }
-
- }
-
public PendingAsk getNextPendingAsk() {
try {
readLock.lock();
@@ -666,56 +435,6 @@ public class AppSchedulingInfo {
}
}
- public void increaseContainer(SchedContainerChangeRequest increaseRequest) {
- NodeId nodeId = increaseRequest.getNodeId();
- SchedulerRequestKey schedulerKey =
- increaseRequest.getRMContainer().getAllocatedSchedulerKey();
- ContainerId containerId = increaseRequest.getContainerId();
- Resource deltaCapacity = increaseRequest.getDeltaCapacity();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("allocated increase request : applicationId=" + applicationId
- + " container=" + containerId + " host="
- + increaseRequest.getNodeId() + " user=" + user + " resource="
- + deltaCapacity);
- }
- try {
- this.writeLock.lock();
- // Set queue metrics
- queue.getMetrics().allocateResources(user, deltaCapacity);
- // remove the increase request from pending increase request map
- removeIncreaseRequest(nodeId, schedulerKey, containerId);
- // update usage
- appResourceUsage.incUsed(increaseRequest.getNodePartition(),
- deltaCapacity);
- } finally {
- this.writeLock.unlock();
- }
- }
-
- public void decreaseContainer(SchedContainerChangeRequest decreaseRequest) {
- // Delta is negative when it's a decrease request
- Resource absDelta = Resources.negate(decreaseRequest.getDeltaCapacity());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Decrease container : applicationId=" + applicationId
- + " container=" + decreaseRequest.getContainerId() + " host="
- + decreaseRequest.getNodeId() + " user=" + user + " resource="
- + absDelta);
- }
-
- try {
- this.writeLock.lock();
- // Set queue metrics
- queue.getMetrics().releaseResources(user, absDelta);
-
- // update usage
- appResourceUsage.decUsed(decreaseRequest.getNodePartition(), absDelta);
- } finally {
- this.writeLock.unlock();
- }
- }
-
public List<ResourceRequest> allocate(NodeType type,
SchedulerNode node, SchedulerRequestKey schedulerKey,
Container containerAllocated) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
index 7381250..5ac2ac5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
@@ -28,17 +28,19 @@ import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer
+ .RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement
+ .SchedulingPlacementSet;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+import org.apache.hadoop.yarn.util.resource.Resources;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -58,43 +60,37 @@ public class ContainerUpdateContext {
private final Map<SchedulerRequestKey, Map<Resource,
Map<NodeId, Set<ContainerId>>>> outstandingIncreases = new HashMap<>();
- private final Set<ContainerId> outstandingDecreases = new HashSet<>();
+ private final Map<ContainerId, Resource> outstandingDecreases =
+ new HashMap<>();
private final AppSchedulingInfo appSchedulingInfo;
ContainerUpdateContext(AppSchedulingInfo appSchedulingInfo) {
this.appSchedulingInfo = appSchedulingInfo;
}
- private synchronized boolean isBeingIncreased(Container container) {
- Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap =
- outstandingIncreases.get(
- new SchedulerRequestKey(container.getPriority(),
- container.getAllocationRequestId(), container.getId()));
- if (resourceMap != null) {
- Map<NodeId, Set<ContainerId>> locationMap =
- resourceMap.get(container.getResource());
- if (locationMap != null) {
- Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
- if (containerIds != null && !containerIds.isEmpty()) {
- return containerIds.contains(container.getId());
- }
- }
- }
- return false;
- }
-
/**
* Add the container to outstanding decreases.
+ * @param updateReq UpdateContainerRequest.
+ * @param schedulerNode SchedulerNode.
* @param container Container.
- * @return true if updated to outstanding decreases was successful.
+ * @return If it was possible to decrease the container.
*/
public synchronized boolean checkAndAddToOutstandingDecreases(
+ UpdateContainerRequest updateReq, SchedulerNode schedulerNode,
Container container) {
- if (isBeingIncreased(container)
- || outstandingDecreases.contains(container.getId())) {
+ if (outstandingDecreases.containsKey(container.getId())) {
return false;
}
- outstandingDecreases.add(container.getId());
+ if (ContainerUpdateType.DECREASE_RESOURCE ==
+ updateReq.getContainerUpdateType()) {
+ SchedulerRequestKey updateKey = new SchedulerRequestKey
+ (container.getPriority(),
+ container.getAllocationRequestId(), container.getId());
+ cancelPreviousRequest(schedulerNode, updateKey);
+ outstandingDecreases.put(container.getId(), updateReq.getCapability());
+ } else {
+ outstandingDecreases.put(container.getId(), container.getResource());
+ }
return true;
}
@@ -117,35 +113,63 @@ public class ContainerUpdateContext {
if (resourceMap == null) {
resourceMap = new HashMap<>();
outstandingIncreases.put(schedulerKey, resourceMap);
+ } else {
+ // Updating Resource for and existing increase container
+ if (ContainerUpdateType.INCREASE_RESOURCE ==
+ updateRequest.getContainerUpdateType()) {
+ cancelPreviousRequest(schedulerNode, schedulerKey);
+ } else {
+ return false;
+ }
}
+ Resource resToIncrease = getResourceToIncrease(updateRequest, rmContainer);
Map<NodeId, Set<ContainerId>> locationMap =
- resourceMap.get(container.getResource());
+ resourceMap.get(resToIncrease);
if (locationMap == null) {
locationMap = new HashMap<>();
- resourceMap.put(container.getResource(), locationMap);
+ resourceMap.put(resToIncrease, locationMap);
}
Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
if (containerIds == null) {
containerIds = new HashSet<>();
locationMap.put(container.getNodeId(), containerIds);
}
- if (containerIds.contains(container.getId())
- || outstandingDecreases.contains(container.getId())) {
+ if (outstandingDecreases.containsKey(container.getId())) {
return false;
}
- containerIds.add(container.getId());
- Map<SchedulerRequestKey, Map<String, ResourceRequest>> updateResReqs =
- new HashMap<>();
- Resource resToIncrease = getResourceToIncrease(updateRequest, rmContainer);
- Map<String, ResourceRequest> resMap =
- createResourceRequests(rmContainer, schedulerNode,
- schedulerKey, resToIncrease);
- updateResReqs.put(schedulerKey, resMap);
- appSchedulingInfo.addToPlacementSets(false, updateResReqs);
+ containerIds.add(container.getId());
+ if (!Resources.isNone(resToIncrease)) {
+ Map<SchedulerRequestKey, Map<String, ResourceRequest>> updateResReqs =
+ new HashMap<>();
+ Map<String, ResourceRequest> resMap =
+ createResourceRequests(rmContainer, schedulerNode,
+ schedulerKey, resToIncrease);
+ updateResReqs.put(schedulerKey, resMap);
+ appSchedulingInfo.addToPlacementSets(false, updateResReqs);
+ }
return true;
}
+ private void cancelPreviousRequest(SchedulerNode schedulerNode,
+ SchedulerRequestKey schedulerKey) {
+ SchedulingPlacementSet<SchedulerNode> schedulingPlacementSet =
+ appSchedulingInfo.getSchedulingPlacementSet(schedulerKey);
+ if (schedulingPlacementSet != null) {
+ Map<String, ResourceRequest> resourceRequests = schedulingPlacementSet
+ .getResourceRequests();
+ ResourceRequest prevReq = resourceRequests.get(ResourceRequest.ANY);
+ // Decrement the pending using a dummy RR with
+ // resource = prev update req capability
+ if (prevReq != null) {
+ appSchedulingInfo.allocate(NodeType.OFF_SWITCH, schedulerNode,
+ schedulerKey, Container.newInstance(UNDEFINED,
+ schedulerNode.getNodeID(), "host:port",
+ prevReq.getCapability(), schedulerKey.getPriority(), null));
+ }
+ }
+ }
+
private Map<String, ResourceRequest> createResourceRequests(
RMContainer rmContainer, SchedulerNode schedulerNode,
SchedulerRequestKey schedulerKey, Resource resToIncrease) {
@@ -171,10 +195,16 @@ public class ContainerUpdateContext {
ContainerUpdateType.PROMOTE_EXECUTION_TYPE) {
return rmContainer.getContainer().getResource();
}
- // TODO: Fix this for container increase..
- // This has to equal the Resources in excess of fitsIn()
- // for container increase and is equal to the container total
- // resource for Promotion.
+ if (updateReq.getContainerUpdateType() ==
+ ContainerUpdateType.INCREASE_RESOURCE) {
+ // This has to equal the Resources in excess of fitsIn()
+ // for container increase and is equal to the container total
+ // resource for Promotion.
+ Resource maxCap = Resources.componentwiseMax(updateReq.getCapability(),
+ rmContainer.getContainer().getResource());
+ return Resources.add(maxCap,
+ Resources.negate(rmContainer.getContainer().getResource()));
+ }
return null;
}
@@ -228,6 +258,7 @@ public class ContainerUpdateContext {
/**
* Check if a new container is to be matched up against an outstanding
* Container increase request.
+ * @param node SchedulerNode.
* @param schedulerKey SchedulerRequestKey.
* @param rmContainer RMContainer.
* @return ContainerId.
@@ -264,4 +295,80 @@ public class ContainerUpdateContext {
}
return retVal;
}
+
+ /**
+ * Swaps the existing RMContainer's and the temp RMContainers internal
+ * container references after adjusting the resources in each.
+ * @param tempRMContainer Temp RMContainer.
+ * @param existingRMContainer Existing RMContainer.
+ * @param updateType Update Type.
+ * @return Existing RMContainer after swapping the container references.
+ */
+ public RMContainer swapContainer(RMContainer tempRMContainer,
+ RMContainer existingRMContainer, ContainerUpdateType updateType) {
+ ContainerId matchedContainerId = existingRMContainer.getContainerId();
+ // Swap updated container with the existing container
+ Container tempContainer = tempRMContainer.getContainer();
+
+ Resource updatedResource = createUpdatedResource(
+ tempContainer, existingRMContainer.getContainer(), updateType);
+ Resource resourceToRelease = createResourceToRelease(
+ existingRMContainer.getContainer(), updateType);
+ Container newContainer = Container.newInstance(matchedContainerId,
+ existingRMContainer.getContainer().getNodeId(),
+ existingRMContainer.getContainer().getNodeHttpAddress(),
+ updatedResource,
+ existingRMContainer.getContainer().getPriority(), null,
+ tempContainer.getExecutionType());
+ newContainer.setAllocationRequestId(
+ existingRMContainer.getContainer().getAllocationRequestId());
+ newContainer.setVersion(existingRMContainer.getContainer().getVersion());
+
+ tempRMContainer.getContainer().setResource(resourceToRelease);
+ tempRMContainer.getContainer().setExecutionType(
+ existingRMContainer.getContainer().getExecutionType());
+
+ ((RMContainerImpl)existingRMContainer).setContainer(newContainer);
+ return existingRMContainer;
+ }
+
+ /**
+ * Returns the resource that the container will finally be assigned with
+ * at the end of the update operation.
+ * @param tempContainer Temporary Container created for the operation.
+ * @param existingContainer Existing Container.
+ * @param updateType Update Type.
+ * @return Final Resource.
+ */
+ private Resource createUpdatedResource(Container tempContainer,
+ Container existingContainer, ContainerUpdateType updateType) {
+ if (ContainerUpdateType.INCREASE_RESOURCE == updateType) {
+ return Resources.add(existingContainer.getResource(),
+ tempContainer.getResource());
+ } else if (ContainerUpdateType.DECREASE_RESOURCE == updateType) {
+ return outstandingDecreases.get(existingContainer.getId());
+ } else {
+ return existingContainer.getResource();
+ }
+ }
+
+ /**
+ * Returns the resources that need to be released at the end of the update
+ * operation.
+ * @param existingContainer Existing Container.
+ * @param updateType Updated type.
+ * @return Resources to be released.
+ */
+ private Resource createResourceToRelease(Container existingContainer,
+ ContainerUpdateType updateType) {
+ if (ContainerUpdateType.INCREASE_RESOURCE == updateType) {
+ return Resources.none();
+ } else if (ContainerUpdateType.DECREASE_RESOURCE == updateType){
+ return Resources.add(existingContainer.getResource(),
+ Resources.negate(
+ outstandingDecreases.get(existingContainer.getId())));
+ } else {
+ return existingContainer.getResource();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 0e79838..f894a40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -65,7 +66,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppR
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerChangeResourceEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
@@ -73,6 +73,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRese
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerUpdatesAcquiredEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
+
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode
+ .RMNodeDecreaseContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
@@ -136,9 +139,9 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private AtomicLong firstContainerAllocatedTime = new AtomicLong(0);
protected List<RMContainer> newlyAllocatedContainers = new ArrayList<>();
+ protected List<RMContainer> tempContainerToKill = new ArrayList<>();
protected Map<ContainerId, RMContainer> newlyPromotedContainers = new HashMap<>();
protected Map<ContainerId, RMContainer> newlyDemotedContainers = new HashMap<>();
- protected List<RMContainer> tempContainerToKill = new ArrayList<>();
protected Map<ContainerId, RMContainer> newlyDecreasedContainers = new HashMap<>();
protected Map<ContainerId, RMContainer> newlyIncreasedContainers = new HashMap<>();
protected Set<NMToken> updatedNMTokens = new HashSet<>();
@@ -670,6 +673,11 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
rmContainer.handle(new RMContainerUpdatesAcquiredEvent(
rmContainer.getContainerId(),
ContainerUpdateType.INCREASE_RESOURCE == updateType));
+ if (ContainerUpdateType.DECREASE_RESOURCE == updateType) {
+ this.rmContext.getDispatcher().getEventHandler().handle(
+ new RMNodeDecreaseContainerEvent(rmContainer.getNodeId(),
+ Collections.singletonList(rmContainer.getContainer())));
+ }
}
return container;
}
@@ -717,11 +725,16 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
}
}
- public void addToNewlyDemotedContainers(ContainerId containerId,
+ public synchronized void addToNewlyDemotedContainers(ContainerId containerId,
RMContainer rmContainer) {
newlyDemotedContainers.put(containerId, rmContainer);
}
+ public synchronized void addToNewlyDecreasedContainers(
+ ContainerId containerId, RMContainer rmContainer) {
+ newlyDecreasedContainers.put(containerId, rmContainer);
+ }
+
protected synchronized void addToUpdateContainerErrors(
UpdateContainerError error) {
updateContainerErrors.add(error);
@@ -729,10 +742,6 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
protected synchronized void addToNewlyAllocatedContainers(
SchedulerNode node, RMContainer rmContainer) {
- if (oppContainerContext == null) {
- newlyAllocatedContainers.add(rmContainer);
- return;
- }
ContainerId matchedContainerId =
getUpdateContext().matchContainerToOutstandingIncreaseReq(
node, rmContainer.getAllocatedSchedulerKey(), rmContainer);
@@ -745,7 +754,21 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
// occurs when using MiniYARNCluster to test).
tempContainerToKill.add(rmContainer);
} else {
- newlyPromotedContainers.put(matchedContainerId, rmContainer);
+ RMContainer existingContainer = getRMContainer(matchedContainerId);
+ // If this container was already GUARANTEED, then it is an
+ // increase, else its a promotion
+ if (existingContainer == null ||
+ EnumSet.of(RMContainerState.COMPLETED, RMContainerState.KILLED,
+ RMContainerState.EXPIRED, RMContainerState.RELEASED).contains(
+ existingContainer.getState())) {
+ tempContainerToKill.add(rmContainer);
+ } else {
+ if (ExecutionType.GUARANTEED == existingContainer.getExecutionType()) {
+ newlyIncreasedContainers.put(matchedContainerId, rmContainer);
+ } else {
+ newlyPromotedContainers.put(matchedContainerId, rmContainer);
+ }
+ }
}
} else {
newlyAllocatedContainers.add(rmContainer);
@@ -753,15 +776,25 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
}
public List<Container> pullNewlyPromotedContainers() {
- return pullContainersWithUpdatedExecType(newlyPromotedContainers,
+ return pullNewlyUpdatedContainers(newlyPromotedContainers,
ContainerUpdateType.PROMOTE_EXECUTION_TYPE);
}
public List<Container> pullNewlyDemotedContainers() {
- return pullContainersWithUpdatedExecType(newlyDemotedContainers,
+ return pullNewlyUpdatedContainers(newlyDemotedContainers,
ContainerUpdateType.DEMOTE_EXECUTION_TYPE);
}
+ public List<Container> pullNewlyIncreasedContainers() {
+ return pullNewlyUpdatedContainers(newlyIncreasedContainers,
+ ContainerUpdateType.INCREASE_RESOURCE);
+ }
+
+ public List<Container> pullNewlyDecreasedContainers() {
+ return pullNewlyUpdatedContainers(newlyDecreasedContainers,
+ ContainerUpdateType.DECREASE_RESOURCE);
+ }
+
public List<UpdateContainerError> pullUpdateContainerErrors() {
List<UpdateContainerError> errors =
new ArrayList<>(updateContainerErrors);
@@ -775,11 +808,13 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
* GUARANTEED to OPPORTUNISTIC.
* @return Newly Promoted and Demoted containers
*/
- private List<Container> pullContainersWithUpdatedExecType(
+ private List<Container> pullNewlyUpdatedContainers(
Map<ContainerId, RMContainer> newlyUpdatedContainers,
ContainerUpdateType updateTpe) {
List<Container> updatedContainers = new ArrayList<>();
- if (oppContainerContext == null) {
+ if (oppContainerContext == null &&
+ (ContainerUpdateType.DEMOTE_EXECUTION_TYPE == updateTpe
+ || ContainerUpdateType.PROMOTE_EXECUTION_TYPE == updateTpe)) {
return updatedContainers;
}
try {
@@ -789,19 +824,22 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
while (i.hasNext()) {
Map.Entry<ContainerId, RMContainer> entry = i.next();
ContainerId matchedContainerId = entry.getKey();
- RMContainer rmContainer = entry.getValue();
-
- // swap containers
- RMContainer existingRMContainer = swapContainer(
- rmContainer, matchedContainerId);
- getUpdateContext().removeFromOutstandingUpdate(
- rmContainer.getAllocatedSchedulerKey(),
- existingRMContainer.getContainer());
- Container updatedContainer = updateContainerAndNMToken(
- existingRMContainer, updateTpe);
- updatedContainers.add(updatedContainer);
-
- tempContainerToKill.add(rmContainer);
+ RMContainer tempRMContainer = entry.getValue();
+
+ RMContainer existingRMContainer =
+ getRMContainer(matchedContainerId);
+ if (existingRMContainer != null) {
+ // swap containers
+ existingRMContainer = getUpdateContext().swapContainer(
+ tempRMContainer, existingRMContainer, updateTpe);
+ getUpdateContext().removeFromOutstandingUpdate(
+ tempRMContainer.getAllocatedSchedulerKey(),
+ existingRMContainer.getContainer());
+ Container updatedContainer = updateContainerAndNMToken(
+ existingRMContainer, updateTpe);
+ updatedContainers.add(updatedContainer);
+ }
+ tempContainerToKill.add(tempRMContainer);
i.remove();
}
// Release all temporary containers
@@ -823,68 +861,6 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
}
}
- private RMContainer swapContainer(RMContainer rmContainer, ContainerId
- matchedContainerId) {
- RMContainer existingRMContainer =
- getRMContainer(matchedContainerId);
- if (existingRMContainer != null) {
- // Swap updated container with the existing container
- Container updatedContainer = rmContainer.getContainer();
-
- Container newContainer = Container.newInstance(matchedContainerId,
- existingRMContainer.getContainer().getNodeId(),
- existingRMContainer.getContainer().getNodeHttpAddress(),
- updatedContainer.getResource(),
- existingRMContainer.getContainer().getPriority(), null,
- updatedContainer.getExecutionType());
- newContainer.setAllocationRequestId(
- existingRMContainer.getContainer().getAllocationRequestId());
- newContainer.setVersion(existingRMContainer.getContainer().getVersion());
-
- rmContainer.getContainer().setResource(
- existingRMContainer.getContainer().getResource());
- rmContainer.getContainer().setExecutionType(
- existingRMContainer.getContainer().getExecutionType());
-
- ((RMContainerImpl)existingRMContainer).setContainer(newContainer);
- }
- return existingRMContainer;
- }
-
- private List<Container> pullNewlyUpdatedContainers(
- Map<ContainerId, RMContainer> updatedContainerMap, boolean increase) {
- try {
- writeLock.lock();
- List <Container> returnContainerList = new ArrayList <Container>(
- updatedContainerMap.size());
-
- Iterator<Entry<ContainerId, RMContainer>> i =
- updatedContainerMap.entrySet().iterator();
- while (i.hasNext()) {
- RMContainer rmContainer = i.next().getValue();
- Container updatedContainer = updateContainerAndNMToken(rmContainer,
- increase ? ContainerUpdateType.INCREASE_RESOURCE :
- ContainerUpdateType.DECREASE_RESOURCE);
- if (updatedContainer != null) {
- returnContainerList.add(updatedContainer);
- i.remove();
- }
- }
- return returnContainerList;
- } finally {
- writeLock.unlock();
- }
-
- }
-
- public List<Container> pullNewlyIncreasedContainers() {
- return pullNewlyUpdatedContainers(newlyIncreasedContainers, true);
- }
-
- public List<Container> pullNewlyDecreasedContainers() {
- return pullNewlyUpdatedContainers(newlyDecreasedContainers, false);
- }
-
public List<NMToken> pullUpdatedNMTokens() {
try {
writeLock.lock();
@@ -1252,68 +1228,6 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
public ResourceUsage getSchedulingResourceUsage() {
return attemptResourceUsage;
}
-
- public boolean removeIncreaseRequest(NodeId nodeId,
- SchedulerRequestKey schedulerKey, ContainerId containerId) {
- try {
- writeLock.lock();
- return appSchedulingInfo.removeIncreaseRequest(nodeId, schedulerKey,
- containerId);
- } finally {
- writeLock.unlock();
- }
- }
-
- public boolean updateIncreaseRequests(
- List<SchedContainerChangeRequest> increaseRequests) {
- try {
- writeLock.lock();
- return appSchedulingInfo.updateIncreaseRequests(increaseRequests);
- } finally {
- writeLock.unlock();
- }
- }
-
- private void changeContainerResource(
- SchedContainerChangeRequest changeRequest, boolean increase) {
- try {
- writeLock.lock();
- if (increase) {
- appSchedulingInfo.increaseContainer(changeRequest);
- } else{
- appSchedulingInfo.decreaseContainer(changeRequest);
- }
-
- RMContainer changedRMContainer = changeRequest.getRMContainer();
- changedRMContainer.handle(
- new RMContainerChangeResourceEvent(changeRequest.getContainerId(),
- changeRequest.getTargetCapacity(), increase));
-
- // remove pending and not pulled by AM newly-increased or
- // decreased-containers and add the new one
- if (increase) {
- newlyDecreasedContainers.remove(changeRequest.getContainerId());
- newlyIncreasedContainers.put(changeRequest.getContainerId(),
- changedRMContainer);
- } else{
- newlyIncreasedContainers.remove(changeRequest.getContainerId());
- newlyDecreasedContainers.put(changeRequest.getContainerId(),
- changedRMContainer);
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- public void decreaseContainer(
- SchedContainerChangeRequest decreaseRequest) {
- changeContainerResource(decreaseRequest, false);
- }
-
- public void increaseContainer(
- SchedContainerChangeRequest increaseRequest) {
- changeContainerResource(increaseRequest, true);
- }
public void setAppAMNodePartitionName(String partitionName) {
this.appAMNodePartitionName = partitionName;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 9c2dff3..db17b42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -180,49 +180,6 @@ public abstract class SchedulerNode {
}
/**
- * Change the resources allocated for a container.
- * @param containerId Identifier of the container to change.
- * @param deltaResource Change in the resource allocation.
- * @param increase True if the change is an increase of allocation.
- */
- protected synchronized void changeContainerResource(ContainerId containerId,
- Resource deltaResource, boolean increase) {
- if (increase) {
- deductUnallocatedResource(deltaResource);
- } else {
- addUnallocatedResource(deltaResource);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug((increase ? "Increased" : "Decreased") + " container "
- + containerId + " of capacity " + deltaResource + " on host "
- + rmNode.getNodeAddress() + ", which has " + numContainers
- + " containers, " + getAllocatedResource() + " used and "
- + getUnallocatedResource() + " available after allocation");
- }
- }
-
- /**
- * Increase the resources allocated to a container.
- * @param containerId Identifier of the container to change.
- * @param deltaResource Increase of resource allocation.
- */
- public synchronized void increaseContainer(ContainerId containerId,
- Resource deltaResource) {
- changeContainerResource(containerId, deltaResource, true);
- }
-
- /**
- * Decrease the resources allocated to a container.
- * @param containerId Identifier of the container to change.
- * @param deltaResource Decrease of resource allocation.
- */
- public synchronized void decreaseContainer(ContainerId containerId,
- Resource deltaResource) {
- changeContainerResource(containerId, deltaResource, false);
- }
-
- /**
* Get unallocated resources on the node.
* @return Unallocated resources on the node
*/
@@ -280,7 +237,6 @@ public abstract class SchedulerNode {
if (info == null) {
return;
}
-
if (!releasedByNode && info.launchedOnNode) {
// wait until node reports container has completed
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index e9ef319..aa60c9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -453,14 +453,13 @@ public abstract class AbstractCSQueue implements CSQueue {
}
void allocateResource(Resource clusterResource,
- Resource resource, String nodePartition, boolean changeContainerResource) {
+ Resource resource, String nodePartition) {
try {
writeLock.lock();
queueUsage.incUsed(nodePartition, resource);
- if (!changeContainerResource) {
- ++numContainers;
- }
+ ++numContainers;
+
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
minimumAllocation, this, labelManager, nodePartition);
} finally {
@@ -469,7 +468,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
protected void releaseResource(Resource clusterResource,
- Resource resource, String nodePartition, boolean changeContainerResource) {
+ Resource resource, String nodePartition) {
try {
writeLock.lock();
queueUsage.decUsed(nodePartition, resource);
@@ -477,9 +476,7 @@ public abstract class AbstractCSQueue implements CSQueue {
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
minimumAllocation, this, labelManager, nodePartition);
- if (!changeContainerResource) {
- --numContainers;
- }
+ --numContainers;
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index a65b3d2..6d30386 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -231,14 +231,6 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
boolean sortQueues);
/**
- * We have a reserved increased container in the queue, we need to unreserve
- * it. Since we just want to cancel the reserved increase request instead of
- * stop the container, we shouldn't call completedContainer for such purpose.
- */
- public void unreserveIncreasedContainer(Resource clusterResource,
- FiCaSchedulerApp app, FiCaSchedulerNode node, RMContainer rmContainer);
-
- /**
* Get the number of applications in the queue.
* @return number of applications
*/
@@ -333,13 +325,6 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
* new resource asked
*/
public void decPendingResource(String nodeLabel, Resource resourceToDec);
-
- /**
- * Decrease container resource in the queue
- */
- public void decreaseContainer(Resource clusterResource,
- SchedContainerChangeRequest decreaseRequest,
- FiCaSchedulerApp app) throws InvalidResourceRequestException;
/**
* Get valid Node Labels for this queue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 3517764..20ea607 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
@@ -60,9 +59,7 @@ import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
@@ -85,7 +82,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeDecreaseContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
@@ -99,7 +95,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -872,43 +868,6 @@ public class CapacityScheduler extends
}
}
- private LeafQueue updateIncreaseRequests(
- List<UpdateContainerRequest> increaseRequests, FiCaSchedulerApp app) {
- // When application has some pending to-be-removed resource requests,
- app.removedToBeRemovedIncreaseRequests();
-
- if (null == increaseRequests || increaseRequests.isEmpty()) {
- return null;
- }
-
- // Pre-process increase requests
- List<SchedContainerChangeRequest> schedIncreaseRequests =
- createSchedContainerChangeRequests(increaseRequests, true);
- LeafQueue leafQueue = (LeafQueue) app.getQueue();
-
- try {
- /*
- * Acquire application's lock here to make sure application won't
- * finish when updateIncreaseRequest is called.
- */
- app.getWriteLock().lock();
- // make sure we aren't stopping/removing the application
- // when the allocate comes in
- if (app.isStopped()) {
- return null;
- }
- // Process increase resource requests
- if (app.updateIncreaseRequests(schedIncreaseRequests)) {
- return leafQueue;
- }
- } finally {
- app.getWriteLock().unlock();
- }
-
-
- return null;
- }
-
@Override
@Lock(Lock.NoLock.class)
public Allocation allocate(ApplicationAttemptId applicationAttemptId,
@@ -920,21 +879,13 @@ public class CapacityScheduler extends
return EMPTY_ALLOCATION;
}
- // Handle promotions and demotions
- handleExecutionTypeUpdates(
- application, updateRequests.getPromotionRequests(),
- updateRequests.getDemotionRequests());
+ // Handle all container updates
+ handleContainerUpdates(application, updateRequests);
// Release containers
releaseContainers(release, application);
- // update increase requests
- LeafQueue updateDemandForQueue =
- updateIncreaseRequests(updateRequests.getIncreaseRequests(),
- application);
-
- // Decrease containers
- decreaseContainers(updateRequests.getDecreaseRequests(), application);
+ LeafQueue updateDemandForQueue = null;
// Sanity check for new allocation requests
normalizeRequests(ask);
@@ -959,8 +910,7 @@ public class CapacityScheduler extends
}
// Update application requests
- if (application.updateResourceRequests(ask) && (updateDemandForQueue
- == null)) {
+ if (application.updateResourceRequests(ask)) {
updateDemandForQueue = (LeafQueue) application.getQueue();
}
@@ -1466,7 +1416,7 @@ public class CapacityScheduler extends
(ContainerExpiredSchedulerEvent) event;
ContainerId containerId = containerExpiredEvent.getContainerId();
if (containerExpiredEvent.isIncrease()) {
- rollbackContainerResource(containerId);
+ rollbackContainerUpdate(containerId);
} else {
completedContainer(getRMContainer(containerId),
SchedulerUtils.createAbnormalContainerStatus(
@@ -1618,31 +1568,6 @@ public class CapacityScheduler extends
}
}
- private void rollbackContainerResource(
- ContainerId containerId) {
- RMContainer rmContainer = getRMContainer(containerId);
- if (rmContainer == null) {
- LOG.info("Cannot rollback resource for container " + containerId
- + ". The container does not exist.");
- return;
- }
- FiCaSchedulerApp application = getCurrentAttemptForContainer(containerId);
- if (application == null) {
- LOG.info("Cannot rollback resource for container " + containerId
- + ". The application that the container "
- + "belongs to does not exist.");
- return;
- }
- LOG.info("Roll back resource for container " + containerId);
-
- SchedulerNode schedulerNode = getSchedulerNode(
- rmContainer.getAllocatedNode());
- SchedContainerChangeRequest decreaseRequest =
- new SchedContainerChangeRequest(this.rmContext, schedulerNode,
- rmContainer, rmContainer.getLastConfirmedResource());
- decreaseContainer(decreaseRequest, application);
- }
-
@Override
protected void completedContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
@@ -1676,32 +1601,6 @@ public class CapacityScheduler extends
rmContainer, containerStatus, event, null, true);
}
- @Override
- protected void decreaseContainer(SchedContainerChangeRequest decreaseRequest,
- SchedulerApplicationAttempt attempt) {
- RMContainer rmContainer = decreaseRequest.getRMContainer();
- // Check container status before doing decrease
- if (rmContainer.getState() != RMContainerState.RUNNING) {
- LOG.info(
- "Trying to decrease a container not in RUNNING state, container="
- + rmContainer + " state=" + rmContainer.getState().name());
- return;
- }
- FiCaSchedulerApp app = (FiCaSchedulerApp) attempt;
- LeafQueue queue = (LeafQueue) attempt.getQueue();
- try {
- queue.decreaseContainer(getClusterResource(), decreaseRequest, app);
- // Notify RMNode that the container can be pulled by NodeManager in the
- // next heartbeat
- this.rmContext.getDispatcher().getEventHandler().handle(
- new RMNodeDecreaseContainerEvent(decreaseRequest.getNodeId(),
- Collections.singletonList(rmContainer.getContainer())));
- } catch (InvalidResourceRequestException e) {
- LOG.warn("Error happens when checking decrease request, Ignoring.."
- + " exception=", e);
- }
- }
-
@Lock(Lock.NoLock.class)
@VisibleForTesting
@Override
@@ -2386,8 +2285,8 @@ public class CapacityScheduler extends
getSchedulerContainer(rmContainer, true),
getSchedulerContainersToRelease(csAssignment),
getSchedulerContainer(csAssignment.getFulfilledReservedContainer(),
- false), csAssignment.isIncreasedAllocation(),
- csAssignment.getType(), csAssignment.getRequestLocalityType(),
+ false), csAssignment.getType(),
+ csAssignment.getRequestLocalityType(),
csAssignment.getSchedulingMode() != null ?
csAssignment.getSchedulingMode() :
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
@@ -2403,8 +2302,8 @@ public class CapacityScheduler extends
getSchedulerContainer(rmContainer, false),
getSchedulerContainersToRelease(csAssignment),
getSchedulerContainer(csAssignment.getFulfilledReservedContainer(),
- false), csAssignment.isIncreasedAllocation(),
- csAssignment.getType(), csAssignment.getRequestLocalityType(),
+ false), csAssignment.getType(),
+ csAssignment.getRequestLocalityType(),
csAssignment.getSchedulingMode() != null ?
csAssignment.getSchedulingMode() :
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-6216. Unify Container
Resizing code paths with Container Updates making it scheduler agnostic.
(Arun Suresh via wangda)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index b65f16a..1b20556 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1134,12 +1134,7 @@ public class LeafQueue extends AbstractCSQueue {
if (targetLeafQueue == this) {
// When trying to preempt containers from the same queue
- if (rmContainer.hasIncreaseReservation()) {
- // Increased container reservation
- unreserveIncreasedContainer(clusterResource,
- schedulerContainer.getSchedulerApplicationAttempt(),
- schedulerContainer.getSchedulerNode(), rmContainer);
- } else if (rmContainer.getState() == RMContainerState.RESERVED) {
+ if (rmContainer.getState() == RMContainerState.RESERVED) {
// For other reserved containers
// This is a reservation exchange, complete previous reserved container
completedContainer(clusterResource,
@@ -1212,8 +1207,7 @@ public class LeafQueue extends AbstractCSQueue {
schedulerContainer.getSchedulerApplicationAttempt(),
allocation.getAllocatedOrReservedResource(),
schedulerContainer.getNodePartition(),
- schedulerContainer.getRmContainer(),
- allocation.isIncreasedAllocation());
+ schedulerContainer.getRmContainer());
orderingPolicy.containerAllocated(
schedulerContainer.getSchedulerApplicationAttempt(),
schedulerContainer.getRmContainer());
@@ -1446,40 +1440,6 @@ public class LeafQueue extends AbstractCSQueue {
readLock.unlock();
}
}
-
- @Override
- public void unreserveIncreasedContainer(Resource clusterResource,
- FiCaSchedulerApp app, FiCaSchedulerNode node, RMContainer rmContainer) {
- boolean removed = false;
- Priority priority = null;
-
- try {
- writeLock.lock();
- if (rmContainer.getContainer() != null) {
- priority = rmContainer.getContainer().getPriority();
- }
-
- if (null != priority) {
- removed = app.unreserve(rmContainer.getAllocatedSchedulerKey(), node,
- rmContainer);
- }
-
- if (removed) {
- // Inform the ordering policy
- orderingPolicy.containerReleased(app, rmContainer);
-
- releaseResource(clusterResource, app, rmContainer.getReservedResource(),
- node.getPartition(), rmContainer, true);
- }
- } finally {
- writeLock.unlock();
- }
-
- if (removed) {
- getParent().unreserveIncreasedContainer(clusterResource, app, node,
- rmContainer);
- }
- }
private void updateSchedulerHealthForCompletedContainer(
RMContainer rmContainer, ContainerStatus containerStatus) {
@@ -1538,16 +1498,6 @@ public class LeafQueue extends AbstractCSQueue {
updateSchedulerHealthForCompletedContainer(rmContainer, containerStatus);
if (application != null) {
- // unreserve container increase request if it previously reserved.
- if (rmContainer.hasIncreaseReservation()) {
- unreserveIncreasedContainer(clusterResource, application, node,
- rmContainer);
- }
-
- // Remove container increase request if it exists
- application.removeIncreaseRequest(node.getNodeID(),
- rmContainer.getAllocatedSchedulerKey(), rmContainer.getContainerId());
-
boolean removed = false;
// Careful! Locking order is important!
@@ -1576,7 +1526,7 @@ public class LeafQueue extends AbstractCSQueue {
orderingPolicy.containerReleased(application, rmContainer);
releaseResource(clusterResource, application, container.getResource(),
- node.getPartition(), rmContainer, false);
+ node.getPartition(), rmContainer);
}
} finally {
writeLock.unlock();
@@ -1597,12 +1547,10 @@ public class LeafQueue extends AbstractCSQueue {
void allocateResource(Resource clusterResource,
SchedulerApplicationAttempt application, Resource resource,
- String nodePartition, RMContainer rmContainer,
- boolean isIncreasedAllocation) {
+ String nodePartition, RMContainer rmContainer) {
try {
writeLock.lock();
- super.allocateResource(clusterResource, resource, nodePartition,
- isIncreasedAllocation);
+ super.allocateResource(clusterResource, resource, nodePartition);
// handle ignore exclusivity container
if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
@@ -1643,11 +1591,10 @@ public class LeafQueue extends AbstractCSQueue {
void releaseResource(Resource clusterResource,
FiCaSchedulerApp application, Resource resource, String nodePartition,
- RMContainer rmContainer, boolean isChangeResource) {
+ RMContainer rmContainer) {
try {
writeLock.lock();
- super.releaseResource(clusterResource, resource, nodePartition,
- isChangeResource);
+ super.releaseResource(clusterResource, resource, nodePartition);
// handle ignore exclusivity container
if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
@@ -1787,7 +1734,7 @@ public class LeafQueue extends AbstractCSQueue {
rmContainer.getContainer().getNodeId());
allocateResource(clusterResource, attempt,
rmContainer.getContainer().getResource(), node.getPartition(),
- rmContainer, false);
+ rmContainer);
} finally {
writeLock.unlock();
}
@@ -1912,7 +1859,7 @@ public class LeafQueue extends AbstractCSQueue {
FiCaSchedulerNode node =
scheduler.getNode(rmContainer.getContainer().getNodeId());
allocateResource(clusterResource, application, rmContainer.getContainer()
- .getResource(), node.getPartition(), rmContainer, false);
+ .getResource(), node.getPartition(), rmContainer);
LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
@@ -1931,7 +1878,7 @@ public class LeafQueue extends AbstractCSQueue {
FiCaSchedulerNode node =
scheduler.getNode(rmContainer.getContainer().getNodeId());
releaseResource(clusterResource, application, rmContainer.getContainer()
- .getResource(), node.getPartition(), rmContainer, false);
+ .getResource(), node.getPartition(), rmContainer);
LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
@@ -2000,85 +1947,6 @@ public class LeafQueue extends AbstractCSQueue {
return defaultAppPriorityPerQueue;
}
- /**
- *
- * @param clusterResource Total cluster resource
- * @param decreaseRequest The decrease request
- * @param app The application of interest
- */
- @Override
- public void decreaseContainer(Resource clusterResource,
- SchedContainerChangeRequest decreaseRequest,
- FiCaSchedulerApp app) throws InvalidResourceRequestException {
- // If the container being decreased is reserved, we need to unreserve it
- // first.
- RMContainer rmContainer = decreaseRequest.getRMContainer();
- if (rmContainer.hasIncreaseReservation()) {
- unreserveIncreasedContainer(clusterResource, app,
- (FiCaSchedulerNode)decreaseRequest.getSchedulerNode(), rmContainer);
- }
- boolean resourceDecreased = false;
- Resource resourceBeforeDecrease;
- // Grab queue lock to avoid race condition when getting container resource
-
- try {
- writeLock.lock();
- // Make sure the decrease request is valid in terms of current resource
- // and target resource. This must be done under the leaf queue lock.
- // Throws exception if the check fails.
- RMServerUtils.checkSchedContainerChangeRequest(decreaseRequest, false);
- // Save resource before decrease for debug log
- resourceBeforeDecrease = Resources.clone(
- rmContainer.getAllocatedResource());
- // Do we have increase request for the same container? If so, remove it
- boolean hasIncreaseRequest = app.removeIncreaseRequest(
- decreaseRequest.getNodeId(),
- decreaseRequest.getRMContainer().getAllocatedSchedulerKey(),
- decreaseRequest.getContainerId());
- if (hasIncreaseRequest) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("While processing decrease requests, found an increase"
- + " request for the same container " + decreaseRequest
- .getContainerId() + ", removed the increase request");
- }
- }
- // Delta capacity is negative when it's a decrease request
- Resource absDelta = Resources.negate(decreaseRequest.getDeltaCapacity());
- if (Resources.equals(absDelta, Resources.none())) {
- // If delta capacity of this decrease request is 0, this decrease
- // request serves the purpose of cancelling an existing increase request
- // if any
- if (LOG.isDebugEnabled()) {
- LOG.debug("Decrease target resource equals to existing resource for"
- + " container:" + decreaseRequest.getContainerId()
- + " ignore this decrease request.");
- }
- } else{
- // Release the delta resource
- releaseResource(clusterResource, app, absDelta,
- decreaseRequest.getNodePartition(),
- decreaseRequest.getRMContainer(), true);
- // Notify application
- app.decreaseContainer(decreaseRequest);
- // Notify node
- decreaseRequest.getSchedulerNode().decreaseContainer(
- decreaseRequest.getContainerId(), absDelta);
- resourceDecreased = true;
- }
- } finally {
- writeLock.unlock();
- }
-
- if (resourceDecreased) {
- // Notify parent queue outside of leaf queue lock
- getParent().decreaseContainer(clusterResource, decreaseRequest, app);
- LOG.info("Application attempt " + app.getApplicationAttemptId()
- + " decreased container:" + decreaseRequest.getContainerId()
- + " from " + resourceBeforeDecrease + " to "
- + decreaseRequest.getTargetCapacity());
- }
- }
-
public void updateApplicationPriority(SchedulerApplication<FiCaSchedulerApp> app,
Priority newAppPriority) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 9c42c61..6f82fcc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -773,12 +773,11 @@ public class ParentQueue extends AbstractCSQueue {
}
private void internalReleaseResource(Resource clusterResource,
- FiCaSchedulerNode node, Resource releasedResource,
- boolean changeResource) {
+ FiCaSchedulerNode node, Resource releasedResource) {
try {
writeLock.lock();
super.releaseResource(clusterResource, releasedResource,
- node.getPartition(), changeResource);
+ node.getPartition());
if (LOG.isDebugEnabled()) {
LOG.debug(
@@ -789,38 +788,6 @@ public class ParentQueue extends AbstractCSQueue {
writeLock.unlock();
}
}
-
- @Override
- public void decreaseContainer(Resource clusterResource,
- SchedContainerChangeRequest decreaseRequest, FiCaSchedulerApp app)
- throws InvalidResourceRequestException {
- // delta capacity is negative when it's a decrease request
- Resource absDeltaCapacity =
- Resources.negate(decreaseRequest.getDeltaCapacity());
-
- internalReleaseResource(clusterResource,
- csContext.getNode(decreaseRequest.getNodeId()), absDeltaCapacity, false);
-
- // Inform the parent
- if (parent != null) {
- parent.decreaseContainer(clusterResource, decreaseRequest, app);
- }
- }
-
- @Override
- public void unreserveIncreasedContainer(Resource clusterResource,
- FiCaSchedulerApp app, FiCaSchedulerNode node, RMContainer rmContainer) {
- if (app != null) {
- internalReleaseResource(clusterResource, node,
- rmContainer.getReservedResource(), false);
-
- // Inform the parent
- if (parent != null) {
- parent.unreserveIncreasedContainer(clusterResource, app, node,
- rmContainer);
- }
- }
- }
@Override
public void completedContainer(Resource clusterResource,
@@ -830,7 +797,7 @@ public class ParentQueue extends AbstractCSQueue {
boolean sortQueues) {
if (application != null) {
internalReleaseResource(clusterResource, node,
- rmContainer.getContainer().getResource(), false);
+ rmContainer.getContainer().getResource());
// Inform the parent
if (parent != null) {
@@ -886,7 +853,7 @@ public class ParentQueue extends AbstractCSQueue {
FiCaSchedulerNode node = scheduler.getNode(
rmContainer.getContainer().getNodeId());
allocateResource(clusterResource,
- rmContainer.getContainer().getResource(), node.getPartition(), false);
+ rmContainer.getContainer().getResource(), node.getPartition());
} finally {
writeLock.unlock();
}
@@ -923,7 +890,7 @@ public class ParentQueue extends AbstractCSQueue {
FiCaSchedulerNode node =
scheduler.getNode(rmContainer.getContainer().getNodeId());
allocateResource(clusterResource, rmContainer.getContainer()
- .getResource(), node.getPartition(), false);
+ .getResource(), node.getPartition());
LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
@@ -943,7 +910,7 @@ public class ParentQueue extends AbstractCSQueue {
scheduler.getNode(rmContainer.getContainer().getNodeId());
super.releaseResource(clusterResource,
rmContainer.getContainer().getResource(),
- node.getPartition(), false);
+ node.getPartition());
LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
@@ -960,11 +927,10 @@ public class ParentQueue extends AbstractCSQueue {
}
void allocateResource(Resource clusterResource,
- Resource resource, String nodePartition, boolean changeContainerResource) {
+ Resource resource, String nodePartition) {
try {
writeLock.lock();
- super.allocateResource(clusterResource, resource, nodePartition,
- changeContainerResource);
+ super.allocateResource(clusterResource, resource, nodePartition);
/**
* check if we need to kill (killable) containers if maximum resource violated.
@@ -1054,8 +1020,7 @@ public class ParentQueue extends AbstractCSQueue {
// Book-keeping
// Note: Update headroom to account for current allocation too...
allocateResource(cluster, allocation.getAllocatedOrReservedResource(),
- schedulerContainer.getNodePartition(),
- allocation.isIncreasedAllocation());
+ schedulerContainer.getNodePartition());
LOG.info("assignedContainer" + " queue=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
index 57188d8..4879fae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
public class ContainerAllocator extends AbstractContainerAllocator {
- private AbstractContainerAllocator increaseContainerAllocator;
private AbstractContainerAllocator regularContainerAllocator;
public ContainerAllocator(FiCaSchedulerApp application,
@@ -45,8 +44,6 @@ public class ContainerAllocator extends AbstractContainerAllocator {
RMContext rmContext, ActivitiesManager activitiesManager) {
super(application, rc, rmContext);
- increaseContainerAllocator =
- new IncreaseContainerAllocator(application, rc, rmContext);
regularContainerAllocator = new RegularContainerAllocator(application, rc,
rmContext, activitiesManager);
}
@@ -55,32 +52,8 @@ public class ContainerAllocator extends AbstractContainerAllocator {
public CSAssignment assignContainers(Resource clusterResource,
PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode,
ResourceLimits resourceLimits, RMContainer reservedContainer) {
- if (reservedContainer != null) {
- if (reservedContainer.getState() == RMContainerState.RESERVED) {
- // It's a regular container
- return regularContainerAllocator.assignContainers(clusterResource,
- ps, schedulingMode, resourceLimits, reservedContainer);
- } else {
- // It's a increase container
- return increaseContainerAllocator.assignContainers(clusterResource,
- ps, schedulingMode, resourceLimits, reservedContainer);
- }
- } else {
- /*
- * Try to allocate increase container first, and if we failed to allocate
- * anything, we will try to allocate regular container
- */
- CSAssignment assign =
- increaseContainerAllocator.assignContainers(clusterResource, ps,
- schedulingMode, resourceLimits, null);
- if (Resources.greaterThan(rc, clusterResource, assign.getResource(),
- Resources.none())) {
- return assign;
- }
-
- return regularContainerAllocator.assignContainers(clusterResource, ps,
- schedulingMode, resourceLimits, null);
- }
+ return regularContainerAllocator.assignContainers(clusterResource,
+ ps, schedulingMode, resourceLimits, reservedContainer);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
deleted file mode 100644
index 0dc527f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
+++ /dev/null
@@ -1,337 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.Resources;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
-public class IncreaseContainerAllocator extends AbstractContainerAllocator {
- private static final Log LOG =
- LogFactory.getLog(IncreaseContainerAllocator.class);
-
- public IncreaseContainerAllocator(FiCaSchedulerApp application,
- ResourceCalculator rc, RMContext rmContext) {
- super(application, rc, rmContext);
- }
-
- /**
- * Quick check if we can allocate anything here:
- * We will not continue if:
- * - Headroom doesn't support allocate minimumAllocation
- * -
- */
- private boolean checkHeadroom(Resource clusterResource,
- ResourceLimits currentResourceLimits, Resource required) {
- return Resources.greaterThanOrEqual(rc, clusterResource,
- currentResourceLimits.getHeadroom(), required);
- }
-
- private CSAssignment createReservedIncreasedCSAssignment(
- SchedContainerChangeRequest request) {
- CSAssignment assignment =
- new CSAssignment(request.getDeltaCapacity(), NodeType.NODE_LOCAL, null,
- application, CSAssignment.SkippedType.NONE, false);
- Resources.addTo(assignment.getAssignmentInformation().getReserved(),
- request.getDeltaCapacity());
- assignment.getAssignmentInformation().incrReservations();
- assignment.getAssignmentInformation().addReservationDetails(
- request.getRMContainer(), application.getCSLeafQueue().getQueuePath());
- assignment.setIncreasedAllocation(true);
-
- LOG.info("Reserved increase container request:" + request.toString());
-
- return assignment;
- }
-
- private CSAssignment createSuccessfullyIncreasedCSAssignment(
- SchedContainerChangeRequest request, boolean fromReservation) {
- CSAssignment assignment =
- new CSAssignment(request.getDeltaCapacity(), NodeType.NODE_LOCAL, null,
- application, CSAssignment.SkippedType.NONE, fromReservation);
- Resources.addTo(assignment.getAssignmentInformation().getAllocated(),
- request.getDeltaCapacity());
- assignment.getAssignmentInformation().incrAllocations();
- assignment.getAssignmentInformation().addAllocationDetails(
- request.getRMContainer(), application.getCSLeafQueue().getQueuePath());
- assignment.setIncreasedAllocation(true);
-
- if (fromReservation) {
- assignment.setFulfilledReservedContainer(request.getRMContainer());
- }
-
- // notify application
- application
- .getCSLeafQueue()
- .getOrderingPolicy()
- .containerAllocated(application,
- application.getRMContainer(request.getContainerId()));
-
- LOG.info("Approved increase container request:" + request.toString()
- + " fromReservation=" + fromReservation);
-
- return assignment;
- }
-
- private CSAssignment allocateIncreaseRequestFromReservedContainer(
- SchedulerNode node, Resource cluster,
- SchedContainerChangeRequest increaseRequest) {
- if (Resources.fitsIn(rc, cluster, increaseRequest.getDeltaCapacity(),
- node.getUnallocatedResource())) {
- return createSuccessfullyIncreasedCSAssignment(increaseRequest, true);
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Failed to allocate reserved increase request:"
- + increaseRequest.toString()
- + ". There's no enough available resource");
- }
-
- // We still cannot allocate this container, will wait for next turn
- return CSAssignment.SKIP_ASSIGNMENT;
- }
- }
-
- private CSAssignment allocateIncreaseRequest(FiCaSchedulerNode node,
- Resource cluster, SchedContainerChangeRequest increaseRequest) {
- if (Resources.fitsIn(rc, cluster, increaseRequest.getDeltaCapacity(),
- node.getUnallocatedResource())) {
- return createSuccessfullyIncreasedCSAssignment(increaseRequest, false);
- } else{
- // We cannot allocate this container, but since queue capacity /
- // user-limit matches, we can reserve this container on this node.
- return createReservedIncreasedCSAssignment(increaseRequest);
- }
- }
-
- @Override
- public CSAssignment assignContainers(Resource clusterResource,
- PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode,
- ResourceLimits resourceLimits, RMContainer reservedContainer) {
- AppSchedulingInfo sinfo = application.getAppSchedulingInfo();
- FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
-
- if (null == node) {
- // This is global scheduling enabled
- // FIXME, support container increase when global scheduling enabled
- return CSAssignment.SKIP_ASSIGNMENT;
- }
- NodeId nodeId = node.getNodeID();
-
- if (reservedContainer == null) {
- // Do we have increase request on this node?
- if (!sinfo.hasIncreaseRequest(nodeId)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skip allocating increase request since we don't have any"
- + " increase request on this node=" + node.getNodeID());
- }
-
- return CSAssignment.SKIP_ASSIGNMENT;
- }
-
- // Check if we need to unreserve something, note that we don't support
- // continuousReservationLooking now. TODO, need think more about how to
- // support it.
- boolean shouldUnreserve =
- Resources.greaterThan(rc, clusterResource,
- resourceLimits.getAmountNeededUnreserve(), Resources.none());
-
- // Check if we can allocate minimum resource according to headroom
- boolean cannotAllocateAnything =
- !checkHeadroom(clusterResource, resourceLimits, rmContext
- .getScheduler().getMinimumResourceCapability());
-
- // Skip the app if we failed either of above check
- if (cannotAllocateAnything || shouldUnreserve) {
- if (LOG.isDebugEnabled()) {
- if (shouldUnreserve) {
- LOG.debug("Cannot continue since we have to unreserve some resource"
- + ", now increase container allocation doesn't "
- + "support continuous reservation looking..");
- }
- if (cannotAllocateAnything) {
- LOG.debug("We cannot allocate anything because of low headroom, "
- + "headroom=" + resourceLimits.getHeadroom());
- }
- }
-
- return CSAssignment.SKIP_ASSIGNMENT;
- }
-
- CSAssignment assigned = null;
-
- /*
- * Loop each priority, and containerId. Container priority is not
- * equivalent to request priority, application master can run an important
- * task on a less prioritized container.
- *
- * So behavior here is, we still try to increase container with higher
- * priority, but will skip increase request and move to next increase
- * request if queue-limit or user-limit aren't satisfied
- */
- for (SchedulerRequestKey schedulerKey : application.getSchedulerKeys()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Looking at increase request for application="
- + application.getApplicationAttemptId() + " priority="
- + schedulerKey.getPriority());
- }
-
- /*
- * If we have multiple to-be-increased containers under same priority on
- * a same host, we will try to increase earlier launched container
- * first. And again - we will skip a request and move to next if it
- * cannot be allocated.
- */
- Map<ContainerId, SchedContainerChangeRequest> increaseRequestMap =
- sinfo.getIncreaseRequests(nodeId, schedulerKey);
-
- // We don't have more increase request on this priority, skip..
- if (null == increaseRequestMap) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("There's no increase request for "
- + application.getApplicationAttemptId() + " priority="
- + schedulerKey.getPriority());
- }
- continue;
- }
- Iterator<Entry<ContainerId, SchedContainerChangeRequest>> iter =
- increaseRequestMap.entrySet().iterator();
-
- while (iter.hasNext()) {
- Entry<ContainerId, SchedContainerChangeRequest> entry =
- iter.next();
- SchedContainerChangeRequest increaseRequest =
- entry.getValue();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Looking at increase request=" + increaseRequest.toString());
- }
-
- boolean headroomSatisifed = checkHeadroom(clusterResource,
- resourceLimits, increaseRequest.getDeltaCapacity());
- if (!headroomSatisifed) {
- // skip if doesn't satisfy headroom limit
- if (LOG.isDebugEnabled()) {
- LOG.debug(" Headroom is not satisfied, skip..");
- }
- continue;
- }
-
- RMContainer rmContainer = increaseRequest.getRMContainer();
- if (rmContainer.getContainerState() != ContainerState.RUNNING) {
- // if the container is not running, we should remove the
- // increaseRequest and continue;
- if (LOG.isDebugEnabled()) {
- LOG.debug(" Container is not running any more, skip...");
- }
- application.addToBeRemovedIncreaseRequest(increaseRequest);
- continue;
- }
-
- if (!Resources.fitsIn(rc, clusterResource,
- increaseRequest.getTargetCapacity(), node.getTotalResource())) {
- // if the target capacity is more than what the node can offer, we
- // will simply remove and skip it.
- // The reason of doing check here instead of adding increase request
- // to scheduler because node's resource could be updated after
- // request added.
- if (LOG.isDebugEnabled()) {
- LOG.debug(" Target capacity is more than what node can offer,"
- + " node.resource=" + node.getTotalResource());
- }
- application.addToBeRemovedIncreaseRequest(increaseRequest);
- continue;
- }
-
- // Try to allocate the increase request
- assigned =
- allocateIncreaseRequest(node, clusterResource, increaseRequest);
- if (assigned.getSkippedType()
- == CSAssignment.SkippedType.NONE) {
- // When we don't skip this request, which means we either allocated
- // OR reserved this request. We will break
- break;
- }
- }
-
- // We may have allocated something
- if (assigned != null && assigned.getSkippedType()
- == CSAssignment.SkippedType.NONE) {
- break;
- }
- }
-
- return assigned == null ? CSAssignment.SKIP_ASSIGNMENT : assigned;
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Trying to allocate reserved increase container request..");
- }
-
- // We already reserved this increase container
- SchedContainerChangeRequest request =
- sinfo.getIncreaseRequest(nodeId,
- reservedContainer.getAllocatedSchedulerKey(),
- reservedContainer.getContainerId());
-
- // We will cancel the reservation any of following happens
- // - Container finished
- // - No increase request needed
- // - Target resource updated
- if (null == request
- || reservedContainer.getContainerState() != ContainerState.RUNNING
- || (!Resources.equals(reservedContainer.getReservedResource(),
- request.getDeltaCapacity()))) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("We don't need reserved increase container request "
- + "for container=" + reservedContainer.getContainerId()
- + ". Unreserving and return...");
- }
-
- // We don't need this container now, just return excessive reservation
- return new CSAssignment(application, reservedContainer);
- }
-
- return allocateIncreaseRequestFromReservedContainer(node, clusterResource,
- request);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/ContainerAllocationProposal.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/ContainerAllocationProposal.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/ContainerAllocationProposal.java
index ac83d6f..2921e7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/ContainerAllocationProposal.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/ContainerAllocationProposal.java
@@ -43,8 +43,6 @@ public class ContainerAllocationProposal<A extends SchedulerApplicationAttempt,
// not be included by toRelease list
private SchedulerContainer<A, N> allocateFromReservedContainer;
- private boolean isIncreasedAllocation;
-
private NodeType allocationLocalityType;
private NodeType requestLocalityType;
@@ -57,7 +55,7 @@ public class ContainerAllocationProposal<A extends SchedulerApplicationAttempt,
SchedulerContainer<A, N> allocatedOrReservedContainer,
List<SchedulerContainer<A, N>> toRelease,
SchedulerContainer<A, N> allocateFromReservedContainer,
- boolean isIncreasedAllocation, NodeType allocationLocalityType,
+ NodeType allocationLocalityType,
NodeType requestLocalityType, SchedulingMode schedulingMode,
Resource allocatedResource) {
this.allocatedOrReservedContainer = allocatedOrReservedContainer;
@@ -65,7 +63,6 @@ public class ContainerAllocationProposal<A extends SchedulerApplicationAttempt,
this.toRelease = toRelease;
}
this.allocateFromReservedContainer = allocateFromReservedContainer;
- this.isIncreasedAllocation = isIncreasedAllocation;
this.allocationLocalityType = allocationLocalityType;
this.requestLocalityType = requestLocalityType;
this.schedulingMode = schedulingMode;
@@ -84,10 +81,6 @@ public class ContainerAllocationProposal<A extends SchedulerApplicationAttempt,
return allocationLocalityType;
}
- public boolean isIncreasedAllocation() {
- return isIncreasedAllocation;
- }
-
public SchedulerContainer<A, N> getAllocateFromReservedContainer() {
return allocateFromReservedContainer;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 30b7305..fea29bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -312,54 +312,6 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
return false;
}
- private SchedContainerChangeRequest getResourceChangeRequest(
- SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer) {
- return appSchedulingInfo.getIncreaseRequest(
- schedulerContainer.getSchedulerNode().getNodeID(),
- schedulerContainer.getSchedulerRequestKey(),
- schedulerContainer.getRmContainer().getContainerId());
- }
-
- private boolean checkIncreaseContainerAllocation(
- ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocation,
- SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer) {
- // When increase a container
- if (schedulerContainer.getRmContainer().getState()
- != RMContainerState.RUNNING) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Trying to increase a container, but container="
- + schedulerContainer.getRmContainer().getContainerId()
- + " is not in running state.");
- }
- return false;
- }
-
- // Check if increase request is still valid
- SchedContainerChangeRequest increaseRequest = getResourceChangeRequest(
- schedulerContainer);
-
- if (null == increaseRequest || !Resources.equals(
- increaseRequest.getDeltaCapacity(),
- allocation.getAllocatedOrReservedResource())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Increase request has been changed, reject this proposal");
- }
- return false;
- }
-
- if (allocation.getAllocateFromReservedContainer() != null) {
- // In addition, if allocation is from a reserved container, check
- // if the reserved container has enough reserved space
- if (!Resources.equals(
- allocation.getAllocateFromReservedContainer().getRmContainer()
- .getReservedResource(), increaseRequest.getDeltaCapacity())) {
- return false;
- }
- }
-
- return true;
- }
-
private boolean commonCheckContainerAllocation(
Resource cluster,
ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocation,
@@ -445,30 +397,23 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
schedulerContainer = allocation.getAllocatedOrReservedContainer();
if (schedulerContainer.isAllocated()) {
- if (!allocation.isIncreasedAllocation()) {
- // When allocate a new container
- resourceRequests =
- schedulerContainer.getRmContainer().getResourceRequests();
-
- // Check pending resource request
- if (!appSchedulingInfo.checkAllocation(allocation.getAllocationLocalityType(),
- schedulerContainer.getSchedulerNode(),
- schedulerContainer.getSchedulerRequestKey())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No pending resource for: nodeType=" + allocation
- .getAllocationLocalityType() + ", node=" + schedulerContainer
- .getSchedulerNode() + ", requestKey=" + schedulerContainer
- .getSchedulerRequestKey() + ", application="
- + getApplicationAttemptId());
- }
-
- return false;
- }
- } else {
- if (!checkIncreaseContainerAllocation(allocation,
- schedulerContainer)) {
- return false;
+ // When allocate a new container
+ resourceRequests =
+ schedulerContainer.getRmContainer().getResourceRequests();
+
+ // Check pending resource request
+ if (!appSchedulingInfo.checkAllocation(allocation.getAllocationLocalityType(),
+ schedulerContainer.getSchedulerNode(),
+ schedulerContainer.getSchedulerRequestKey())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No pending resource for: nodeType=" + allocation
+ .getAllocationLocalityType() + ", node=" + schedulerContainer
+ .getSchedulerNode() + ", requestKey=" + schedulerContainer
+ .getSchedulerRequestKey() + ", application="
+ + getApplicationAttemptId());
}
+
+ return false;
}
// Common part of check container allocation regardless if it is a
@@ -541,12 +486,10 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
// Generate new containerId if it is not an allocation for increasing
// Or re-reservation
- if (!allocation.isIncreasedAllocation()) {
- if (rmContainer.getContainer().getId() == null) {
- rmContainer.setContainerId(BuilderUtils
- .newContainerId(getApplicationAttemptId(),
- getNewContainerId()));
- }
+ if (rmContainer.getContainer().getId() == null) {
+ rmContainer.setContainerId(BuilderUtils
+ .newContainerId(getApplicationAttemptId(),
+ getNewContainerId()));
}
ContainerId containerId = rmContainer.getContainerId();
@@ -562,77 +505,50 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
schedulerContainer.getSchedulerNode(), reservedContainer);
}
- // Update this application for the allocated container
- if (!allocation.isIncreasedAllocation()) {
- // Allocate a new container
- addToNewlyAllocatedContainers(
- schedulerContainer.getSchedulerNode(), rmContainer);
- liveContainers.put(containerId, rmContainer);
+ // Allocate a new container
+ addToNewlyAllocatedContainers(
+ schedulerContainer.getSchedulerNode(), rmContainer);
+ liveContainers.put(containerId, rmContainer);
- // Deduct pending resource requests
- List<ResourceRequest> requests = appSchedulingInfo.allocate(
- allocation.getAllocationLocalityType(),
- schedulerContainer.getSchedulerNode(),
- schedulerContainer.getSchedulerRequestKey(),
- schedulerContainer.getRmContainer().getContainer());
- ((RMContainerImpl) rmContainer).setResourceRequests(requests);
+ // Deduct pending resource requests
+ List<ResourceRequest> requests = appSchedulingInfo.allocate(
+ allocation.getAllocationLocalityType(),
+ schedulerContainer.getSchedulerNode(),
+ schedulerContainer.getSchedulerRequestKey(),
+ schedulerContainer.getRmContainer().getContainer());
+ ((RMContainerImpl) rmContainer).setResourceRequests(requests);
- attemptResourceUsage.incUsed(schedulerContainer.getNodePartition(),
- allocation.getAllocatedOrReservedResource());
+ attemptResourceUsage.incUsed(schedulerContainer.getNodePartition(),
+ allocation.getAllocatedOrReservedResource());
- rmContainer.handle(
- new RMContainerEvent(containerId, RMContainerEventType.START));
+ rmContainer.handle(
+ new RMContainerEvent(containerId, RMContainerEventType.START));
- // Inform the node
- schedulerContainer.getSchedulerNode().allocateContainer(
- rmContainer);
+ // Inform the node
+ schedulerContainer.getSchedulerNode().allocateContainer(
+ rmContainer);
- // update locality statistics,
- incNumAllocatedContainers(allocation.getAllocationLocalityType(),
- allocation.getRequestLocalityType());
+ // update locality statistics,
+ incNumAllocatedContainers(allocation.getAllocationLocalityType(),
+ allocation.getRequestLocalityType());
- if (LOG.isDebugEnabled()) {
- LOG.debug("allocate: applicationAttemptId=" + containerId
- .getApplicationAttemptId() + " container=" + containerId
- + " host=" + rmContainer.getAllocatedNode().getHost()
- + " type=" + allocation.getAllocationLocalityType());
- }
- RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER,
- "SchedulerApp", getApplicationId(), containerId,
- allocation.getAllocatedOrReservedResource());
- } else{
- SchedContainerChangeRequest increaseRequest =
- getResourceChangeRequest(schedulerContainer);
-
- // allocate resource for an increase request
- // Notify node
- schedulerContainer.getSchedulerNode().increaseContainer(
- increaseRequest.getContainerId(),
- increaseRequest.getDeltaCapacity());
-
- // OK, we can allocate this increase request
- // Notify application
- increaseContainer(increaseRequest);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("allocate: applicationAttemptId=" + containerId
+ .getApplicationAttemptId() + " container=" + containerId
+ + " host=" + rmContainer.getAllocatedNode().getHost()
+ + " type=" + allocation.getAllocationLocalityType());
}
+ RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER,
+ "SchedulerApp", getApplicationId(), containerId,
+ allocation.getAllocatedOrReservedResource());
} else {
- if (!allocation.isIncreasedAllocation()) {
- // If the rmContainer's state is already updated to RESERVED, this is
- // a reReservation
- reserve(schedulerContainer.getSchedulerRequestKey(),
- schedulerContainer.getSchedulerNode(),
- schedulerContainer.getRmContainer(),
- schedulerContainer.getRmContainer().getContainer(),
- reReservation);
- } else{
- SchedContainerChangeRequest increaseRequest =
- getResourceChangeRequest(schedulerContainer);
-
- reserveIncreasedContainer(
- schedulerContainer.getSchedulerRequestKey(),
- schedulerContainer.getSchedulerNode(),
- increaseRequest.getRMContainer(),
- increaseRequest.getDeltaCapacity());
- }
+ // If the rmContainer's state is already updated to RESERVED, this is
+ // a reReservation
+ reserve(schedulerContainer.getSchedulerRequestKey(),
+ schedulerContainer.getSchedulerNode(),
+ schedulerContainer.getRmContainer(),
+ schedulerContainer.getRmContainer().getContainer(),
+ reReservation);
}
}
} finally {
@@ -649,9 +565,6 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
FiCaSchedulerNode node, RMContainer rmContainer) {
try {
writeLock.lock();
- // Cancel increase request (if it has reserved increase request
- rmContainer.cancelIncreaseReservation();
-
// Done with the reservation?
if (internalUnreserve(node, schedulerKey)) {
node.unreserveResource(this);
@@ -807,13 +720,6 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
.entrySet()) {
NodeId nodeId = entry.getKey();
RMContainer reservedContainer = entry.getValue();
- if (reservedContainer.hasIncreaseReservation()) {
- // Currently, only regular container allocation supports continuous
- // reservation looking, we don't support canceling increase request
- // reservation when allocating regular container.
- continue;
- }
-
Resource reservedResource = reservedContainer.getReservedResource();
// make sure we unreserve one with at least the same amount of
@@ -869,25 +775,6 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
}
}
- public boolean reserveIncreasedContainer(SchedulerRequestKey schedulerKey,
- FiCaSchedulerNode node,
- RMContainer rmContainer, Resource reservedResource) {
- // Inform the application
- if (super.reserveIncreasedContainer(node, schedulerKey, rmContainer,
- reservedResource)) {
-
- queue.getMetrics().reserveResource(getUser(), reservedResource);
-
- // Update the node
- node.reserveResource(this, schedulerKey, rmContainer);
-
- // Succeeded
- return true;
- }
-
- return false;
- }
-
public void reserve(SchedulerRequestKey schedulerKey, FiCaSchedulerNode node,
RMContainer rmContainer, Container container, boolean reReservation) {
// Update reserved metrics if this is the first reservation
@@ -1114,26 +1001,6 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
toBeRemovedIncRequests.put(request.getContainerId(), request);
}
- public void removedToBeRemovedIncreaseRequests() {
- // Remove invalid in request requests
- if (!toBeRemovedIncRequests.isEmpty()) {
- try {
- writeLock.lock();
- Iterator<Map.Entry<ContainerId, SchedContainerChangeRequest>> iter =
- toBeRemovedIncRequests.entrySet().iterator();
- while (iter.hasNext()) {
- SchedContainerChangeRequest req = iter.next().getValue();
- appSchedulingInfo.removeIncreaseRequest(req.getNodeId(),
- req.getRMContainer().getAllocatedSchedulerKey(),
- req.getContainerId());
- iter.remove();
- }
- } finally {
- writeLock.unlock();
- }
- }
- }
-
/*
* Overriding to appease findbugs
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
index 8eac929..c26a11b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
@@ -153,20 +153,6 @@ public class FiCaSchedulerNode extends SchedulerNode {
}
}
- @Override
- protected synchronized void changeContainerResource(ContainerId containerId,
- Resource deltaResource, boolean increase) {
- super.changeContainerResource(containerId, deltaResource, increase);
-
- if (killableContainers.containsKey(containerId)) {
- if (increase) {
- Resources.addTo(totalKillableResources, deltaResource);
- } else {
- Resources.subtractFrom(totalKillableResources, deltaResource);
- }
- }
- }
-
public synchronized Resource getTotalKillableResources() {
return totalKillableResources;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 3246778..0599414 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -819,9 +819,7 @@ public class FairScheduler extends
}
// Handle promotions and demotions
- handleExecutionTypeUpdates(
- application, updateRequests.getPromotionRequests(),
- updateRequests.getDemotionRequests());
+ handleContainerUpdates(application, updateRequests);
// Sanity check
normalizeRequests(ask);
@@ -1769,13 +1767,6 @@ public class FairScheduler extends
return targetQueueName;
}
- @Override
- protected void decreaseContainer(
- SchedContainerChangeRequest decreaseRequest,
- SchedulerApplicationAttempt attempt) {
- // TODO Auto-generated method stub
- }
-
public float getReservableNodesRatio() {
return reservableNodesRatio;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 64dbc7d..a8d4f48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -937,14 +937,6 @@ public class FifoScheduler extends
}
@Override
- protected void decreaseContainer(
- SchedContainerChangeRequest decreaseRequest,
- SchedulerApplicationAttempt attempt) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
protected synchronized void nodeUpdate(RMNode nm) {
super.nodeUpdate(nm);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index 899523c..e34665d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -137,11 +137,11 @@ public class TestChildQueueOrder {
final Resource allocatedResource = Resources.createResource(allocation);
if (queue instanceof ParentQueue) {
((ParentQueue)queue).allocateResource(clusterResource,
- allocatedResource, RMNodeLabelsManager.NO_LABEL, false);
+ allocatedResource, RMNodeLabelsManager.NO_LABEL);
} else {
FiCaSchedulerApp app1 = getMockApplication(0, "");
((LeafQueue)queue).allocateResource(clusterResource, app1,
- allocatedResource, null, null, false);
+ allocatedResource, null, null);
}
// Next call - nothing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
index 0696f57..b4b05ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
@@ -88,18 +88,6 @@ public class TestContainerResizing {
}
@Override
- protected void decreaseContainers(
- List<UpdateContainerRequest> decreaseRequests,
- SchedulerApplicationAttempt attempt) {
- try {
- Thread.sleep(1000);
- } catch(InterruptedException e) {
- LOG.debug("Thread interrupted.");
- }
- super.decreaseContainers(decreaseRequests, attempt);
- }
-
- @Override
public CSAssignment allocateContainersToNode(
PlacementSet<FiCaSchedulerNode> ps, boolean withNodeHeartbeat) {
try {
@@ -288,13 +276,9 @@ public class TestContainerResizing {
CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
-
- RMContainer rmContainer1 = app.getLiveContainersMap().get(containerId1);
/* Check reservation statuses */
// Increase request should be reserved
- Assert.assertTrue(rmContainer1.hasIncreaseReservation());
- Assert.assertEquals(6 * GB, rmContainer1.getReservedResource().getMemorySize());
Assert.assertFalse(app.getReservedContainers().isEmpty());
Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
// Pending resource will not be changed since it's not satisfied
@@ -319,7 +303,6 @@ public class TestContainerResizing {
/* Check statuses after reservation satisfied */
// Increase request should be unreserved
- Assert.assertFalse(rmContainer1.hasIncreaseReservation());
Assert.assertTrue(app.getReservedContainers().isEmpty());
Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
// Pending resource will be changed since it's satisfied
@@ -391,11 +374,8 @@ public class TestContainerResizing {
RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
- RMContainer rmContainer1 = app.getLiveContainersMap().get(containerId2);
-
/* Check reservation statuses */
// Increase request should *NOT* be reserved as it exceeds user limit
- Assert.assertFalse(rmContainer1.hasIncreaseReservation());
Assert.assertTrue(app.getReservedContainers().isEmpty());
Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
// Pending resource will not be changed since it's not satisfied
@@ -471,13 +451,9 @@ public class TestContainerResizing {
CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
-
- RMContainer rmContainer1 = app.getLiveContainersMap().get(containerId1);
/* Check reservation statuses */
// Increase request should be reserved
- Assert.assertTrue(rmContainer1.hasIncreaseReservation());
- Assert.assertEquals(6 * GB, rmContainer1.getReservedResource().getMemorySize());
Assert.assertFalse(app.getReservedContainers().isEmpty());
Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
// Pending resource will not be changed since it's not satisfied
@@ -510,7 +486,6 @@ public class TestContainerResizing {
// Increase request should be unreserved
Assert.assertTrue(app.getReservedContainers().isEmpty());
Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
- Assert.assertFalse(rmContainer1.hasIncreaseReservation());
// Pending resource will be changed since it's satisfied
checkPendingResource(rm1, "default", 0 * GB, null);
Assert.assertEquals(0 * GB,
@@ -585,13 +560,9 @@ public class TestContainerResizing {
CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
-
- RMContainer rmContainer1 = app.getLiveContainersMap().get(containerId1);
/* Check reservation statuses */
// Increase request should be reserved
- Assert.assertTrue(rmContainer1.hasIncreaseReservation());
- Assert.assertEquals(6 * GB, rmContainer1.getReservedResource().getMemorySize());
Assert.assertFalse(app.getReservedContainers().isEmpty());
Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
// Pending resource will not be changed since it's not satisfied
@@ -614,7 +585,7 @@ public class TestContainerResizing {
am1.sendContainerResizingRequest(Arrays.asList(
UpdateContainerRequest
.newInstance(0, containerId1,
- ContainerUpdateType.INCREASE_RESOURCE,
+ ContainerUpdateType.DECREASE_RESOURCE,
Resources.createResource(1 * GB), null)));
// Trigger a node heartbeat..
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
@@ -623,7 +594,6 @@ public class TestContainerResizing {
// Increase request should be unreserved
Assert.assertTrue(app.getReservedContainers().isEmpty());
Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
- Assert.assertFalse(rmContainer1.hasIncreaseReservation());
// Pending resource will be changed since it's satisfied
checkPendingResource(rm1, "default", 0 * GB, null);
Assert.assertEquals(0 * GB,
@@ -698,12 +668,8 @@ public class TestContainerResizing {
RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
- RMContainer rmContainer2 = app.getLiveContainersMap().get(containerId2);
-
/* Check reservation statuses */
// Increase request should be reserved
- Assert.assertTrue(rmContainer2.hasIncreaseReservation());
- Assert.assertEquals(6 * GB, rmContainer2.getReservedResource().getMemorySize());
Assert.assertFalse(app.getReservedContainers().isEmpty());
Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
// Pending resource will not be changed since it's not satisfied
@@ -721,12 +687,13 @@ public class TestContainerResizing {
// Complete container2, container will be unreserved and completed
am1.allocate(null, Arrays.asList(containerId2));
-
+
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ am1.allocate(null, null);
/* Check statuses after reservation satisfied */
// Increase request should be unreserved
Assert.assertTrue(app.getReservedContainers().isEmpty());
Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
- Assert.assertFalse(rmContainer2.hasIncreaseReservation());
// Pending resource will be changed since it's satisfied
checkPendingResource(rm1, "default", 0 * GB, null);
Assert.assertEquals(0 * GB,
@@ -796,12 +763,8 @@ public class TestContainerResizing {
RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
- RMContainer rmContainer2 = app.getLiveContainersMap().get(containerId2);
-
/* Check reservation statuses */
// Increase request should be reserved
- Assert.assertTrue(rmContainer2.hasIncreaseReservation());
- Assert.assertEquals(6 * GB, rmContainer2.getReservedResource().getMemorySize());
Assert.assertFalse(app.getReservedContainers().isEmpty());
Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
// Pending resource will not be changed since it's not satisfied
@@ -825,11 +788,11 @@ public class TestContainerResizing {
// Increase request should be unreserved
Assert.assertTrue(app.getReservedContainers().isEmpty());
Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
- Assert.assertFalse(rmContainer2.hasIncreaseReservation());
// Pending resource will be changed since it's satisfied
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
checkPendingResource(rm1, "default", 0 * GB, null);
- Assert.assertEquals(0 * GB,
- app.getAppAttemptResourceUsage().getPending().getMemorySize());
+
// Queue/user/application's usage will be updated
checkUsedResource(rm1, "default", 0 * GB, null);
// User will be removed
@@ -949,89 +912,6 @@ public class TestContainerResizing {
rm1.close();
}
- @Test
- public void testIncreaseContainerRequestGetPreferrence()
- throws Exception {
- /**
- * There're multiple containers need to be increased, and there're several
- * container allocation request, scheduler will try to increase container
- * before allocate new containers
- */
- MockRM rm1 = new MockRM() {
- @Override
- public RMNodeLabelsManager createNodeLabelManager() {
- return mgr;
- }
- };
- rm1.start();
- MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB);
-
- // app1 -> a1
- RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
- MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
-
- FiCaSchedulerApp app = TestUtils.getFiCaSchedulerApp(
- rm1, app1.getApplicationId());
- ApplicationAttemptId attemptId = am1.getApplicationAttemptId();
-
- // Container 2, 3 (priority=3)
- allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 3, 2);
-
- // Container 4, 5 (priority=2)
- allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 2, 4);
-
- // Container 6, 7 (priority=4)
- allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 4, 6);
-
- // am1 asks to change its container[2-7] from 1G to 2G
- List<UpdateContainerRequest> increaseRequests = new ArrayList<>();
- for (int cId = 2; cId <= 7; cId++) {
- ContainerId containerId =
- ContainerId.newContainerId(am1.getApplicationAttemptId(), cId);
- increaseRequests.add(UpdateContainerRequest
- .newInstance(0, containerId,
- ContainerUpdateType.INCREASE_RESOURCE,
- Resources.createResource(2 * GB), null));
- }
- am1.sendContainerResizingRequest(increaseRequests);
-
- checkPendingResource(rm1, "default", 6 * GB, null);
- Assert.assertEquals(6 * GB,
- app.getAppAttemptResourceUsage().getPending().getMemorySize());
-
- // Get rmNode1
- CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
- RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
-
- // assignContainer, container-4/5/2 increased (which has highest priority OR
- // earlier allocated)
- cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
- AllocateResponse allocateResponse = am1.allocate(null, null);
- Assert.assertEquals(3, allocateResponse.getUpdatedContainers().size());
- verifyContainerIncreased(allocateResponse,
- ContainerId.newContainerId(attemptId, 4), 2 * GB);
- verifyContainerIncreased(allocateResponse,
- ContainerId.newContainerId(attemptId, 5), 2 * GB);
- verifyContainerIncreased(allocateResponse,
- ContainerId.newContainerId(attemptId, 2), 2 * GB);
-
- /* Check statuses after allocation */
- // There're still 3 pending increase requests
- checkPendingResource(rm1, "default", 3 * GB, null);
- Assert.assertEquals(3 * GB,
- app.getAppAttemptResourceUsage().getPending().getMemorySize());
- // Queue/user/application's usage will be updated
- checkUsedResource(rm1, "default", 10 * GB, null);
- Assert.assertEquals(10 * GB, ((LeafQueue) cs.getQueue("default"))
- .getUser("user").getUsed().getMemorySize());
- Assert.assertEquals(0 * GB,
- app.getAppAttemptResourceUsage().getReserved().getMemorySize());
- Assert.assertEquals(10 * GB,
- app.getAppAttemptResourceUsage().getUsed().getMemorySize());
-
- rm1.close();
- }
-
@Test (timeout = 60000)
public void testDecreaseContainerWillNotDeadlockContainerAllocation()
throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
index 74cecf2..184e854 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
@@ -200,6 +200,7 @@ public class TestIncreaseAllocationExpirer {
// back action to complete
Thread.sleep(10000);
// Verify container size is 1G
+ am1.allocate(null, null);
Assert.assertEquals(
1 * GB, rm1.getResourceScheduler().getRMContainer(containerId2)
.getAllocatedResource().getMemorySize());
@@ -304,6 +305,8 @@ public class TestIncreaseAllocationExpirer {
// Wait long enough for the second token (5G) to expire, and verify that
// the roll back action is completed as expected
Thread.sleep(10000);
+ am1.allocate(null, null);
+ Thread.sleep(2000);
// Verify container size is rolled back to 3G
Assert.assertEquals(
3 * GB, rm1.getResourceScheduler().getRMContainer(containerId2)
@@ -400,13 +403,13 @@ public class TestIncreaseAllocationExpirer {
// Decrease containers
List<UpdateContainerRequest> decreaseRequests = new ArrayList<>();
decreaseRequests.add(UpdateContainerRequest.newInstance(1, containerId2,
- ContainerUpdateType.INCREASE_RESOURCE,
+ ContainerUpdateType.DECREASE_RESOURCE,
Resources.createResource(2 * GB), null));
decreaseRequests.add(UpdateContainerRequest.newInstance(1, containerId3,
- ContainerUpdateType.INCREASE_RESOURCE,
+ ContainerUpdateType.DECREASE_RESOURCE,
Resources.createResource(4 * GB), null));
decreaseRequests.add(UpdateContainerRequest.newInstance(1, containerId4,
- ContainerUpdateType.INCREASE_RESOURCE,
+ ContainerUpdateType.DECREASE_RESOURCE,
Resources.createResource(4 * GB), null));
AllocateResponse response =
am1.sendContainerResizingRequest(decreaseRequests);
@@ -418,6 +421,9 @@ public class TestIncreaseAllocationExpirer {
rm1, containerId4, Resources.createResource(6 * GB)));
// Wait for containerId3 token to expire,
Thread.sleep(10000);
+
+ am1.allocate(null, null);
+
Assert.assertEquals(
2 * GB, rm1.getResourceScheduler().getRMContainer(containerId2)
.getAllocatedResource().getMemorySize());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index ec1b84d..3fbbae3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -1061,11 +1061,11 @@ public class TestLeafQueue {
qb.releaseResource(clusterResource, app_0,
app_0.getAppSchedulingInfo().getPendingAsk(u0SchedKey)
.getPerAllocationResource(),
- null, null, false);
+ null, null);
qb.releaseResource(clusterResource, app_2,
app_2.getAppSchedulingInfo().getPendingAsk(u1SchedKey)
.getPerAllocationResource(),
- null, null, false);
+ null, null);
qb.setUserLimit(50);
qb.setUserLimitFactor(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac6b4c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 11fea82..c4b7a0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -171,11 +171,11 @@ public class TestParentQueue {
final Resource allocatedResource = Resources.createResource(allocation);
if (queue instanceof ParentQueue) {
((ParentQueue)queue).allocateResource(clusterResource,
- allocatedResource, RMNodeLabelsManager.NO_LABEL, false);
+ allocatedResource, RMNodeLabelsManager.NO_LABEL);
} else {
FiCaSchedulerApp app1 = getMockApplication(0, "");
((LeafQueue)queue).allocateResource(clusterResource, app1,
- allocatedResource, null, null, false);
+ allocatedResource, null, null);
}
// Next call - nothing
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HADOOP-14138. Remove S3A ref from
META-INF service discovery,
rely on existing core-default entry. Contributed by Steve Loughran
Posted by jh...@apache.org.
HADOOP-14138. Remove S3A ref from META-INF service discovery, rely on existing core-default entry.
Contributed by Steve Loughran
(cherry picked from commit 51fefbe38bf40373d2855163fa6444821b0f994c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a97833e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a97833e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a97833e0
Branch: refs/heads/YARN-5734
Commit: a97833e0ed4b31f0403ee3d789163615c7cdd9af
Parents: b3ec531
Author: Steve Loughran <st...@apache.org>
Authored: Thu Mar 2 21:44:30 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Mar 2 21:44:30 2017 +0000
----------------------------------------------------------------------
.../resources/META-INF/services/org.apache.hadoop.fs.FileSystem | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a97833e0/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
index e606fd9..1938bde 100644
--- a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
+++ b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
@@ -14,4 +14,3 @@
# limitations under the License.
org.apache.hadoop.fs.s3native.NativeS3FileSystem
-org.apache.hadoop.fs.s3a.S3AFileSystem
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDFS-11462. Fix occasional
BindException in TestNameNodeMetricsLogger.
Posted by jh...@apache.org.
HDFS-11462. Fix occasional BindException in TestNameNodeMetricsLogger.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/120bef7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/120bef7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/120bef7d
Branch: refs/heads/YARN-5734
Commit: 120bef7de81de96854156da192c855035b7d3e7a
Parents: 4a58870
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Feb 24 19:53:39 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Feb 24 19:53:39 2017 -0800
----------------------------------------------------------------------
.../hdfs/server/namenode/TestNameNodeMetricsLogger.java | 11 ++---------
1 file changed, 2 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/120bef7d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetricsLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetricsLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetricsLogger.java
index 6968bc4..9a0e67c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetricsLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetricsLogger.java
@@ -25,7 +25,6 @@ import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Appender;
import org.apache.log4j.AppenderSkeleton;
@@ -38,7 +37,6 @@ import org.junit.rules.Timeout;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
-import java.util.Random;
import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
@@ -51,7 +49,6 @@ import static org.mockito.Mockito.mock;
*/
public class TestNameNodeMetricsLogger {
static final Log LOG = LogFactory.getLog(TestNameNodeMetricsLogger.class);
- static final Random random = new Random(System.currentTimeMillis());
@Rule
public Timeout timeout = new Timeout(300000);
@@ -112,17 +109,13 @@ public class TestNameNodeMetricsLogger {
private NameNode makeNameNode(boolean enableMetricsLogging)
throws IOException {
Configuration conf = new HdfsConfiguration();
- conf.set(FS_DEFAULT_NAME_KEY, "hdfs://localhost:" + getRandomPort());
- conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:" + getRandomPort());
+ conf.set(FS_DEFAULT_NAME_KEY, "hdfs://localhost:0");
+ conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
conf.setInt(DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_KEY,
enableMetricsLogging ? 1 : 0); // If enabled, log early and log often
return new TestNameNode(conf);
}
- private int getRandomPort() throws IOException {
- return ServerSocketUtil.getPort(0, 10);
- }
-
private void addAppender(Log log, Appender appender) {
org.apache.log4j.Logger logger = ((Log4JLogger) log).getLogger();
@SuppressWarnings("unchecked")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HADOOP-14118. move jets3t into a
dependency on hadoop-aws JAR.
Posted by jh...@apache.org.
HADOOP-14118. move jets3t into a dependency on hadoop-aws JAR.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/258342e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/258342e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/258342e7
Branch: refs/heads/YARN-5734
Commit: 258342e76c87479db38a5404753ae91376a74299
Parents: d235dcd
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Feb 28 13:47:44 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Feb 28 13:47:44 2017 +0900
----------------------------------------------------------------------
hadoop-client-modules/hadoop-client/pom.xml | 4 ----
hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml | 4 ----
hadoop-common-project/hadoop-common/pom.xml | 5 -----
hadoop-common-project/hadoop-kms/pom.xml | 4 ----
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 8 --------
hadoop-tools/hadoop-aws/pom.xml | 5 +++++
6 files changed, 5 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258342e7/hadoop-client-modules/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml
index b16edf3..6bc7232 100644
--- a/hadoop-client-modules/hadoop-client/pom.xml
+++ b/hadoop-client-modules/hadoop-client/pom.xml
@@ -86,10 +86,6 @@
<artifactId>kfs</artifactId>
</exclusion>
<exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258342e7/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
index f96c1db..3211474 100644
--- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
@@ -94,10 +94,6 @@
<artifactId>kfs</artifactId>
</exclusion>
<exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258342e7/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 909cd78..0db6841 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -162,11 +162,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258342e7/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index cd4108f..479af91 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -122,10 +122,6 @@
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
<exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258342e7/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 9e5b221..f5cfd01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -123,10 +123,6 @@
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
<exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
</exclusion>
@@ -166,10 +162,6 @@
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
<exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258342e7/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 0fdbc5d..e5bbbfd 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -330,6 +330,11 @@
<type>test-jar</type>
</dependency>
<dependency>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bundle</artifactId>
<scope>compile</scope>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-6215. FairScheduler
preemption and update should not run concurrently. (Tao Jie via kasha)
Posted by jh...@apache.org.
YARN-6215. FairScheduler preemption and update should not run concurrently. (Tao Jie via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/815d5350
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/815d5350
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/815d5350
Branch: refs/heads/YARN-5734
Commit: 815d53506fb0c5ca029c993d6b094db2ac0ca6eb
Parents: 05391c1
Author: Karthik Kambatla <ka...@apache.org>
Authored: Sun Feb 26 20:16:36 2017 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Sun Feb 26 20:16:36 2017 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSPreemptionThread.java | 12 +++++++++++-
.../resourcemanager/scheduler/fair/FairScheduler.java | 5 +++++
2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/815d5350/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index af73c10..65df0c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;
+import java.util.concurrent.locks.Lock;
/**
* Thread that handles FairScheduler preemption.
@@ -43,6 +44,7 @@ class FSPreemptionThread extends Thread {
private final long warnTimeBeforeKill;
private final long delayBeforeNextStarvationCheck;
private final Timer preemptionTimer;
+ private final Lock schedulerReadLock;
FSPreemptionThread(FairScheduler scheduler) {
setDaemon(true);
@@ -61,6 +63,7 @@ class FSPreemptionThread extends Thread {
: 4 * scheduler.getNMHeartbeatInterval()); // 4 heartbeats
delayBeforeNextStarvationCheck = warnTimeBeforeKill + allocDelay +
fsConf.getWaitTimeBeforeNextStarvationCheck();
+ schedulerReadLock = scheduler.getSchedulerReadLock();
}
public void run() {
@@ -68,7 +71,14 @@ class FSPreemptionThread extends Thread {
FSAppAttempt starvedApp;
try{
starvedApp = context.getStarvedApps().take();
- preemptContainers(identifyContainersToPreempt(starvedApp));
+ // Hold the scheduler readlock so this is not concurrent with the
+ // update thread.
+ schedulerReadLock.lock();
+ try {
+ preemptContainers(identifyContainersToPreempt(starvedApp));
+ } finally {
+ schedulerReadLock.unlock();
+ }
starvedApp.preemptionTriggered(delayBeforeNextStarvationCheck);
} catch (InterruptedException e) {
LOG.info("Preemption thread interrupted! Exiting.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/815d5350/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index c946bfb..3246778 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -104,6 +104,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
/**
* A scheduler that schedules resources between a set of queues. The scheduler
@@ -1782,4 +1783,8 @@ public class FairScheduler extends
long getNMHeartbeatInterval() {
return nmHeartbeatInterval;
}
+
+ ReadLock getSchedulerReadLock() {
+ return this.readLock;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-4779. Fix AM container
allocation logic in SLS. Contributed by Wangda Tan.
Posted by jh...@apache.org.
YARN-4779. Fix AM container allocation logic in SLS. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b32ffa27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b32ffa27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b32ffa27
Branch: refs/heads/YARN-5734
Commit: b32ffa2753e83615b980721b6067fcc35ce54372
Parents: e8694de
Author: Sunil G <su...@apache.org>
Authored: Fri Feb 24 21:39:25 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Fri Feb 24 21:39:25 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 20 +-
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 89 +++++---
.../yarn/sls/appmaster/MRAMSimulator.java | 218 ++++++++-----------
.../sls/resourcemanager/MockAMLauncher.java | 115 ++++++++++
.../sls/scheduler/SLSCapacityScheduler.java | 24 ++
5 files changed, 305 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ffa27/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 61738fb..61b7f36 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -55,12 +56,14 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.sls.appmaster.AMSimulator;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.sls.nodemanager.NMSimulator;
+import org.apache.hadoop.yarn.sls.resourcemanager.MockAMLauncher;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
import org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper;
import org.apache.hadoop.yarn.sls.scheduler.SLSCapacityScheduler;
@@ -119,10 +122,10 @@ public class SLSRunner {
this.printSimulation = printsimulation;
metricsOutputDir = outputDir;
- nmMap = new HashMap<NodeId, NMSimulator>();
- queueAppNumMap = new HashMap<String, Integer>();
- amMap = new HashMap<String, AMSimulator>();
- amClassMap = new HashMap<String, Class>();
+ nmMap = new HashMap<>();
+ queueAppNumMap = new HashMap<>();
+ amMap = new ConcurrentHashMap<>();
+ amClassMap = new HashMap<>();
// runner configuration
conf = new Configuration(false);
@@ -179,7 +182,14 @@ public class SLSRunner {
}
rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
- rm = new ResourceManager();
+
+ final SLSRunner se = this;
+ rm = new ResourceManager() {
+ @Override
+ protected ApplicationMasterLauncher createAMLauncher() {
+ return new MockAMLauncher(se, this.rmContext, amMap);
+ }
+ };
rm.init(rmConf);
rm.start();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ffa27/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
index d61bf02..5b03d51 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -66,6 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.log4j.Logger;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
@@ -107,11 +109,19 @@ public abstract class AMSimulator extends TaskRunner.Task {
// progress
protected int totalContainers;
protected int finishedContainers;
+
+ // waiting for AM container
+ volatile boolean isAMContainerRunning = false;
+ volatile Container amContainer;
protected final Logger LOG = Logger.getLogger(AMSimulator.class);
-
+
+ // resource for AM container
+ private final static int MR_AM_CONTAINER_RESOURCE_MEMORY_MB = 1024;
+ private final static int MR_AM_CONTAINER_RESOURCE_VCORES = 1;
+
public AMSimulator() {
- this.responseQueue = new LinkedBlockingQueue<AllocateResponse>();
+ this.responseQueue = new LinkedBlockingQueue<>();
}
public void init(int id, int heartbeatInterval,
@@ -142,23 +152,30 @@ public abstract class AMSimulator extends TaskRunner.Task {
// submit application, waiting until ACCEPTED
submitApp();
- // register application master
- registerAM();
-
// track app metrics
trackApp();
}
+ public synchronized void notifyAMContainerLaunched(Container masterContainer)
+ throws Exception {
+ this.amContainer = masterContainer;
+ this.appAttemptId = masterContainer.getId().getApplicationAttemptId();
+ registerAM();
+ isAMContainerRunning = true;
+ }
+
@Override
public void middleStep() throws Exception {
- // process responses in the queue
- processResponseQueue();
-
- // send out request
- sendContainerRequest();
-
- // check whether finish
- checkStop();
+ if (isAMContainerRunning) {
+ // process responses in the queue
+ processResponseQueue();
+
+ // send out request
+ sendContainerRequest();
+
+ // check whether finish
+ checkStop();
+ }
}
@Override
@@ -168,6 +185,22 @@ public abstract class AMSimulator extends TaskRunner.Task {
if (isTracked) {
untrackApp();
}
+
+ // Finish AM container
+ if (amContainer != null) {
+ LOG.info("AM container = " + amContainer.getId() + " reported to finish");
+ se.getNmMap().get(amContainer.getNodeId()).cleanupContainer(
+ amContainer.getId());
+ } else {
+ LOG.info("AM container is null");
+ }
+
+ if (null == appAttemptId) {
+ // If appAttemptId == null, AM is not launched from RM's perspective, so
+ // it's unnecessary to finish am as well
+ return;
+ }
+
// unregister application master
final FinishApplicationMasterRequest finishAMRequest = recordFactory
.newRecordInstance(FinishApplicationMasterRequest.class);
@@ -256,7 +289,9 @@ public abstract class AMSimulator extends TaskRunner.Task {
conLauContext.setLocalResources(new HashMap<String, LocalResource>());
conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
appSubContext.setAMContainerSpec(conLauContext);
- appSubContext.setUnmanagedAM(true);
+ appSubContext.setResource(Resources
+ .createResource(MR_AM_CONTAINER_RESOURCE_MEMORY_MB,
+ MR_AM_CONTAINER_RESOURCE_VCORES));
subAppRequest.setApplicationSubmissionContext(appSubContext);
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
ugi.doAs(new PrivilegedExceptionAction<Object>() {
@@ -267,22 +302,6 @@ public abstract class AMSimulator extends TaskRunner.Task {
}
});
LOG.info(MessageFormat.format("Submit a new application {0}", appId));
-
- // waiting until application ACCEPTED
- RMApp app = rm.getRMContext().getRMApps().get(appId);
- while(app.getState() != RMAppState.ACCEPTED) {
- Thread.sleep(10);
- }
-
- // Waiting until application attempt reach LAUNCHED
- // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
- this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
- .getCurrentAppAttempt().getAppAttemptId();
- RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
- .getCurrentAppAttempt();
- while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
- Thread.sleep(10);
- }
}
private void registerAM()
@@ -335,7 +354,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
for (ContainerSimulator cs : csList) {
String rackHostNames[] = SLSUtils.getRackHostName(cs.getHostname());
// check rack local
- String rackname = rackHostNames[0];
+ String rackname = "/" + rackHostNames[0];
if (rackLocalRequestMap.containsKey(rackname)) {
rackLocalRequestMap.get(rackname).setNumContainers(
rackLocalRequestMap.get(rackname).getNumContainers() + 1);
@@ -383,4 +402,12 @@ public abstract class AMSimulator extends TaskRunner.Task {
public int getNumTasks() {
return totalContainers;
}
+
+ public ApplicationId getApplicationId() {
+ return appId;
+ }
+
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ffa27/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
index da267a1..e726b09 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
@@ -27,6 +27,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.avro.Protocol;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
@@ -63,10 +64,10 @@ public class MRAMSimulator extends AMSimulator {
private static final int PRIORITY_REDUCE = 10;
private static final int PRIORITY_MAP = 20;
-
+
// pending maps
private LinkedList<ContainerSimulator> pendingMaps =
- new LinkedList<ContainerSimulator>();
+ new LinkedList<>();
// pending failed maps
private LinkedList<ContainerSimulator> pendingFailedMaps =
@@ -107,14 +108,9 @@ public class MRAMSimulator extends AMSimulator {
private int mapTotal = 0;
private int reduceFinished = 0;
private int reduceTotal = 0;
- // waiting for AM container
- private boolean isAMContainerRunning = false;
- private Container amContainer;
+
// finished
private boolean isFinished = false;
- // resource for AM container
- private final static int MR_AM_CONTAINER_RESOURCE_MEMORY_MB = 1024;
- private final static int MR_AM_CONTAINER_RESOURCE_VCORES = 1;
public final Logger LOG = Logger.getLogger(MRAMSimulator.class);
@@ -131,83 +127,34 @@ public class MRAMSimulator extends AMSimulator {
for (ContainerSimulator cs : containerList) {
if (cs.getType().equals("map")) {
cs.setPriority(PRIORITY_MAP);
- pendingMaps.add(cs);
+ allMaps.add(cs);
} else if (cs.getType().equals("reduce")) {
cs.setPriority(PRIORITY_REDUCE);
- pendingReduces.add(cs);
+ allReduces.add(cs);
}
}
- allMaps.addAll(pendingMaps);
- allReduces.addAll(pendingReduces);
- mapTotal = pendingMaps.size();
- reduceTotal = pendingReduces.size();
+
+ LOG.info(MessageFormat
+ .format("Added new job with {0} mapper and {1} reducers",
+ allMaps.size(), allReduces.size()));
+
+ mapTotal = allMaps.size();
+ reduceTotal = allReduces.size();
totalContainers = mapTotal + reduceTotal;
}
@Override
- public void firstStep() throws Exception {
- super.firstStep();
-
- requestAMContainer();
- }
-
- /**
- * send out request for AM container
- */
- protected void requestAMContainer()
- throws YarnException, IOException, InterruptedException {
- List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
- ResourceRequest amRequest = createResourceRequest(
- BuilderUtils.newResource(MR_AM_CONTAINER_RESOURCE_MEMORY_MB,
- MR_AM_CONTAINER_RESOURCE_VCORES),
- ResourceRequest.ANY, 1, 1);
- ask.add(amRequest);
- LOG.debug(MessageFormat.format("Application {0} sends out allocate " +
- "request for its AM", appId));
- final AllocateRequest request = this.createAllocateRequest(ask);
-
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps()
- .get(appAttemptId.getApplicationId())
- .getRMAppAttempt(appAttemptId).getAMRMToken();
- ugi.addTokenIdentifier(token.decodeIdentifier());
- AllocateResponse response = ugi.doAs(
- new PrivilegedExceptionAction<AllocateResponse>() {
- @Override
- public AllocateResponse run() throws Exception {
- return rm.getApplicationMasterService().allocate(request);
- }
- });
- if (response != null) {
- responseQueue.put(response);
+ public synchronized void notifyAMContainerLaunched(Container masterContainer)
+ throws Exception {
+ if (null != masterContainer) {
+ restart();
+ super.notifyAMContainerLaunched(masterContainer);
}
}
@Override
@SuppressWarnings("unchecked")
- protected void processResponseQueue()
- throws InterruptedException, YarnException, IOException {
- // Check whether receive the am container
- if (!isAMContainerRunning) {
- if (!responseQueue.isEmpty()) {
- AllocateResponse response = responseQueue.take();
- if (response != null
- && !response.getAllocatedContainers().isEmpty()) {
- // Get AM container
- Container container = response.getAllocatedContainers().get(0);
- se.getNmMap().get(container.getNodeId())
- .addNewContainer(container, -1L);
- // Start AM container
- amContainer = container;
- LOG.debug(MessageFormat.format("Application {0} starts its " +
- "AM container ({1}).", appId, amContainer.getId()));
- isAMContainerRunning = true;
- }
- }
- return;
- }
-
+ protected void processResponseQueue() throws Exception {
while (! responseQueue.isEmpty()) {
AllocateResponse response = responseQueue.take();
@@ -228,12 +175,16 @@ public class MRAMSimulator extends AMSimulator {
assignedReduces.remove(containerId);
reduceFinished ++;
finishedContainers ++;
- } else {
+ } else if (amContainer.getId().equals(containerId)){
// am container released event
isFinished = true;
LOG.info(MessageFormat.format("Application {0} goes to " +
"finish.", appId));
}
+
+ if (mapFinished >= mapTotal && reduceFinished >= reduceTotal) {
+ lastStep();
+ }
} else {
// container to be killed
if (assignedMaps.containsKey(containerId)) {
@@ -244,10 +195,9 @@ public class MRAMSimulator extends AMSimulator {
LOG.debug(MessageFormat.format("Application {0} has one " +
"reducer killed ({1}).", appId, containerId));
pendingFailedReduces.add(assignedReduces.remove(containerId));
- } else {
+ } else if (amContainer.getId().equals(containerId)){
LOG.info(MessageFormat.format("Application {0}'s AM is " +
- "going to be killed. Restarting...", appId));
- restart();
+ "going to be killed. Waiting for rescheduling...", appId));
}
}
}
@@ -255,11 +205,8 @@ public class MRAMSimulator extends AMSimulator {
// check finished
if (isAMContainerRunning &&
- (mapFinished == mapTotal) &&
- (reduceFinished == reduceTotal)) {
- // to release the AM container
- se.getNmMap().get(amContainer.getNodeId())
- .cleanupContainer(amContainer.getId());
+ (mapFinished >= mapTotal) &&
+ (reduceFinished >= reduceTotal)) {
isAMContainerRunning = false;
LOG.debug(MessageFormat.format("Application {0} sends out event " +
"to clean up its AM container.", appId));
@@ -293,21 +240,38 @@ public class MRAMSimulator extends AMSimulator {
*/
private void restart()
throws YarnException, IOException, InterruptedException {
- // clear
- finishedContainers = 0;
+ // clear
isFinished = false;
- mapFinished = 0;
- reduceFinished = 0;
pendingFailedMaps.clear();
pendingMaps.clear();
pendingReduces.clear();
pendingFailedReduces.clear();
- pendingMaps.addAll(allMaps);
- pendingReduces.addAll(pendingReduces);
- isAMContainerRunning = false;
+
+ // Only add totalMaps - finishedMaps
+ int added = 0;
+ for (ContainerSimulator cs : allMaps) {
+ if (added >= mapTotal - mapFinished) {
+ break;
+ }
+ pendingMaps.add(cs);
+ }
+
+ // And same, only add totalReduces - finishedReduces
+ added = 0;
+ for (ContainerSimulator cs : allReduces) {
+ if (added >= reduceTotal - reduceFinished) {
+ break;
+ }
+ pendingReduces.add(cs);
+ }
amContainer = null;
- // resent am container request
- requestAMContainer();
+ }
+
+ private List<ContainerSimulator> mergeLists(List<ContainerSimulator> left, List<ContainerSimulator> right) {
+ List<ContainerSimulator> list = new ArrayList<>();
+ list.addAll(left);
+ list.addAll(right);
+ return list;
}
@Override
@@ -319,44 +283,48 @@ public class MRAMSimulator extends AMSimulator {
// send out request
List<ResourceRequest> ask = null;
- if (isAMContainerRunning) {
- if (mapFinished != mapTotal) {
- // map phase
- if (! pendingMaps.isEmpty()) {
- ask = packageRequests(pendingMaps, PRIORITY_MAP);
- LOG.debug(MessageFormat.format("Application {0} sends out " +
- "request for {1} mappers.", appId, pendingMaps.size()));
- scheduledMaps.addAll(pendingMaps);
- pendingMaps.clear();
- } else if (! pendingFailedMaps.isEmpty() && scheduledMaps.isEmpty()) {
- ask = packageRequests(pendingFailedMaps, PRIORITY_MAP);
- LOG.debug(MessageFormat.format("Application {0} sends out " +
- "requests for {1} failed mappers.", appId,
- pendingFailedMaps.size()));
- scheduledMaps.addAll(pendingFailedMaps);
- pendingFailedMaps.clear();
- }
- } else if (reduceFinished != reduceTotal) {
- // reduce phase
- if (! pendingReduces.isEmpty()) {
- ask = packageRequests(pendingReduces, PRIORITY_REDUCE);
- LOG.debug(MessageFormat.format("Application {0} sends out " +
- "requests for {1} reducers.", appId, pendingReduces.size()));
- scheduledReduces.addAll(pendingReduces);
- pendingReduces.clear();
- } else if (! pendingFailedReduces.isEmpty()
- && scheduledReduces.isEmpty()) {
- ask = packageRequests(pendingFailedReduces, PRIORITY_REDUCE);
- LOG.debug(MessageFormat.format("Application {0} sends out " +
- "request for {1} failed reducers.", appId,
- pendingFailedReduces.size()));
- scheduledReduces.addAll(pendingFailedReduces);
- pendingFailedReduces.clear();
- }
+ if (mapFinished != mapTotal) {
+ // map phase
+ if (!pendingMaps.isEmpty()) {
+ ask = packageRequests(mergeLists(pendingMaps, scheduledMaps),
+ PRIORITY_MAP);
+ LOG.debug(MessageFormat
+ .format("Application {0} sends out " + "request for {1} mappers.",
+ appId, pendingMaps.size()));
+ scheduledMaps.addAll(pendingMaps);
+ pendingMaps.clear();
+ } else if (!pendingFailedMaps.isEmpty()) {
+ ask = packageRequests(mergeLists(pendingFailedMaps, scheduledMaps),
+ PRIORITY_MAP);
+ LOG.debug(MessageFormat.format(
+ "Application {0} sends out " + "requests for {1} failed mappers.",
+ appId, pendingFailedMaps.size()));
+ scheduledMaps.addAll(pendingFailedMaps);
+ pendingFailedMaps.clear();
+ }
+ } else if (reduceFinished != reduceTotal) {
+ // reduce phase
+ if (!pendingReduces.isEmpty()) {
+ ask = packageRequests(mergeLists(pendingReduces, scheduledReduces),
+ PRIORITY_REDUCE);
+ LOG.debug(MessageFormat
+ .format("Application {0} sends out " + "requests for {1} reducers.",
+ appId, pendingReduces.size()));
+ scheduledReduces.addAll(pendingReduces);
+ pendingReduces.clear();
+ } else if (!pendingFailedReduces.isEmpty()) {
+ ask = packageRequests(mergeLists(pendingFailedReduces, scheduledReduces),
+ PRIORITY_REDUCE);
+ LOG.debug(MessageFormat.format(
+ "Application {0} sends out " + "request for {1} failed reducers.",
+ appId, pendingFailedReduces.size()));
+ scheduledReduces.addAll(pendingFailedReduces);
+ pendingFailedReduces.clear();
}
}
+
if (ask == null) {
- ask = new ArrayList<ResourceRequest>();
+ ask = new ArrayList<>();
}
final AllocateRequest request = createAllocateRequest(ask);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ffa27/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java
new file mode 100644
index 0000000..20cf3e5
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls.resourcemanager;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
+import org.apache.hadoop.yarn.sls.SLSRunner;
+import org.apache.hadoop.yarn.sls.appmaster.AMSimulator;
+
+import java.util.Map;
+
+public class MockAMLauncher extends ApplicationMasterLauncher
+ implements EventHandler<AMLauncherEvent> {
+ private static final Log LOG = LogFactory.getLog(
+ MockAMLauncher.class);
+
+ Map<String, AMSimulator> amMap;
+ SLSRunner se;
+
+ public MockAMLauncher(SLSRunner se, RMContext rmContext,
+ Map<String, AMSimulator> amMap) {
+ super(rmContext);
+ this.amMap = amMap;
+ this.se = se;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ // Do nothing
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ // Do nothing
+ }
+
+ private void setupAMRMToken(RMAppAttempt appAttempt) {
+ // Setup AMRMToken
+ Token<AMRMTokenIdentifier> amrmToken =
+ super.context.getAMRMTokenSecretManager().createAndGetAMRMToken(
+ appAttempt.getAppAttemptId());
+ ((RMAppAttemptImpl) appAttempt).setAMRMToken(amrmToken);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void handle(AMLauncherEvent event) {
+ if (AMLauncherEventType.LAUNCH == event.getType()) {
+ ApplicationId appId =
+ event.getAppAttempt().getAppAttemptId().getApplicationId();
+
+ // find AMSimulator
+ for (AMSimulator ams : amMap.values()) {
+ if (ams.getApplicationId() != null && ams.getApplicationId().equals(
+ appId)) {
+ try {
+ Container amContainer = event.getAppAttempt().getMasterContainer();
+
+ setupAMRMToken(event.getAppAttempt());
+
+ // Notify RMAppAttempt to change state
+ super.context.getDispatcher().getEventHandler().handle(
+ new RMAppAttemptEvent(event.getAppAttempt().getAppAttemptId(),
+ RMAppAttemptEventType.LAUNCHED));
+
+ ams.notifyAMContainerLaunched(
+ event.getAppAttempt().getMasterContainer());
+ LOG.info("Notify AM launcher launched:" + amContainer.getId());
+
+ se.getNmMap().get(amContainer.getNodeId())
+ .addNewContainer(amContainer, 100000000L);
+
+ return;
+ } catch (Exception e) {
+ throw new YarnRuntimeException(e);
+ }
+ }
+ }
+
+ throw new YarnRuntimeException(
+ "Didn't find any AMSimulator for applicationId=" + appId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ffa27/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index 8388273..cd4377e 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -556,6 +556,30 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
}
}
);
+ metrics.register("variable.cluster.reserved.memory",
+ new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ if(getRootQueueMetrics() == null) {
+ return 0L;
+ } else {
+ return getRootQueueMetrics().getReservedMB();
+ }
+ }
+ }
+ );
+ metrics.register("variable.cluster.reserved.vcores",
+ new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ if(getRootQueueMetrics() == null) {
+ return 0;
+ } else {
+ return getRootQueueMetrics().getReservedVirtualCores();
+ }
+ }
+ }
+ );
}
private void registerContainerAppNumMetrics() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HADOOP-14127. Add log4j
configuration to enable logging in hadoop-distcp's tests. (Xiao Chen via
Yongjun Zhang)
Posted by jh...@apache.org.
HADOOP-14127. Add log4j configuration to enable logging in hadoop-distcp's tests. (Xiao Chen via Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d235dcdf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d235dcdf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d235dcdf
Branch: refs/heads/YARN-5734
Commit: d235dcdf0b6f36162280e32c7c0342b7970c9b7b
Parents: 55c07bb
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Mon Feb 27 20:42:13 2017 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Mon Feb 27 20:42:13 2017 -0800
----------------------------------------------------------------------
.../src/test/resources/log4j.properties | 22 ++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d235dcdf/hadoop-tools/hadoop-distcp/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/resources/log4j.properties b/hadoop-tools/hadoop-distcp/src/test/resources/log4j.properties
new file mode 100644
index 0000000..22990ca
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=debug,stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HADOOP-14119. Remove unused
imports from GzipCodec.java. Contributed by Yiqun Lin.
Posted by jh...@apache.org.
HADOOP-14119. Remove unused imports from GzipCodec.java. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9db2e0c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9db2e0c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9db2e0c8
Branch: refs/heads/YARN-5734
Commit: 9db2e0c8d4b29971da60e2628ff55b5dabed2f7b
Parents: fbfe86d
Author: Yiqun Lin <yq...@apache.org>
Authored: Mon Feb 27 18:39:14 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Mon Feb 27 18:39:14 2017 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/io/compress/GzipCodec.java | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9db2e0c8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
index d079412..11fcf60 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
@@ -18,18 +18,21 @@
package org.apache.hadoop.io.compress;
-import java.io.*;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.util.zip.GZIPOutputStream;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.compress.DefaultCodec;
-import org.apache.hadoop.io.compress.zlib.*;
-import org.apache.hadoop.io.compress.zlib.ZlibDecompressor.ZlibDirectDecompressor;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
-import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
+import org.apache.hadoop.io.compress.zlib.BuiltInGzipDecompressor;
+import org.apache.hadoop.io.compress.zlib.ZlibCompressor;
+import org.apache.hadoop.io.compress.zlib.ZlibDecompressor;
+import org.apache.hadoop.io.compress.zlib.ZlibFactory;
/**
* This class creates gzip compressors/decompressors.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: Revert "HDFS-11477. Simplify file
IO profiling configuration. Contributed by Hanisha Koneru."
Posted by jh...@apache.org.
Revert "HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru."
This reverts commit e61491d4769536ce8b77b416005ce531278ce729.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f4817f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f4817f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f4817f2
Branch: refs/heads/YARN-5734
Commit: 8f4817f2c5524c1891a3b3b50e396dd4ca294a2d
Parents: e61491d
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Mar 2 16:20:48 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Mar 2 16:20:48 2017 -0800
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 7 +----
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 ++++-
.../hdfs/server/datanode/FileIoProvider.java | 2 +-
.../server/datanode/ProfilingFileIoEvents.java | 27 +++++---------------
.../datanode/TestDataNodeVolumeMetrics.java | 4 +--
.../hadoop/tools/TestHdfsConfigFields.java | 2 ++
6 files changed, 17 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index a8bdbeb..7900692 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -332,12 +332,7 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
FsVolume
--------
-Per-volume metrics contain Datanode Volume IO related statistics. Per-volume
-metrics are off by default. They can be enabled by setting `dfs.datanode
-.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0.
-Setting this value to 0.0 would mean profiling is not enabled. But enabling
-per-volume metrics may have a performance impact. Each metrics record
-contains tags such as Hostname as additional information along with metrics.
+Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics.
| Name | Description |
|:---- |:---- |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 82d6073..be20829 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -718,6 +718,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT = 0; //no throttling
// Datanode File IO Stats
+ public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
+ "dfs.datanode.enable.fileio.profiling";
+ public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
+ false;
public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY =
"dfs.datanode.enable.fileio.fault.injection";
public static final boolean
@@ -726,7 +730,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
"dfs.datanode.fileio.profiling.sampling.fraction";
public static final double
- DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0;
+ DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
//Keys with no defaults
public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index 5508e0b..9def2e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*;
*
* Behavior can be injected into these events by enabling the
* profiling and/or fault injection event hooks through
- * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and
+ * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and
* {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}.
* These event hooks are disabled by default.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 7eab969..9af84fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.datanode;
-import static java.lang.Math.abs;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -44,32 +43,18 @@ class ProfilingFileIoEvents {
public ProfilingFileIoEvents(@Nullable Configuration conf) {
if (conf != null) {
+ isEnabled = conf.getBoolean(DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT);
- if (abs(fileIOSamplingFraction) < 0.000001) {
- LOG.info(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
- + fileIOSamplingFraction + ". Disabling file IO profiling");
- isEnabled = false;
- } else if (fileIOSamplingFraction < 0.000001) {
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
+ if (fileIOSamplingFraction > 1) {
LOG.warn(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
- " value cannot be less than 0. Disabling file IO profiling.");
- isEnabled = false;
- } else if (fileIOSamplingFraction > 1) {
- LOG.warn(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
- " value cannot be more than 1. Setting value to 1 and enabling " +
- "file IO profiling");
- isEnabled = true;
+ " value cannot be more than 1. Setting value to 1");
fileIOSamplingFraction = 1;
- } else {
- LOG.info(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
- + fileIOSamplingFraction + ". Enabling file IO profiling");
- isEnabled = true;
}
sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
index 03e1dee..6a8ac9c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
@@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics {
private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException {
Configuration conf = new HdfsConfiguration();
- conf.setDouble(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
+ conf.setBoolean(DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true);
SimulatedFSDataset.setFactory(conf);
return new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATANODES)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 1fdf713..cdce342 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -103,6 +103,8 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
.add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
configurationPropsToSkipCompare
.add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
.DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDFS-11466. Change
dfs.namenode.write-lock-reporting-threshold-ms default from 1000ms to 5000ms.
Contributed by Andrew Wang.
Posted by jh...@apache.org.
HDFS-11466. Change dfs.namenode.write-lock-reporting-threshold-ms default from 1000ms to 5000ms. Contributed by Andrew Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d269b488
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d269b488
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d269b488
Branch: refs/heads/YARN-5734
Commit: d269b488a71a158d3ddcbdea96992abe29725c69
Parents: 0f35443
Author: Zhe Zhang <zh...@apache.org>
Authored: Tue Feb 28 09:00:49 2017 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Feb 28 09:00:49 2017 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d269b488/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index cfd16aa..68cef36 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -425,7 +425,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
// event to be logged
public static final String DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY =
"dfs.namenode.write-lock-reporting-threshold-ms";
- public static final long DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT = 1000L;
+ public static final long DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT = 5000L;
public static final String DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY =
"dfs.namenode.read-lock-reporting-threshold-ms";
public static final long DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT = 5000L;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d269b488/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index d23b967..c220025 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2720,7 +2720,7 @@
<property>
<name>dfs.namenode.write-lock-reporting-threshold-ms</name>
- <value>1000</value>
+ <value>5000</value>
<description>When a write lock is held on the namenode for a long time,
this will be logged as the lock is released. This sets how long the
lock must be held for logging to occur.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDFS-11432. Federation : Support
fully qualified path for Quota/Snapshot/cacheadmin/cryptoadmin commands.
Contributed by Brahma Reddy Battula.
Posted by jh...@apache.org.
HDFS-11432. Federation : Support fully qualified path for Quota/Snapshot/cacheadmin/cryptoadmin commands. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcd03df9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcd03df9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcd03df9
Branch: refs/heads/YARN-5734
Commit: dcd03df9f9e0080d7e179060ffc8148336c31b3e
Parents: 989bd56
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Wed Mar 1 10:45:56 2017 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Wed Mar 1 10:45:56 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/fs/shell/Command.java | 14 ++++-
.../apache/hadoop/hdfs/tools/CacheAdmin.java | 8 +--
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 18 +++----
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 57 +++++++++++---------
.../hdfs/tools/snapshot/SnapshotDiff.java | 19 ++++++-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 26 +++++++++
.../hadoop/hdfs/TestSnapshotCommands.java | 34 ++++++++++++
7 files changed, 136 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
index c573aa0..4c5cbad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
@@ -101,7 +101,17 @@ abstract public class Command extends Configured {
* @throws IOException if any error occurs
*/
abstract protected void run(Path path) throws IOException;
-
+
+ /**
+ * Execute the command on the input path data. Commands can override to make
+ * use of the resolved filesystem.
+ * @param pathData The input path with resolved filesystem
+ * @throws IOException
+ */
+ protected void run(PathData pathData) throws IOException {
+ run(pathData.path);
+ }
+
/**
* For each source path, execute the command
*
@@ -113,7 +123,7 @@ abstract public class Command extends Configured {
try {
PathData[] srcs = PathData.expandAsGlob(src, getConf());
for (PathData s : srcs) {
- run(s.path);
+ run(s);
}
} catch (IOException e) {
exitCode = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
index 522f701..d8cbfc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
@@ -189,8 +189,9 @@ public class CacheAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
-
- DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+
+ DistributedFileSystem dfs =
+ AdminHelper.getDFS(new Path(path).toUri(), conf);
CacheDirectiveInfo directive = builder.build();
EnumSet<CacheFlag> flags = EnumSet.noneOf(CacheFlag.class);
if (force) {
@@ -409,7 +410,8 @@ public class CacheAdmin extends Configured implements Tool {
}
int exitCode = 0;
try {
- DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+ DistributedFileSystem dfs =
+ AdminHelper.getDFS(new Path(path).toUri(), conf);
RemoteIterator<CacheDirectiveEntry> iter =
dfs.listCacheDirectives(
new CacheDirectiveInfo.Builder().
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
index 225f11a..4c7335f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
@@ -139,12 +139,12 @@ public class CryptoAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
-
- HdfsAdmin admin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ Path p = new Path(path);
+ HdfsAdmin admin = new HdfsAdmin(p.toUri(), conf);
EnumSet<CreateEncryptionZoneFlag> flags =
EnumSet.of(CreateEncryptionZoneFlag.PROVISION_TRASH);
try {
- admin.createEncryptionZone(new Path(path), keyName, flags);
+ admin.createEncryptionZone(p, keyName, flags);
System.out.println("Added encryption zone " + path);
} catch (IOException e) {
System.err.println(prettifyException(e));
@@ -226,12 +226,12 @@ public class CryptoAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
-
+ Path p = new Path(path);
final HdfsAdmin admin =
- new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ new HdfsAdmin(p.toUri(), conf);
try {
final FileEncryptionInfo fei =
- admin.getFileEncryptionInfo(new Path(path));
+ admin.getFileEncryptionInfo(p);
if (fei == null) {
System.out.println("No FileEncryptionInfo found for path " + path);
return 2;
@@ -273,10 +273,10 @@ public class CryptoAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
-
- HdfsAdmin admin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ Path p = new Path(path);
+ HdfsAdmin admin = new HdfsAdmin(p.toUri(), conf);
try {
- admin.provisionEncryptionZoneTrash(new Path(path));
+ admin.provisionEncryptionZoneTrash(p);
System.out.println("Created a trash directory for " + path);
} catch (IOException ioe) {
System.err.println(prettifyException(ioe));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index 4dd9d7e..c1f79e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFormat;
+import org.apache.hadoop.fs.shell.PathData;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HAUtilClient;
@@ -109,15 +110,21 @@ public class DFSAdmin extends FsShell {
* An abstract class for the execution of a file system command
*/
abstract private static class DFSAdminCommand extends Command {
- final DistributedFileSystem dfs;
+ protected DistributedFileSystem dfs;
/** Constructor */
- public DFSAdminCommand(FileSystem fs) {
- super(fs.getConf());
+ public DFSAdminCommand(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public void run(PathData pathData) throws IOException {
+ FileSystem fs = pathData.fs;
if (!(fs instanceof DistributedFileSystem)) {
- throw new IllegalArgumentException("FileSystem " + fs.getUri() +
- " is not an HDFS file system");
+ throw new IllegalArgumentException("FileSystem " + fs.getUri()
+ + " is not an HDFS file system");
}
- this.dfs = (DistributedFileSystem)fs;
+ this.dfs = (DistributedFileSystem) fs;
+ run(pathData.path);
}
}
@@ -133,8 +140,8 @@ public class DFSAdmin extends FsShell {
"\t\tIt does not fault if the directory has no quota.";
/** Constructor */
- ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
- super(fs);
+ ClearQuotaCommand(String[] args, int pos, Configuration conf) {
+ super(conf);
CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
List<String> parameters = c.parse(args, pos);
this.args = parameters.toArray(new String[parameters.size()]);
@@ -179,8 +186,8 @@ public class DFSAdmin extends FsShell {
private final long quota; // the quota to be set
/** Constructor */
- SetQuotaCommand(String[] args, int pos, FileSystem fs) {
- super(fs);
+ SetQuotaCommand(String[] args, int pos, Configuration conf) {
+ super(conf);
CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
List<String> parameters = c.parse(args, pos);
this.quota = Long.parseLong(parameters.remove(0));
@@ -230,8 +237,8 @@ public class DFSAdmin extends FsShell {
private StorageType type;
/** Constructor */
- ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
- super(fs);
+ ClearSpaceQuotaCommand(String[] args, int pos, Configuration conf) {
+ super(conf);
CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
c.addOptionWithValue("storageType");
List<String> parameters = c.parse(args, pos);
@@ -294,8 +301,8 @@ public class DFSAdmin extends FsShell {
private StorageType type;
/** Constructor */
- SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
- super(fs);
+ SetSpaceQuotaCommand(String[] args, int pos, Configuration conf) {
+ super(conf);
CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
List<String> parameters = c.parse(args, pos);
String str = parameters.remove(0).trim();
@@ -705,10 +712,11 @@ public class DFSAdmin extends FsShell {
* @param argv List of of command line parameters.
* @exception IOException
*/
- public void allowSnapshot(String[] argv) throws IOException {
- DistributedFileSystem dfs = getDFS();
+ public void allowSnapshot(String[] argv) throws IOException {
+ Path p = new Path(argv[1]);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), getConf());
try {
- dfs.allowSnapshot(new Path(argv[1]));
+ dfs.allowSnapshot(p);
} catch (SnapshotException e) {
throw new RemoteException(e.getClass().getName(), e.getMessage());
}
@@ -721,10 +729,11 @@ public class DFSAdmin extends FsShell {
* @param argv List of of command line parameters.
* @exception IOException
*/
- public void disallowSnapshot(String[] argv) throws IOException {
- DistributedFileSystem dfs = getDFS();
+ public void disallowSnapshot(String[] argv) throws IOException {
+ Path p = new Path(argv[1]);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), getConf());
try {
- dfs.disallowSnapshot(new Path(argv[1]));
+ dfs.disallowSnapshot(p);
} catch (SnapshotException e) {
throw new RemoteException(e.getClass().getName(), e.getMessage());
}
@@ -2042,13 +2051,13 @@ public class DFSAdmin extends FsShell {
} else if ("-metasave".equals(cmd)) {
exitCode = metaSave(argv, i);
} else if (ClearQuotaCommand.matches(cmd)) {
- exitCode = new ClearQuotaCommand(argv, i, getDFS()).runAll();
+ exitCode = new ClearQuotaCommand(argv, i, getConf()).runAll();
} else if (SetQuotaCommand.matches(cmd)) {
- exitCode = new SetQuotaCommand(argv, i, getDFS()).runAll();
+ exitCode = new SetQuotaCommand(argv, i, getConf()).runAll();
} else if (ClearSpaceQuotaCommand.matches(cmd)) {
- exitCode = new ClearSpaceQuotaCommand(argv, i, getDFS()).runAll();
+ exitCode = new ClearSpaceQuotaCommand(argv, i, getConf()).runAll();
} else if (SetSpaceQuotaCommand.matches(cmd)) {
- exitCode = new SetSpaceQuotaCommand(argv, i, getDFS()).runAll();
+ exitCode = new SetSpaceQuotaCommand(argv, i, getConf()).runAll();
} else if ("-refreshServiceAcl".equals(cmd)) {
exitCode = refreshServiceAcl();
} else if ("-refreshUserToGroupsMappings".equals(cmd)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
index b87d69f..3838ca1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
@@ -20,10 +20,12 @@ package org.apache.hadoop.hdfs.tools.snapshot;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.util.Tool;
@@ -41,6 +43,19 @@ import org.apache.hadoop.util.ToolRunner;
*/
@InterfaceAudience.Private
public class SnapshotDiff extends Configured implements Tool {
+ /**
+ * Construct a SnapshotDiff object.
+ */
+ public SnapshotDiff() {
+ this(new HdfsConfiguration());
+ }
+
+ /**
+ * Construct a SnapshotDiff object.
+ */
+ public SnapshotDiff(Configuration conf) {
+ super(conf);
+ }
private static String getSnapshotName(String name) {
if (Path.CUR_DIR.equals(name)) { // current directory
return "";
@@ -72,8 +87,8 @@ public class SnapshotDiff extends Configured implements Tool {
System.err.println("Usage: \n" + description);
return 1;
}
-
- FileSystem fs = FileSystem.get(getConf());
+
+ FileSystem fs = FileSystem.get(new Path(argv[0]).toUri(), getConf());
if (! (fs instanceof DistributedFileSystem)) {
System.err.println(
"SnapshotDiff can only be used in DistributedFileSystem");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index d58dd9e..4bfb62c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -1246,6 +1246,32 @@ public class TestQuota {
-1);
}
+ /**
+ * Test to all the commands by passing the fully qualified path.
+ */
+ @Test(timeout = 30000)
+ public void testQuotaCommandsWithURI() throws Exception {
+ DFSAdmin dfsAdmin = new DFSAdmin(conf);
+ final Path dir = new Path("/" + this.getClass().getSimpleName(),
+ GenericTestUtils.getMethodName());
+ assertTrue(dfs.mkdirs(dir));
+
+ /* set space quota */
+ testSetAndClearSpaceQuotaRegularInternal(
+ new String[] { "-setSpaceQuota", "1024",
+ dfs.getUri() + "/" + dir.toString() }, dir, 0, 1024);
+
+ /* clear space quota */
+ testSetAndClearSpaceQuotaRegularInternal(
+ new String[] { "-clrSpaceQuota", dfs.getUri() + "/" + dir.toString() },
+ dir, 0, -1);
+ runCommand(dfsAdmin, false, "-setQuota", "1000",
+ dfs.getUri() + "/" + dir.toString());
+
+ runCommand(dfsAdmin, false, "-clrQuota",
+ dfs.getUri() + "/" + dir.toString());
+ }
+
private void testSetAndClearSpaceQuotaRegularInternal(
final String[] args,
final Path dir,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
index eec4e99..1d5e071 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -60,6 +62,7 @@ public class TestSnapshotCommands {
@Before
public void setUp() throws IOException {
fs.mkdirs(new Path("/sub1"));
+ fs.mkdirs(new Path("/Fully/QPath"));
fs.allowSnapshot(new Path("/sub1"));
fs.mkdirs(new Path("/sub1/sub1sub1"));
fs.mkdirs(new Path("/sub1/sub1sub2"));
@@ -161,4 +164,35 @@ public class TestSnapshotCommands {
// now it can be deleted
DFSTestUtil.FsShellRun("-rmr /sub1", conf);
}
+
+ @Test (timeout=60000)
+ public void testSnapshotCommandsWithURI()throws Exception {
+ Configuration config = new HdfsConfiguration();
+ //fs.defaultFS should not be used, when path is fully qualified.
+ config.set("fs.defaultFS", "hdfs://127.0.0.1:1024");
+ String path = fs.getUri() + "/Fully/QPath";
+ DFSTestUtil.DFSAdminRun("-allowSnapshot " + path, 0,
+ "Allowing snaphot on " + path + " succeeded", config);
+ DFSTestUtil.FsShellRun("-createSnapshot " + path + " sn1", config);
+ // create file1
+ DFSTestUtil
+ .createFile(fs, new Path(fs.getUri() + "/Fully/QPath/File1"), 1024,
+ (short) 1, 100);
+ // create file2
+ DFSTestUtil
+ .createFile(fs, new Path(fs.getUri() + "/Fully/QPath/File2"), 1024,
+ (short) 1, 100);
+ DFSTestUtil.FsShellRun("-createSnapshot " + path + " sn2", config);
+ // verify the snapshotdiff using api and command line
+ SnapshotDiffReport report =
+ fs.getSnapshotDiffReport(new Path(path), "sn1", "sn2");
+ DFSTestUtil.toolRun(new SnapshotDiff(config), path + " sn1 sn2", 0,
+ report.toString());
+ DFSTestUtil.FsShellRun("-renameSnapshot " + path + " sn2 sn3", config);
+ DFSTestUtil.FsShellRun("-deleteSnapshot " + path + " sn1", config);
+ DFSTestUtil.FsShellRun("-deleteSnapshot " + path + " sn3", config);
+ DFSTestUtil.DFSAdminRun("-disallowSnapshot " + path, 0,
+ "Disallowing snaphot on " + path + " succeeded", config);
+ fs.delete(new Path("/Fully/QPath"), true);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HADOOP-14097. Remove Java6
specific code from GzipCodec.java. Contributed by Elek, Marton.
Posted by jh...@apache.org.
HADOOP-14097. Remove Java6 specific code from GzipCodec.java. Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50decd36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50decd36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50decd36
Branch: refs/heads/YARN-5734
Commit: 50decd36130945e184734dcd55b8912be6f4550a
Parents: e60c654
Author: Akira Ajisaka <aa...@apache.org>
Authored: Sat Feb 25 00:28:31 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Sat Feb 25 00:28:31 2017 +0900
----------------------------------------------------------------------
.../apache/hadoop/io/compress/GzipCodec.java | 59 --------------------
1 file changed, 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50decd36/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
index 01b6434..d079412 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
@@ -45,10 +45,6 @@ public class GzipCodec extends DefaultCodec {
protected static class GzipOutputStream extends CompressorStream {
private static class ResetableGZIPOutputStream extends GZIPOutputStream {
- private static final int TRAILER_SIZE = 8;
- public static final String JVMVersion= System.getProperty("java.version");
- private static final boolean HAS_BROKEN_FINISH =
- (IBM_JAVA && JVMVersion.contains("1.6.0"));
public ResetableGZIPOutputStream(OutputStream out) throws IOException {
super(out);
@@ -57,61 +53,6 @@ public class GzipCodec extends DefaultCodec {
public void resetState() throws IOException {
def.reset();
}
-
- /**
- * Override this method for HADOOP-8419.
- * Override because IBM implementation calls def.end() which
- * causes problem when reseting the stream for reuse.
- *
- */
- @Override
- public void finish() throws IOException {
- if (HAS_BROKEN_FINISH) {
- if (!def.finished()) {
- def.finish();
- while (!def.finished()) {
- int i = def.deflate(this.buf, 0, this.buf.length);
- if ((def.finished()) && (i <= this.buf.length - TRAILER_SIZE)) {
- writeTrailer(this.buf, i);
- i += TRAILER_SIZE;
- out.write(this.buf, 0, i);
-
- return;
- }
- if (i > 0) {
- out.write(this.buf, 0, i);
- }
- }
-
- byte[] arrayOfByte = new byte[TRAILER_SIZE];
- writeTrailer(arrayOfByte, 0);
- out.write(arrayOfByte);
- }
- } else {
- super.finish();
- }
- }
-
- /** re-implement for HADOOP-8419 because the relative method in jdk is invisible */
- private void writeTrailer(byte[] paramArrayOfByte, int paramInt)
- throws IOException {
- writeInt((int)this.crc.getValue(), paramArrayOfByte, paramInt);
- writeInt(this.def.getTotalIn(), paramArrayOfByte, paramInt + 4);
- }
-
- /** re-implement for HADOOP-8419 because the relative method in jdk is invisible */
- private void writeInt(int paramInt1, byte[] paramArrayOfByte, int paramInt2)
- throws IOException {
- writeShort(paramInt1 & 0xFFFF, paramArrayOfByte, paramInt2);
- writeShort(paramInt1 >> 16 & 0xFFFF, paramArrayOfByte, paramInt2 + 2);
- }
-
- /** re-implement for HADOOP-8419 because the relative method in jdk is invisible */
- private void writeShort(int paramInt1, byte[] paramArrayOfByte, int paramInt2)
- throws IOException {
- paramArrayOfByte[paramInt2] = (byte)(paramInt1 & 0xFF);
- paramArrayOfByte[(paramInt2 + 1)] = (byte)(paramInt1 >> 8 & 0xFF);
- }
}
public GzipOutputStream(OutputStream out) throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-5703. ReservationAgents are
not correctly configured. Contributed by Manikandan R.
Posted by jh...@apache.org.
YARN-5703. ReservationAgents are not correctly configured. Contributed by Manikandan R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f5b031d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f5b031d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f5b031d
Branch: refs/heads/YARN-5734
Commit: 5f5b031d1f20cb7f621db41979e963eaa42cf52f
Parents: 4d33683
Author: Naganarasimha <na...@apache.org>
Authored: Mon Feb 27 20:38:29 2017 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Feb 27 20:38:29 2017 +0530
----------------------------------------------------------------------
.../reservation/AbstractReservationSystem.java | 8 ++++++--
.../planning/AlignedPlannerWithGreedy.java | 16 +++++++++-------
.../planning/GreedyReservationAgent.java | 19 ++++++-------------
.../reservation/planning/PlanningAlgorithm.java | 4 ++++
.../reservation/planning/ReservationAgent.java | 8 ++++++++
.../planning/TryManyReservationAgents.java | 5 ++++-
.../reservation/planning/TestAlignedPlanner.java | 3 +++
.../planning/TestGreedyReservationAgent.java | 4 ++--
8 files changed, 42 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index 8769ca1..5ef4912 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -451,12 +451,16 @@ public abstract class AbstractReservationSystem extends AbstractService
try {
Class<?> agentClazz = conf.getClassByName(agentClassName);
if (ReservationAgent.class.isAssignableFrom(agentClazz)) {
- return (ReservationAgent) ReflectionUtils.newInstance(agentClazz, conf);
+ ReservationAgent resevertionAgent =
+ (ReservationAgent) agentClazz.newInstance();
+ resevertionAgent.init(conf);
+ return resevertionAgent;
} else {
throw new YarnRuntimeException("Class: " + agentClassName
+ " not instance of " + ReservationAgent.class.getCanonicalName());
}
- } catch (ClassNotFoundException e) {
+ } catch (ClassNotFoundException | InstantiationException
+ | IllegalAccessException e) {
throw new YarnRuntimeException("Could not instantiate Agent: "
+ agentClassName + " for queue: " + queueName, e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
index b23cf1e..00c2333 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
import java.util.LinkedList;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -35,22 +36,25 @@ import org.slf4j.LoggerFactory;
public class AlignedPlannerWithGreedy implements ReservationAgent {
// Default smoothness factor
- private static final int DEFAULT_SMOOTHNESS_FACTOR = 10;
+ public static final int DEFAULT_SMOOTHNESS_FACTOR = 10;
+ public static final String SMOOTHNESS_FACTOR =
+ "yarn.resourcemanager.reservation-system.smoothness-factor";
// Log
private static final Logger LOG = LoggerFactory
.getLogger(AlignedPlannerWithGreedy.class);
// Smoothness factor
- private final ReservationAgent planner;
+ private ReservationAgent planner;
// Constructor
public AlignedPlannerWithGreedy() {
- this(DEFAULT_SMOOTHNESS_FACTOR);
}
- // Constructor
- public AlignedPlannerWithGreedy(int smoothnessFactor) {
+ @Override
+ public void init(Configuration conf) {
+ int smoothnessFactor =
+ conf.getInt(SMOOTHNESS_FACTOR, DEFAULT_SMOOTHNESS_FACTOR);
// List of algorithms
List<ReservationAgent> listAlg = new LinkedList<ReservationAgent>();
@@ -71,7 +75,6 @@ public class AlignedPlannerWithGreedy implements ReservationAgent {
// 1. Attempt to execute algAligned
// 2. If failed, fall back to algGreedy
planner = new TryManyReservationAgents(listAlg);
-
}
@Override
@@ -119,5 +122,4 @@ public class AlignedPlannerWithGreedy implements ReservationAgent {
return planner.deleteReservation(reservationId, user, plan);
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
index 915a834..1559b97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
@@ -46,25 +46,19 @@ public class GreedyReservationAgent implements ReservationAgent {
.getLogger(GreedyReservationAgent.class);
// Greedy planner
- private final ReservationAgent planner;
-
+ private ReservationAgent planner;
public final static String GREEDY_FAVOR_EARLY_ALLOCATION =
"yarn.resourcemanager.reservation-system.favor-early-allocation";
-
public final static boolean DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION = true;
-
- private final boolean allocateLeft;
+ private boolean allocateLeft;
public GreedyReservationAgent() {
- this(new Configuration());
}
- public GreedyReservationAgent(Configuration yarnConfiguration) {
-
- allocateLeft =
- yarnConfiguration.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION,
- DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION);
-
+ @Override
+ public void init(Configuration conf) {
+ allocateLeft = conf.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION,
+ DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION);
if (allocateLeft) {
LOG.info("Initializing the GreedyReservationAgent to favor \"early\""
+ " (left) allocations (controlled by parameter: "
@@ -78,7 +72,6 @@ public class GreedyReservationAgent implements ReservationAgent {
planner =
new IterativePlanner(new StageEarliestStartByJobArrival(),
new StageAllocatorGreedyRLE(allocateLeft), allocateLeft);
-
}
public boolean isAllocateLeft(){
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
index e1b508d..199bfa5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -206,4 +207,7 @@ public abstract class PlanningAlgorithm implements ReservationAgent {
}
+ @Override
+ public void init(Configuration conf) {
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
index bdea2f4..52e7055 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
@@ -17,6 +17,7 @@
*******************************************************************************/
package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -70,4 +71,11 @@ public interface ReservationAgent {
public boolean deleteReservation(ReservationId reservationId, String user,
Plan plan) throws PlanningException;
+ /**
+ * Init configuration.
+ *
+ * @param conf Configuration
+ */
+ void init(Configuration conf);
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
index 1d37ce5..ab2e1e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
import java.util.LinkedList;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -110,5 +111,7 @@ public class TryManyReservationAgents implements ReservationAgent {
return plan.deleteReservation(reservationId);
}
-
+ @Override
+ public void init(Configuration conf) {
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
index 4b01eb9..2645366 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
@@ -726,8 +726,11 @@ public class TestAlignedPlanner {
QueueMetrics queueMetrics = mock(QueueMetrics.class);
RMContext context = ReservationSystemTestUtil.createMockRMContext();
+ conf.setInt(AlignedPlannerWithGreedy.SMOOTHNESS_FACTOR,
+ AlignedPlannerWithGreedy.DEFAULT_SMOOTHNESS_FACTOR);
// Set planning agent
agent = new AlignedPlannerWithGreedy();
+ agent.init(conf);
// Create Plan
plan =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
index ec11ffe..6d1cfa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
@@ -110,8 +110,8 @@ public class TestGreedyReservationAgent {
// setting conf to
conf.setBoolean(GreedyReservationAgent.GREEDY_FAVOR_EARLY_ALLOCATION,
allocateLeft);
-
- agent = new GreedyReservationAgent(conf);
+ agent = new GreedyReservationAgent();
+ agent.init(conf);
QueueMetrics queueMetrics = mock(QueueMetrics.class);
RMContext context = ReservationSystemTestUtil.createMockRMContext();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-5335. Use em-table in
app/nodes pages for new YARN UI. Contributed by Sunil G.
Posted by jh...@apache.org.
YARN-5335. Use em-table in app/nodes pages for new YARN UI. Contributed by Sunil G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef488044
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef488044
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef488044
Branch: refs/heads/YARN-5734
Commit: ef488044d08983e84235e8da1cf3a7fdacadd70d
Parents: f187d63
Author: Sunil G <su...@apache.org>
Authored: Tue Feb 28 13:49:19 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Feb 28 13:49:19 2017 +0530
----------------------------------------------------------------------
.../src/main/webapp/app/adapters/yarn-app.js | 4 +-
.../app/components/app-usage-donut-chart.js | 1 -
.../main/webapp/app/components/donut-chart.js | 1 -
.../webapp/app/controllers/app-table-columns.js | 98 +++++++++++++++++
.../webapp/app/controllers/yarn-apps/apps.js | 25 +++++
.../app/controllers/yarn-apps/services.js | 25 +++++
.../main/webapp/app/controllers/yarn-node.js | 3 +-
.../webapp/app/controllers/yarn-nodes/table.js | 109 +++++++++++++++++++
.../webapp/app/controllers/yarn-queue/apps.js | 31 ++++++
.../src/main/webapp/app/models/yarn-rm-node.js | 8 ++
.../src/main/webapp/app/routes/yarn-apps.js | 2 -
.../main/webapp/app/routes/yarn-apps/apps.js | 9 ++
.../webapp/app/routes/yarn-apps/services.js | 11 ++
.../src/main/webapp/app/routes/yarn-queue.js | 4 +-
.../src/main/webapp/app/routes/yarn-queues.js | 3 +-
.../src/main/webapp/app/serializers/yarn-app.js | 3 +-
.../webapp/app/templates/yarn-apps/apps.hbs | 13 ++-
.../webapp/app/templates/yarn-apps/services.hbs | 4 +-
.../webapp/app/templates/yarn-nodes/table.hbs | 42 +------
.../webapp/app/templates/yarn-queue/apps.hbs | 3 +-
.../hadoop-yarn-ui/src/main/webapp/package.json | 9 +-
.../unit/controllers/app-table-columns-test.js | 30 +++++
.../unit/controllers/yarn-apps/apps-test.js | 30 +++++
.../unit/controllers/yarn-apps/services-test.js | 30 +++++
.../unit/controllers/yarn-nodes/table-test.js | 30 +++++
.../unit/controllers/yarn-queue/apps-test.js | 30 +++++
26 files changed, 492 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app.js
index b34c606..fc52f7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app.js
@@ -25,9 +25,7 @@ export default AbstractAdapter.extend({
urlForQuery(query/*, modelName*/) {
var url = this._buildURL();
- if (query.state) {
- url = url + '/apps/?state=' + query.state;
- }
+ url = url + '/apps';
return url;
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/app-usage-donut-chart.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/app-usage-donut-chart.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/app-usage-donut-chart.js
index 274c219..c72d934 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/app-usage-donut-chart.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/app-usage-donut-chart.js
@@ -36,7 +36,6 @@ export default BaseUsageDonutChart.extend({
value: v.toFixed(2)
});
- console.log(v);
avail = avail - v;
}
}.bind(this));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
index e5699b4..82d2d46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
@@ -54,7 +54,6 @@ export default BaseChartComponent.extend({
// 50 is for title
var outerRadius = (h - 50 - 2 * layout.margin) / 2;
var innerRadius = outerRadius * 0.618;
- console.log("inner:" + innerRadius + " outer:" + outerRadius);
var arc = d3.svg.arc()
.innerRadius(innerRadius)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js
new file mode 100644
index 0000000..53a6439
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import Ember from 'ember';
+import ColumnDef from 'em-table/utils/column-definition';
+
+export default Ember.Controller.extend({
+ columns: function() {
+ var colums = [];
+ colums.push({
+ id: 'appId',
+ headerTitle: 'Application ID',
+ contentPath: 'id',
+ cellComponentName: 'em-table-linked-cell',
+ getCellContent: function(row) {
+\u2002\u2002 return {
+\u2002\u2002\u2002\u2002 displayText: row.id,
+\u2002\u2002\u2002\u2002 routeName: 'yarn-app',
+\u2002\u2002\u2002\u2002 id: row.id
+ };
+ },
+ minWidth: "250px"
+ }, {
+ id: 'appType',
+ headerTitle: 'Application Type',
+ contentPath: 'applicationType',
+ }, {
+ id: 'appName',
+ headerTitle: 'Application Name',
+ contentPath: 'appName',
+ }, {
+ id: 'appUsr',
+ headerTitle: 'User',
+ contentPath: 'user',
+ minWidth: "50px"
+ }, {
+ id: 'queue',
+ headerTitle: 'Queue',
+ contentPath: 'queue',
+ }, {
+ id: 'state',
+ headerTitle: 'State',
+ contentPath: 'state',
+ cellComponentName: 'em-table-status-cell',
+ minWidth: "50px"
+ }, {
+ id: 'progress',
+ headerTitle: 'Progress',
+ contentPath: 'progress',
+ cellComponentName: 'em-table-progress-cell',
+ cellDefinition: {
+\u2002\u2002\u2002\u2002\u2002\u2002 valueMax: 100
+\u2002\u2002\u2002\u2002 },
+ }, {
+ id: 'stTime',
+ headerTitle: 'Start Time',
+ contentPath: 'startTime',
+ }, {
+ id: 'elTime',
+ headerTitle: 'Elapsed Time',
+ contentPath: 'elapsedTime',
+ cellDefinition: {
+ type: "duration"
+ }
+ }, {
+ id: 'finishTime',
+ headerTitle: 'Finished Time',
+ contentPath: 'validatedFinishedTs',
+ observePath: true
+ }, {
+ id: 'priority',
+ headerTitle: 'Priority',
+ contentPath: 'priority',
+ }, {
+ id: 'cluster',
+ headerTitle: '%Cluster',
+ contentPath: 'clusterUsagePercentage',
+ observePath: true
+ });
+ return ColumnDef.make(colums);
+ }.property()
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
new file mode 100644
index 0000000..5c5063b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import Ember from 'ember';
+import ColumnDef from 'em-table/utils/column-definition';
+import AppTableController from '../app-table-columns';
+
+export default AppTableController.extend({
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
new file mode 100644
index 0000000..5c5063b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import Ember from 'ember';
+import ColumnDef from 'em-table/utils/column-definition';
+import AppTableController from '../app-table-columns';
+
+export default AppTableController.extend({
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js
index 55d9b3e..335a33c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js
@@ -20,8 +20,9 @@ import Ember from 'ember';
export default Ember.Controller.extend({
- breadcrumbs: Ember.computed("model.attempt.appId", function () {
+ breadcrumbs: Ember.computed("model.nodeInfo", function () {
var nodeInfo = this.get("model.nodeInfo");
+
return [{
text: "Home",
routeName: 'application'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
new file mode 100644
index 0000000..3b7b0f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import Ember from 'ember';
+import ColumnDef from 'em-table/utils/column-definition';
+
+export default Ember.Controller.extend({
+ columns: function() {
+ var colums = [];
+ colums.push({
+ id: 'label',
+ headerTitle: 'Node Label',
+ contentPath: 'nodeLabelsAsString',
+ minWidth: "100px"
+ }, {
+ id: 'rack',
+ headerTitle: 'Rack',
+ contentPath: 'rack',
+ minWidth: "100px"
+ }, {
+ id: 'state',
+ headerTitle: 'Node State',
+ contentPath: 'state',
+ cellComponentName: 'em-table-status-cell',
+ minWidth: "100px"
+ }, {
+ id: 'address',
+ headerTitle: 'Node Address',
+ contentPath: 'id',
+ minWidth: "300px"
+ }, {
+ id: 'nodeId',
+ headerTitle: 'Node HTTP Address',
+ contentPath: 'nodeHTTPAddress',
+ cellComponentName: 'em-table-linked-cell',
+ getCellContent: function(row) {
+ var node_id = row.get("id"),
+ node_addr = row.get("nodeHTTPAddress"),
+ href = `#/yarn-node/${node_id}/${node_addr}`;
+ switch(row.get("nodeState")) {
+ case "SHUTDOWN":
+ case "LOST":
+ href = "";
+ }
+\u2002\u2002 return {
+ text: row.get("nodeHTTPAddress"),
+ href: href
+ };
+ },
+ minWidth: "250px"
+ }, {
+ id: 'containers',
+ headerTitle: 'Containers',
+ contentPath: 'numContainers',
+ }, {
+ id: 'memUsed',
+ headerTitle: 'Mem Used',
+ contentPath: 'usedMemoryBytes',
+ cellDefinition: {
+ type: "memory"
+ }
+ }, {
+ id: 'memAvail',
+ headerTitle: 'Mem Available',
+ contentPath: 'availMemoryBytes',
+ cellDefinition: {
+ type: "memory"
+ }
+ }, {
+ id: 'coresUsed',
+ headerTitle: 'VCores Used',
+ contentPath: 'usedVirtualCores',
+ }, {
+ id: 'coresAvail',
+ headerTitle: 'VCores Available',
+ contentPath: 'availableVirtualCores',
+ }, {
+ id: 'healthUpdate',
+ headerTitle: 'Last Health Update',
+ contentPath: 'lastHealthUpdate',
+ }, {
+ id: 'healthReport',
+ headerTitle: 'Health-Report',
+ contentPath: 'healthReport',
+ }, {
+ id: 'version',
+ headerTitle: 'Version',
+ contentPath: 'version',
+ observePath: true
+ });
+ return ColumnDef.make(colums);
+ }.property()
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-queue/apps.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-queue/apps.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-queue/apps.js
new file mode 100644
index 0000000..905d96d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-queue/apps.js
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Ember from 'ember';
+import TableDefinition from 'em-table/utils/table-definition';
+import AppTableController from '../app-table-columns';
+
+export default AppTableController.extend({
+ // Your custom instance of table definition
+ tableDefinition: TableDefinition.create(),
+
+ // Search text alias, any change in controller's searchText would affect the table's searchText, and vice-versa.
+ _selectedObserver: Ember.on("init", Ember.observer("model.selected", function () {
+ this.set("tableDefinition.searchText", this.get("model.selected"));
+ })),
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
index c7875b4..461bf00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
@@ -96,4 +96,12 @@ export default DS.Model.extend({
"<p>Used Memory: " + Math.round(this.get("usedMemoryMB")) + ' MB</p>' +
"<p>Available Memory: " + Math.round(this.get("availMemoryMB")) + ' MB</p>';
}.property(),
+
+ usedMemoryBytes: function() {
+ return this.get("usedMemoryMB") * 1024 * 1024;
+ }.property("usedMemoryMB"),
+
+ availMemoryBytes: function() {
+ return this.get("availMemoryMB") * 1024 * 1024;
+ }.property("availMemoryMB"),
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps.js
index 1b671da..56349aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps.js
@@ -23,13 +23,11 @@ import AbstractRoute from './abstract';
export default AbstractRoute.extend({
model() {
return Ember.RSVP.hash({
- apps: this.store.findAll('yarn-app', {reload: true}),
clusterMetrics: this.store.findAll('ClusterMetric', {reload: true}),
});
},
unloadAll() {
- this.store.unloadAll('yarn-app');
this.store.unloadAll('ClusterMetric');
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps/apps.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps/apps.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps/apps.js
index 8719170..b0f3745 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps/apps.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps/apps.js
@@ -19,4 +19,13 @@
import Ember from 'ember';
export default Ember.Route.extend({
+ model() {
+ return Ember.RSVP.hash({
+ apps: this.store.findAll('yarn-app', {reload: true}),
+ });
+ },
+
+ unloadAll() {
+ this.store.unloadAll('yarn-app');
+ }
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps/services.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps/services.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps/services.js
index 8719170..34ad1ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps/services.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-apps/services.js
@@ -19,4 +19,15 @@
import Ember from 'ember';
export default Ember.Route.extend({
+ model() {
+ return Ember.RSVP.hash({
+ apps: this.store.query('yarn-app', {
+ applicationTypes: "org-apache-slider"
+ }),
+ });
+ },
+
+ unloadAll() {
+ this.store.unloadAll('yarn-app');
+ }
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queue.js
index faae8b1..1c4546c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queue.js
@@ -26,7 +26,9 @@ export default AbstractRoute.extend({
selected : param.queue_name,
queues: this.store.query('yarn-queue', {}),
selectedQueue : undefined,
- apps: this.store.findAll('yarn-app', {reload: true})
+ apps: this.store.query('yarn-app', {
+ queue: param.queue_name
+ })
});
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues.js
index 7ef7ac3..e4f145d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues.js
@@ -31,8 +31,7 @@ export default AbstractRoute.extend({
return Ember.RSVP.hash({
selected : queueName,
queues: this.store.query('yarn-queue', {}),
- selectedQueue : undefined,
- apps: this.store.findAll('yarn-app', {reload: true})
+ selectedQueue : undefined
});
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js
index 83d1e32..172286f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js
@@ -43,7 +43,8 @@ export default DS.JSONAPISerializer.extend({
queue: payload.queue,
state: payload.state,
startTime: Converter.timeStampToDate(payload.startedTime),
- elapsedTime: Converter.msToElapsedTime(payload.elapsedTime),
+ //elapsedTime: Converter.msToElapsedTime(payload.elapsedTime),
+ elapsedTime: payload.elapsedTime,
finishedTime: Converter.timeStampToDate(payload.finishedTime),
finalStatus: payload.finalStatus,
progress: payload.progress,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-apps/apps.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-apps/apps.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-apps/apps.hbs
index a2ba163..8c72ce4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-apps/apps.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-apps/apps.hbs
@@ -16,9 +16,12 @@
limitations under the License.
--}}
-{{#if model.apps}}
- {{app-table table-id="apps-table" arr=model.apps}}
- {{simple-table table-id="apps-table" bFilter=true colsOrder="0,desc" colTypes="natural elapsed-time" colTargets="0 7"}}
+{{#if model.apps.isPending}}
+ <h4 align="center">Loading...</h4>
{{else}}
- <h4 align="center">Could not find any applications from this cluster</h4>
-{{/if}}
\ No newline at end of file
+ {{#if model.apps}}
+ {{em-table columns=columns rows=model.apps}}
+ {{else}}
+ <h4 align="center">Could not find any applications from this cluster</h4>
+ {{/if}}
+{{/if}}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-apps/services.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-apps/services.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-apps/services.hbs
index e472e8e..7556908 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-apps/services.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-apps/services.hbs
@@ -17,9 +17,7 @@
--}}
{{#if model.apps}}
- {{app-table table-id="apps-table" arr=model.apps}}
- {{simple-table table-id="apps-table" bFilter=true colsOrder="0,desc"
- colTypes="natural elapsed-time" colTargets="0 7" defaultSearch="slider"}}
+ {{em-table columns=columns rows=model.apps}}
{{else}}
<h4 align="center">Could not find any applications from this cluster</h4>
{{/if}}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes/table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes/table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes/table.hbs
index d9fae3a..1390906 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes/table.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes/table.hbs
@@ -18,47 +18,7 @@
<div class="row">
{{#if model.nodes}}
- <table id="nodes-table" class="display table table-striped table-bordered"
- cellspacing="0" width="100%">
- <thead>
- <tr>
- <th>Node Labels</th>
- <th>Rack</th>
- <th>Node State</th>
- <th>Node Address</th>
- <th>Node HTTP Address</th>
- <th>Last Health Update</th>
- <th>Health-Report</th>
- <th>Containers</th>
- <th>Mem Used</th>
- <th>Mem Avail</th>
- <th>VCores Used</th>
- <th>VCores Avail</th>
- <th>Version</th>
- </tr>
- </thead>
- <tbody>
- {{#each model.nodes as |node|}}
- <tr>
- <td>{{node.nodeLabelsAsString}}</td>
- <td>{{node.rack}}</td>
- <td><span class={{node.nodeStateStyle}}>{{node.state}}</span></td>
- <td>{{node.id}}</td>
- {{node-link nodeId=node.id nodeHTTPAddress=node.nodeHTTPAddress nodeState=node.state}}
- <td>{{node.lastHealthUpdate}}</td>
- <td>{{node.healthReport}}</td>
- <td>{{node.numContainers}}</td>
- <td>{{divide num=node.usedMemoryMB den=1024}} GB</td>
- <td>{{divide num=node.availMemoryMB den=1024}} GB</td>
- <td>{{node.usedVirtualCores}}</td>
- <td>{{node.availableVirtualCores}}</td>
- <td>{{node.version}}</td>
- </tr>
- {{/each}}
- </tbody>
- </table>
-
- {{simple-table table-id="nodes-table" bFilter=true}}
+ {{em-table columns=columns rows=model.nodes}}
{{else}}
<h4 align="center">No nodes found on this cluster</h4>
{{/if}}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
index 6d6488f..4a508c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
@@ -19,8 +19,7 @@
<div class="row">
<div class="col-lg-12 container-fluid">
{{#if model.apps}}
- {{app-table table-id="apps-table" arr=model.apps}}
- {{simple-table table-id="apps-table" bFilter=true colTypes="elapsed-time" colTargets="7"}}
+ {{em-table columns=columns rows=model.apps}}
{{else}}
<h4 align = "center">Could not find any applications from this cluster</h4>
{{/if}}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
index 6a4eb16..4cd0fdc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
@@ -20,9 +20,9 @@
"devDependencies": {
"broccoli-asset-rev": "2.4.2",
"broccoli-funnel": "1.0.1",
- "em-table": "0.1.6",
- "ember-bootstrap": "0.5.1",
+ "em-table": "0.7.0",
"ember-array-contains-helper": "1.0.2",
+ "ember-bootstrap": "0.5.1",
"ember-cli": "1.13.13",
"ember-cli-app-version": "1.0.0",
"ember-cli-babel": "5.1.6",
@@ -47,6 +47,9 @@
"select2": "4.0.0"
},
"dependencies": {
- "em-helpers": "^0.5.13"
+ "em-helpers": "^0.8.0",
+ "em-table": "^0.7.0",
+ "ember-cli-moment-shim": "^3.0.1",
+ "ember-cli-numeral": "^0.2.0"
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/app-table-columns-test.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/app-table-columns-test.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/app-table-columns-test.js
new file mode 100644
index 0000000..b8b77ad
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/app-table-columns-test.js
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { moduleFor, test } from 'ember-qunit';
+
+moduleFor('controller:app-table-columns', 'Unit | Controller | app table columns', {
+ // Specify the other units that are required for this test.
+ // needs: ['controller:foo']
+});
+
+// Replace this with your real tests.
+test('it exists', function(assert) {
+ let controller = this.subject();
+ assert.ok(controller);
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-apps/apps-test.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-apps/apps-test.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-apps/apps-test.js
new file mode 100644
index 0000000..a004283
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-apps/apps-test.js
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { moduleFor, test } from 'ember-qunit';
+
+moduleFor('controller:yarn-apps/apps', 'Unit | Controller | yarn apps/apps', {
+ // Specify the other units that are required for this test.
+ // needs: ['controller:foo']
+});
+
+// Replace this with your real tests.
+test('it exists', function(assert) {
+ let controller = this.subject();
+ assert.ok(controller);
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-apps/services-test.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-apps/services-test.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-apps/services-test.js
new file mode 100644
index 0000000..d0a1f85
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-apps/services-test.js
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { moduleFor, test } from 'ember-qunit';
+
+moduleFor('controller:yarn-apps/services', 'Unit | Controller | yarn apps/services', {
+ // Specify the other units that are required for this test.
+ // needs: ['controller:foo']
+});
+
+// Replace this with your real tests.
+test('it exists', function(assert) {
+ let controller = this.subject();
+ assert.ok(controller);
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-nodes/table-test.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-nodes/table-test.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-nodes/table-test.js
new file mode 100644
index 0000000..fdf586e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-nodes/table-test.js
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { moduleFor, test } from 'ember-qunit';
+
+moduleFor('controller:yarn-nodes/table', 'Unit | Controller | yarn nodes/table', {
+ // Specify the other units that are required for this test.
+ // needs: ['controller:foo']
+});
+
+// Replace this with your real tests.
+test('it exists', function(assert) {
+ let controller = this.subject();
+ assert.ok(controller);
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef488044/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-queue/apps-test.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-queue/apps-test.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-queue/apps-test.js
new file mode 100644
index 0000000..3abf21a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-queue/apps-test.js
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { moduleFor, test } from 'ember-qunit';
+
+moduleFor('controller:yarn-queue/apps', 'Unit | Controller | yarn queue/apps', {
+ // Specify the other units that are required for this test.
+ // needs: ['controller:foo']
+});
+
+// Replace this with your real tests.
+test('it exists', function(assert) {
+ let controller = this.subject();
+ assert.ok(controller);
+});
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDFS-11427. Rename rs-default to
rs.
Posted by jh...@apache.org.
HDFS-11427. Rename rs-default to rs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1a52b04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1a52b04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1a52b04
Branch: refs/heads/YARN-5734
Commit: c1a52b04d0cc5ad5c86ae93043655f313386f7f9
Parents: b32ffa2
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Feb 24 10:58:45 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Feb 24 10:58:45 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/io/erasurecode/CodecUtil.java | 28 +++++++--------
.../io/erasurecode/ErasureCodeConstants.java | 8 ++---
.../erasurecode/coder/HHXORErasureDecoder.java | 2 +-
.../erasurecode/coder/HHXORErasureEncoder.java | 2 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 2 +-
.../io/erasurecode/coder/RSErasureEncoder.java | 2 +-
.../src/main/resources/core-default.xml | 4 +--
.../erasurecode/TestCodecRawCoderMapping.java | 10 +++---
.../coder/TestHHXORErasureCoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 2 +-
.../src/site/markdown/HDFSErasureCoding.md | 6 ++--
.../TestDFSRSDefault10x4StripedInputStream.java | 2 +-
...TestDFSRSDefault10x4StripedOutputStream.java | 2 +-
...fault10x4StripedOutputStreamWithFailure.java | 4 +--
.../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +-
.../TestDFSStripedOutputStreamWithFailure.java | 2 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 2 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +-
.../test/resources/testErasureCodingConf.xml | 36 ++++++++++----------
20 files changed, 61 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 977bacb..861451a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -55,9 +55,9 @@ public final class CodecUtil {
public static final String IO_ERASURECODE_CODEC_XOR =
XORErasureCodec.class.getCanonicalName();
/** Erasure coder Reed-Solomon codec. */
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_KEY =
+ public static final String IO_ERASURECODE_CODEC_RS_KEY =
"io.erasurecode.codec.rs";
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT =
+ public static final String IO_ERASURECODE_CODEC_RS =
RSErasureCodec.class.getCanonicalName();
/** Erasure coder hitch hiker XOR codec. */
public static final String IO_ERASURECODE_CODEC_HHXOR_KEY =
@@ -67,10 +67,10 @@ public final class CodecUtil {
/** Supported erasure codec classes. */
- /** Raw coder factory for the RS default codec. */
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY =
- "io.erasurecode.codec.rs-default.rawcoder";
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT =
+ /** Raw coder factory for the RS codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
+ "io.erasurecode.codec.rs.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT =
RSRawErasureCoderFactory.class.getCanonicalName();
/** Raw coder factory for the RS legacy codec. */
@@ -183,10 +183,10 @@ public final class CodecUtil {
private static String getRawCoderFactNameFromCodec(Configuration conf,
String codec) {
switch (codec) {
- case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
+ case ErasureCodeConstants.RS_CODEC_NAME:
return conf.get(
- IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
- IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT);
+ IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT);
case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
return conf.get(
IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
@@ -233,15 +233,15 @@ public final class CodecUtil {
private static String getCodecClassName(Configuration conf, String codec) {
switch (codec) {
- case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
+ case ErasureCodeConstants.RS_CODEC_NAME:
return conf.get(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY,
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT);
+ CodecUtil.IO_ERASURECODE_CODEC_RS_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS);
case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
//TODO:rs-legacy should be handled differently.
return conf.get(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY,
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT);
+ CodecUtil.IO_ERASURECODE_CODEC_RS_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS);
case ErasureCodeConstants.XOR_CODEC_NAME:
return conf.get(
CodecUtil.IO_ERASURECODE_CODEC_XOR_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index e168909..c830bb2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -25,16 +25,16 @@ public final class ErasureCodeConstants {
private ErasureCodeConstants() {
}
- public static final String RS_DEFAULT_CODEC_NAME = "rs-default";
+ public static final String RS_CODEC_NAME = "rs";
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
public static final String XOR_CODEC_NAME = "xor";
public static final String HHXOR_CODEC_NAME = "hhxor";
public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
- RS_DEFAULT_CODEC_NAME, 6, 3);
+ RS_CODEC_NAME, 6, 3);
public static final ECSchema RS_3_2_SCHEMA = new ECSchema(
- RS_DEFAULT_CODEC_NAME, 3, 2);
+ RS_CODEC_NAME, 3, 2);
public static final ECSchema RS_6_3_LEGACY_SCHEMA = new ECSchema(
RS_LEGACY_CODEC_NAME, 6, 3);
@@ -43,5 +43,5 @@ public final class ErasureCodeConstants {
XOR_CODEC_NAME, 2, 1);
public static final ECSchema RS_10_4_SCHEMA = new ECSchema(
- RS_DEFAULT_CODEC_NAME, 10, 4);
+ RS_CODEC_NAME, 10, 4);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
index 05e9384..4747e49 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
@@ -67,7 +67,7 @@ public class HHXORErasureDecoder extends ErasureDecoder {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
rsRawDecoder = CodecUtil.createRawDecoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rsRawDecoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
index 7a15a05..6b858b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
@@ -61,7 +61,7 @@ public class HHXORErasureEncoder extends ErasureEncoder {
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rsRawEncoder == null) {
rsRawEncoder = CodecUtil.createRawEncoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rsRawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
index 6e679c3..ba7f773 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
@@ -52,7 +52,7 @@ public class RSErasureDecoder extends ErasureDecoder {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
rsRawDecoder = CodecUtil.createRawDecoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rsRawDecoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
index 7a09b92..193309d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
@@ -53,7 +53,7 @@ public class RSErasureEncoder extends ErasureEncoder {
if (rawEncoder == null) {
// TODO: we should create the raw coder according to codec.
rawEncoder = CodecUtil.createRawEncoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index d8136ee..08ca05b 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -657,10 +657,10 @@
</property>
<property>
- <name>io.erasurecode.codec.rs-default.rawcoder</name>
+ <name>io.erasurecode.codec.rs.rawcoder</name>
<value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
<description>
- Raw coder implementation for the rs-default codec. The default value is a
+ Raw coder implementation for the rs codec. The default value is a
pure Java implementation. There is also a native implementation. Its value
is org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
index 0db001a..7f7fcf3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
@@ -47,12 +47,12 @@ public class TestCodecRawCoderMapping {
public void testRSDefaultRawCoder() {
ErasureCoderOptions coderOptions = new ErasureCoderOptions(
numDataUnit, numParityUnit);
- // should return default raw coder of rs-default codec
+ // should return default raw coder of rs codec
RawErasureEncoder encoder = CodecUtil.createRawEncoder(
- conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
Assert.assertTrue(encoder instanceof RSRawEncoder);
RawErasureDecoder decoder = CodecUtil.createRawDecoder(
- conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
Assert.assertTrue(decoder instanceof RSRawDecoder);
// should return default raw coder of rs-legacy codec
@@ -71,11 +71,11 @@ public class TestCodecRawCoderMapping {
String dummyFactName = "DummyNoneExistingFactory";
// set the dummy factory to rs-legacy and create a raw coder
- // with rs-default, which is OK as the raw coder key is not used
+ // with rs, which is OK as the raw coder key is not used
conf.set(CodecUtil.
IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName);
RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf,
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
Assert.assertTrue(encoder instanceof RSRawEncoder);
// now create the raw coder with rs-legacy, which should throw exception
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
index 1eca1e1..91c4ba8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
@@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
index 5f36c99..3b18347 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
@@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 0283e2b..36fb61d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -64,7 +64,7 @@ Architecture
2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work.
- There are four policies currently being supported: RS-DEFAULT-3-2-64k, RS-DEFAULT-6-3-64k, RS-DEFAULT-10-4-64k and RS-LEGACY-6-3-64k. All with default cell size of 64KB. The system default policy is RS-DEFAULT-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB.
+ Five policies are currently supported: RS-3-2-64k, RS-6-3-64k, RS-10-4-64k, RS-LEGACY-6-3-64k, and XOR-2-1-64k. All with default cell size of 64KB. The system default policy is RS-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB.
* **Intel ISA-L**
Intel ISA-L stands for Intel Intelligent Storage Acceleration Library. ISA-L is a collection of optimized low-level functions used primarily in storage applications. It includes a fast block Reed-Solomon type erasure codes optimized for Intel AVX and AVX2 instruction sets.
@@ -91,7 +91,7 @@ Deployment
### Configuration keys
The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys:
- `io.erasurecode.codec.rs-default.rawcoder` for the default RS codec,
+ `io.erasurecode.codec.rs.rawcoder` for the default RS codec,
`io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec,
`io.erasurecode.codec.xor.rawcoder` for the XOR codec.
The default implementations for all of these codecs are pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information.
@@ -108,7 +108,7 @@ Deployment
HDFS native implementation of default RS codec leverages Intel ISA-L library to improve the encoding and decoding calculation. To enable and use Intel ISA-L, there are three steps.
1. Build ISA-L library. Please refer to the offical site "https://github.com/01org/isa-l/" for detail information.
2. Build Hadoop with ISA-L support. Please refer to "Intel ISA-L build options" section in "Build instructions for Hadoop"(BUILDING.txt) document. Use -Dbundle.isal to copy the contents of the isal.lib directory into the final tar file. Deploy hadoop with the tar file. Make sure ISA-L library is available on both HDFS client and DataNodes.
- 3. Configure the `io.erasurecode.codec.rs-default.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes.
+ 3. Configure the `io.erasurecode.codec.rs.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes.
To check ISA-L library enable state, try "Hadoop checknative" command. It will tell you if ISA-L library is enabled or not.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
index fc0ee37..3e6d1e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
/**
- * This tests read operation of DFS striped file with RS-DEFAULT-10-4-64k
+ * This tests read operation of DFS striped file with RS-10-4-64k
* erasure code policy.
*/
public class TestDFSRSDefault10x4StripedInputStream extends
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
index 37821c1..1ea839a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
/**
- * This tests write operation of DFS striped file with RS-DEFAULT-10-4-64k
+ * This tests write operation of DFS striped file with RS-10-4-64k
* erasure code policy.
*/
public class TestDFSRSDefault10x4StripedOutputStream
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
index 1b2ec42..340fec5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
/**
- * This tests write operation of DFS striped file with RS-DEFAULT-10-4-64k
+ * This tests write operation of DFS striped file with RS-10-4-64k
* erasure code policy under Datanode failure conditions.
*/
public class TestDFSRSDefault10x4StripedOutputStreamWithFailure
@@ -33,4 +33,4 @@ public class TestDFSRSDefault10x4StripedOutputStreamWithFailure
return ErasureCodingPolicyManager.getPolicyByPolicyID(
HdfsConstants.RS_10_4_POLICY_ID);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 121b9a4..9b69904 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -96,7 +96,7 @@ public class TestDFSStripedInputStream {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
SimulatedFSDataset.setFactory(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index 5bde16e..8834e14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -85,7 +85,7 @@ public class TestDFSStripedOutputStream {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index 0baf9cc..fabd417 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure {
final int numDNs = dataBlocks + parityBlocks;
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index 4960c58..08b53c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -100,7 +100,7 @@ public class TestReconstructStripedFile {
false);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
index 1a4086e..f26aa06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -69,7 +69,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index 82b71def..9b9003a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -133,7 +133,7 @@
<description>setPolicy : set erasure coding policy on a directory to encode files</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -141,7 +141,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output>
+ <expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -150,8 +150,8 @@
<description>setPolicy : set a policy twice</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -159,7 +159,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output>
+ <expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -168,7 +168,7 @@
<description>unsetPolicy : unset policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
@@ -187,8 +187,8 @@
<description>setPolicy : change different policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-3-2-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-3-2-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
@@ -197,7 +197,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-3-2-64k</expected-output>
+ <expected-output>RS-3-2-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -207,7 +207,7 @@
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<command>-fs NAMENODE -mkdir /ecdir/child</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/child/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/child/ecfile</ec-admin-command>
@@ -220,7 +220,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -246,7 +246,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
@@ -255,7 +255,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -264,7 +264,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/ecfile</ec-admin-command>
</test-commands>
@@ -275,7 +275,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -290,7 +290,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3</expected-output>
+ <expected-output>RS-6-3</expected-output>
</comparator>
</comparators>
</test>
@@ -334,7 +334,7 @@
<description>setPolicy : illegal parameters - too many arguments</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-DEFAULT-3-2-64k /ecdir2</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k /ecdir2</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -367,7 +367,7 @@
<test>
<description>setPolicy : illegal parameters - no such file</description>
<test-commands>
- <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-DEFAULT-3-2-64k</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: MAPREDUCE-6841. Fix dead link in
MapReduce tutorial document. Contributed by Victor Nee.
Posted by jh...@apache.org.
MAPREDUCE-6841. Fix dead link in MapReduce tutorial document. Contributed by Victor Nee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d336838
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d336838
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d336838
Branch: refs/heads/YARN-5734
Commit: 4d33683882cc41939c512db221fd8ee59ecd52a8
Parents: 9db2e0c
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Feb 27 19:45:26 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Feb 27 19:46:37 2017 +0900
----------------------------------------------------------------------
.../src/site/markdown/MapReduceTutorial.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d336838/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
index 6747adc..574c404 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
@@ -417,7 +417,7 @@ indicates the set of input files
[FileInputFormat.addInputPath(Job, Path)](../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html)) and
([FileInputFormat.setInputPaths(Job, String...)](../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html)/
[FileInputFormat.addInputPaths(Job, String))](../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html) and where the output files should be written
-([FileOutputFormat.setOutputPath(Path)](../../api/org/apache/hadoop/mapreduce/lib/input/FileOutputFormat.html)).
+([FileOutputFormat.setOutputPath(Path)](../../api/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.html)).
Optionally, `Job` is used to specify other advanced facets of the job such as the `Comparator` to be used, files to be put in the `DistributedCache`, whether intermediate and/or job outputs are to be compressed (and how), whether job tasks can be executed in a *speculative* manner
([setMapSpeculativeExecution(boolean)](../../api/org/apache/hadoop/mapreduce/Job.html))/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: MAPREDUCE-6852. Job#updateStatus()
failed with NPE due to race condition. Contributed by Junping Du
Posted by jh...@apache.org.
MAPREDUCE-6852. Job#updateStatus() failed with NPE due to race condition. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/747bafaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/747bafaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/747bafaf
Branch: refs/heads/YARN-5734
Commit: 747bafaf969857b66233a8b4660590bdd712ed7d
Parents: eeca8b0
Author: Jian He <ji...@apache.org>
Authored: Thu Mar 2 10:58:56 2017 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Mar 2 10:58:56 2017 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/mapreduce/Job.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/747bafaf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
index 45c065d..84d90de 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
@@ -323,7 +323,7 @@ public class Job extends JobContextImpl implements JobContext {
this.status = ugi.doAs(new PrivilegedExceptionAction<JobStatus>() {
@Override
public JobStatus run() throws IOException, InterruptedException {
- return cluster.getClient().getJobStatus(status.getJobID());
+ return cluster.getClient().getJobStatus(getJobID());
}
});
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HADOOP-14028. S3A
BlockOutputStreams doesn't delete temporary files in multipart uploads or
handle part upload failures. Contributed by Steve Loughran.
Posted by jh...@apache.org.
HADOOP-14028. S3A BlockOutputStreams doesn't delete temporary files in multipart uploads or handle part upload failures.
Contributed by Steve Loughran.
(cherry picked from commit 29fe5af017b945d8750c074ca39031b5b777eddd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dab00da1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dab00da1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dab00da1
Branch: refs/heads/YARN-5734
Commit: dab00da19f25619ccc71c7f803a235b21766bf1e
Parents: 120bef7
Author: Steve Loughran <st...@apache.org>
Authored: Sat Feb 25 15:35:19 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Feb 25 15:35:19 2017 +0000
----------------------------------------------------------------------
.../hadoop/fs/s3a/S3ABlockOutputStream.java | 68 ++-
.../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 514 +++++++++++--------
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 78 ++-
.../hadoop/fs/s3a/S3AInstrumentation.java | 40 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 26 +
.../hadoop/fs/s3a/ITestS3ABlockOutputArray.java | 76 ++-
.../fs/s3a/ITestS3ABlockOutputByteBuffer.java | 5 +-
.../hadoop/fs/s3a/ITestS3ABlockOutputDisk.java | 12 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 13 +
.../apache/hadoop/fs/s3a/TestDataBlocks.java | 45 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 28 +-
11 files changed, 639 insertions(+), 266 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
index 89b9b29..1b0929b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.fs.s3a;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
@@ -48,7 +47,6 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.util.Progressable;
@@ -178,10 +176,10 @@ class S3ABlockOutputStream extends OutputStream {
if (activeBlock == null) {
blockCount++;
if (blockCount>= Constants.MAX_MULTIPART_COUNT) {
- LOG.error("Number of partitions in stream exceeds limit for S3: " +
+ LOG.error("Number of partitions in stream exceeds limit for S3: "
+ Constants.MAX_MULTIPART_COUNT + " write may fail.");
}
- activeBlock = blockFactory.create(this.blockSize);
+ activeBlock = blockFactory.create(blockCount, this.blockSize, statistics);
}
return activeBlock;
}
@@ -206,7 +204,9 @@ class S3ABlockOutputStream extends OutputStream {
* Clear the active block.
*/
private void clearActiveBlock() {
- LOG.debug("Clearing active block");
+ if (activeBlock != null) {
+ LOG.debug("Clearing active block");
+ }
synchronized (this) {
activeBlock = null;
}
@@ -356,11 +356,9 @@ class S3ABlockOutputStream extends OutputStream {
writeOperationHelper.writeFailed(ioe);
throw ioe;
} finally {
- LOG.debug("Closing block and factory");
- IOUtils.closeStream(block);
- IOUtils.closeStream(blockFactory);
+ closeAll(LOG, block, blockFactory);
LOG.debug("Statistics: {}", statistics);
- IOUtils.closeStream(statistics);
+ closeAll(LOG, statistics);
clearActiveBlock();
}
// All end of write operations, including deleting fake parent directories
@@ -378,10 +376,10 @@ class S3ABlockOutputStream extends OutputStream {
final S3ADataBlocks.DataBlock block = getActiveBlock();
int size = block.dataSize();
- final PutObjectRequest putObjectRequest =
- writeOperationHelper.newPutRequest(
- block.startUpload(),
- size);
+ final S3ADataBlocks.BlockUploadData uploadData = block.startUpload();
+ final PutObjectRequest putObjectRequest = uploadData.hasFile() ?
+ writeOperationHelper.newPutRequest(uploadData.getFile())
+ : writeOperationHelper.newPutRequest(uploadData.getUploadStream(), size);
fs.setOptionalPutRequestParameters(putObjectRequest);
long transferQueueTime = now();
BlockUploadProgress callback =
@@ -393,8 +391,14 @@ class S3ABlockOutputStream extends OutputStream {
executorService.submit(new Callable<PutObjectResult>() {
@Override
public PutObjectResult call() throws Exception {
- PutObjectResult result = fs.putObjectDirect(putObjectRequest);
- block.close();
+ PutObjectResult result;
+ try {
+ // the putObject call automatically closes the input
+ // stream afterwards.
+ result = writeOperationHelper.putObject(putObjectRequest);
+ } finally {
+ closeAll(LOG, uploadData, block);
+ }
return result;
}
});
@@ -438,13 +442,21 @@ class S3ABlockOutputStream extends OutputStream {
}
/**
+ * Get the statistics for this stream.
+ * @return stream statistics
+ */
+ S3AInstrumentation.OutputStreamStatistics getStatistics() {
+ return statistics;
+ }
+
+ /**
* Multiple partition upload.
*/
private class MultiPartUpload {
private final String uploadId;
private final List<ListenableFuture<PartETag>> partETagsFutures;
- public MultiPartUpload() throws IOException {
+ MultiPartUpload() throws IOException {
this.uploadId = writeOperationHelper.initiateMultiPartUpload();
this.partETagsFutures = new ArrayList<>(2);
LOG.debug("Initiated multi-part upload for {} with " +
@@ -461,14 +473,16 @@ class S3ABlockOutputStream extends OutputStream {
throws IOException {
LOG.debug("Queueing upload of {}", block);
final int size = block.dataSize();
- final InputStream uploadStream = block.startUpload();
+ final S3ADataBlocks.BlockUploadData uploadData = block.startUpload();
final int currentPartNumber = partETagsFutures.size() + 1;
final UploadPartRequest request =
writeOperationHelper.newUploadPartRequest(
uploadId,
- uploadStream,
currentPartNumber,
- size);
+ size,
+ uploadData.getUploadStream(),
+ uploadData.getFile());
+
long transferQueueTime = now();
BlockUploadProgress callback =
new BlockUploadProgress(
@@ -483,12 +497,16 @@ class S3ABlockOutputStream extends OutputStream {
LOG.debug("Uploading part {} for id '{}'", currentPartNumber,
uploadId);
// do the upload
- PartETag partETag = fs.uploadPart(request).getPartETag();
- LOG.debug("Completed upload of {}", block);
- LOG.debug("Stream statistics of {}", statistics);
-
- // close the block
- block.close();
+ PartETag partETag;
+ try {
+ partETag = fs.uploadPart(request).getPartETag();
+ LOG.debug("Completed upload of {} to part {}", block,
+ partETag.getETag());
+ LOG.debug("Stream statistics of {}", statistics);
+ } finally {
+ // close the stream and block
+ closeAll(LOG, uploadData, block);
+ }
return partETag;
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index 05f8efe..9bc8dcd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -24,10 +24,8 @@ import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.EOFException;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
-import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
@@ -42,10 +40,11 @@ import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.util.DirectBufferPool;
import static org.apache.hadoop.fs.s3a.S3ADataBlocks.DataBlock.DestState.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.closeAll;
/**
* Set of classes to support output streaming into blocks which are then
- * uploaded as partitions.
+ * uploaded as to S3 as a single PUT, or as part of a multipart request.
*/
final class S3ADataBlocks {
@@ -97,6 +96,70 @@ final class S3ADataBlocks {
}
/**
+ * The output information for an upload.
+ * It can be one of a file or an input stream.
+ * When closed, any stream is closed. Any source file is untouched.
+ */
+ static final class BlockUploadData implements Closeable {
+ private final File file;
+ private final InputStream uploadStream;
+
+ /**
+ * File constructor; input stream will be null.
+ * @param file file to upload
+ */
+ BlockUploadData(File file) {
+ Preconditions.checkArgument(file.exists(), "No file: " + file);
+ this.file = file;
+ this.uploadStream = null;
+ }
+
+ /**
+ * Stream constructor, file field will be null.
+ * @param uploadStream stream to upload
+ */
+ BlockUploadData(InputStream uploadStream) {
+ Preconditions.checkNotNull(uploadStream, "rawUploadStream");
+ this.uploadStream = uploadStream;
+ this.file = null;
+ }
+
+ /**
+ * Predicate: does this instance contain a file reference.
+ * @return true if there is a file.
+ */
+ boolean hasFile() {
+ return file != null;
+ }
+
+ /**
+ * Get the file, if there is one.
+ * @return the file for uploading, or null.
+ */
+ File getFile() {
+ return file;
+ }
+
+ /**
+ * Get the raw upload stream, if the object was
+ * created with one.
+ * @return the upload stream or null.
+ */
+ InputStream getUploadStream() {
+ return uploadStream;
+ }
+
+ /**
+ * Close: closes any upload stream provided in the constructor.
+ * @throws IOException inherited exception
+ */
+ @Override
+ public void close() throws IOException {
+ closeAll(LOG, uploadStream);
+ }
+ }
+
+ /**
* Base class for block factories.
*/
static abstract class BlockFactory implements Closeable {
@@ -110,15 +173,21 @@ final class S3ADataBlocks {
/**
* Create a block.
+ *
+ * @param index index of block
* @param limit limit of the block.
+ * @param statistics stats to work with
* @return a new block.
*/
- abstract DataBlock create(int limit) throws IOException;
+ abstract DataBlock create(long index, int limit,
+ S3AInstrumentation.OutputStreamStatistics statistics)
+ throws IOException;
/**
* Implement any close/cleanup operation.
* Base class is a no-op
- * @throws IOException -ideally, it shouldn't.
+ * @throws IOException Inherited exception; implementations should
+ * avoid raising it.
*/
@Override
public void close() throws IOException {
@@ -140,6 +209,14 @@ final class S3ADataBlocks {
enum DestState {Writing, Upload, Closed}
private volatile DestState state = Writing;
+ protected final long index;
+ protected final S3AInstrumentation.OutputStreamStatistics statistics;
+
+ protected DataBlock(long index,
+ S3AInstrumentation.OutputStreamStatistics statistics) {
+ this.index = index;
+ this.statistics = statistics;
+ }
/**
* Atomically enter a state, verifying current state.
@@ -243,8 +320,8 @@ final class S3ADataBlocks {
* @return the stream
* @throws IOException trouble
*/
- InputStream startUpload() throws IOException {
- LOG.debug("Start datablock upload");
+ BlockUploadData startUpload() throws IOException {
+ LOG.debug("Start datablock[{}] upload", index);
enterState(Writing, Upload);
return null;
}
@@ -278,6 +355,23 @@ final class S3ADataBlocks {
}
+ /**
+ * A block has been allocated.
+ */
+ protected void blockAllocated() {
+ if (statistics != null) {
+ statistics.blockAllocated();
+ }
+ }
+
+ /**
+ * A block has been released.
+ */
+ protected void blockReleased() {
+ if (statistics != null) {
+ statistics.blockReleased();
+ }
+ }
}
// ====================================================================
@@ -292,8 +386,10 @@ final class S3ADataBlocks {
}
@Override
- DataBlock create(int limit) throws IOException {
- return new ByteArrayBlock(limit);
+ DataBlock create(long index, int limit,
+ S3AInstrumentation.OutputStreamStatistics statistics)
+ throws IOException {
+ return new ByteArrayBlock(0, limit, statistics);
}
}
@@ -334,9 +430,13 @@ final class S3ADataBlocks {
// cache data size so that it is consistent after the buffer is reset.
private Integer dataSize;
- ByteArrayBlock(int limit) {
+ ByteArrayBlock(long index,
+ int limit,
+ S3AInstrumentation.OutputStreamStatistics statistics) {
+ super(index, statistics);
this.limit = limit;
buffer = new S3AByteArrayOutputStream(limit);
+ blockAllocated();
}
/**
@@ -349,12 +449,12 @@ final class S3ADataBlocks {
}
@Override
- InputStream startUpload() throws IOException {
+ BlockUploadData startUpload() throws IOException {
super.startUpload();
dataSize = buffer.size();
ByteArrayInputStream bufferData = buffer.getInputStream();
buffer = null;
- return bufferData;
+ return new BlockUploadData(bufferData);
}
@Override
@@ -378,12 +478,14 @@ final class S3ADataBlocks {
@Override
protected void innerClose() {
buffer = null;
+ blockReleased();
}
@Override
public String toString() {
- return "ByteArrayBlock{" +
- "state=" + getState() +
+ return "ByteArrayBlock{"
+ +"index=" + index +
+ ", state=" + getState() +
", limit=" + limit +
", dataSize=" + dataSize +
'}';
@@ -395,12 +497,6 @@ final class S3ADataBlocks {
/**
* Stream via Direct ByteBuffers; these are allocated off heap
* via {@link DirectBufferPool}.
- * This is actually the most complex of all the block factories,
- * due to the need to explicitly recycle buffers; in comparison, the
- * {@link DiskBlock} buffer delegates the work of deleting files to
- * the {@link DiskBlock.FileDeletingInputStream}. Here the
- * input stream {@link ByteBufferInputStream} has a similar task, along
- * with the foundational work of streaming data from a byte array.
*/
static class ByteBufferBlockFactory extends BlockFactory {
@@ -413,8 +509,10 @@ final class S3ADataBlocks {
}
@Override
- ByteBufferBlock create(int limit) throws IOException {
- return new ByteBufferBlock(limit);
+ ByteBufferBlock create(long index, int limit,
+ S3AInstrumentation.OutputStreamStatistics statistics)
+ throws IOException {
+ return new ByteBufferBlock(index, limit, statistics);
}
private ByteBuffer requestBuffer(int limit) {
@@ -446,21 +544,27 @@ final class S3ADataBlocks {
/**
* A DataBlock which requests a buffer from pool on creation; returns
- * it when the output stream is closed.
+ * it when it is closed.
*/
class ByteBufferBlock extends DataBlock {
- private ByteBuffer buffer;
+ private ByteBuffer blockBuffer;
private final int bufferSize;
// cache data size so that it is consistent after the buffer is reset.
private Integer dataSize;
/**
* Instantiate. This will request a ByteBuffer of the desired size.
+ * @param index block index
* @param bufferSize buffer size
+ * @param statistics statistics to update
*/
- ByteBufferBlock(int bufferSize) {
+ ByteBufferBlock(long index,
+ int bufferSize,
+ S3AInstrumentation.OutputStreamStatistics statistics) {
+ super(index, statistics);
this.bufferSize = bufferSize;
- buffer = requestBuffer(bufferSize);
+ blockBuffer = requestBuffer(bufferSize);
+ blockAllocated();
}
/**
@@ -473,13 +577,14 @@ final class S3ADataBlocks {
}
@Override
- ByteBufferInputStream startUpload() throws IOException {
+ BlockUploadData startUpload() throws IOException {
super.startUpload();
dataSize = bufferCapacityUsed();
// set the buffer up from reading from the beginning
- buffer.limit(buffer.position());
- buffer.position(0);
- return new ByteBufferInputStream(dataSize, buffer);
+ blockBuffer.limit(blockBuffer.position());
+ blockBuffer.position(0);
+ return new BlockUploadData(
+ new ByteBufferInputStream(dataSize, blockBuffer));
}
@Override
@@ -489,182 +594,190 @@ final class S3ADataBlocks {
@Override
public int remainingCapacity() {
- return buffer != null ? buffer.remaining() : 0;
+ return blockBuffer != null ? blockBuffer.remaining() : 0;
}
private int bufferCapacityUsed() {
- return buffer.capacity() - buffer.remaining();
+ return blockBuffer.capacity() - blockBuffer.remaining();
}
@Override
int write(byte[] b, int offset, int len) throws IOException {
super.write(b, offset, len);
int written = Math.min(remainingCapacity(), len);
- buffer.put(b, offset, written);
+ blockBuffer.put(b, offset, written);
return written;
}
+ /**
+ * Closing the block will release the buffer.
+ */
@Override
protected void innerClose() {
- buffer = null;
+ if (blockBuffer != null) {
+ blockReleased();
+ releaseBuffer(blockBuffer);
+ blockBuffer = null;
+ }
}
@Override
public String toString() {
return "ByteBufferBlock{"
- + "state=" + getState() +
+ + "index=" + index +
+ ", state=" + getState() +
", dataSize=" + dataSize() +
", limit=" + bufferSize +
", remainingCapacity=" + remainingCapacity() +
'}';
}
- }
-
- /**
- * Provide an input stream from a byte buffer; supporting
- * {@link #mark(int)}, which is required to enable replay of failed
- * PUT attempts.
- * This input stream returns the buffer to the pool afterwards.
- */
- class ByteBufferInputStream extends InputStream {
+ /**
+ * Provide an input stream from a byte buffer; supporting
+ * {@link #mark(int)}, which is required to enable replay of failed
+ * PUT attempts.
+ */
+ class ByteBufferInputStream extends InputStream {
- private final int size;
- private ByteBuffer byteBuffer;
+ private final int size;
+ private ByteBuffer byteBuffer;
- ByteBufferInputStream(int size, ByteBuffer byteBuffer) {
- LOG.debug("Creating ByteBufferInputStream of size {}", size);
- this.size = size;
- this.byteBuffer = byteBuffer;
- }
+ ByteBufferInputStream(int size,
+ ByteBuffer byteBuffer) {
+ LOG.debug("Creating ByteBufferInputStream of size {}", size);
+ this.size = size;
+ this.byteBuffer = byteBuffer;
+ }
- /**
- * Return the buffer to the pool after the stream is closed.
- */
- @Override
- public synchronized void close() {
- if (byteBuffer != null) {
- LOG.debug("releasing buffer");
- releaseBuffer(byteBuffer);
+ /**
+ * After the stream is closed, set the local reference to the byte
+ * buffer to null; this guarantees that future attempts to use
+ * stream methods will fail.
+ */
+ @Override
+ public synchronized void close() {
+ LOG.debug("ByteBufferInputStream.close() for {}",
+ ByteBufferBlock.super.toString());
byteBuffer = null;
}
- }
- /**
- * Verify that the stream is open.
- * @throws IOException if the stream is closed
- */
- private void verifyOpen() throws IOException {
- if (byteBuffer == null) {
- throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+ /**
+ * Verify that the stream is open.
+ * @throws IOException if the stream is closed
+ */
+ private void verifyOpen() throws IOException {
+ if (byteBuffer == null) {
+ throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+ }
}
- }
- public synchronized int read() throws IOException {
- if (available() > 0) {
- return byteBuffer.get() & 0xFF;
- } else {
- return -1;
+ public synchronized int read() throws IOException {
+ if (available() > 0) {
+ return byteBuffer.get() & 0xFF;
+ } else {
+ return -1;
+ }
}
- }
- @Override
- public synchronized long skip(long offset) throws IOException {
- verifyOpen();
- long newPos = position() + offset;
- if (newPos < 0) {
- throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
+ @Override
+ public synchronized long skip(long offset) throws IOException {
+ verifyOpen();
+ long newPos = position() + offset;
+ if (newPos < 0) {
+ throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
+ }
+ if (newPos > size) {
+ throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
+ }
+ byteBuffer.position((int) newPos);
+ return newPos;
}
- if (newPos > size) {
- throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
+
+ @Override
+ public synchronized int available() {
+ Preconditions.checkState(byteBuffer != null,
+ FSExceptionMessages.STREAM_IS_CLOSED);
+ return byteBuffer.remaining();
}
- byteBuffer.position((int) newPos);
- return newPos;
- }
- @Override
- public synchronized int available() {
- Preconditions.checkState(byteBuffer != null,
- FSExceptionMessages.STREAM_IS_CLOSED);
- return byteBuffer.remaining();
- }
+ /**
+ * Get the current buffer position.
+ * @return the buffer position
+ */
+ public synchronized int position() {
+ return byteBuffer.position();
+ }
- /**
- * Get the current buffer position.
- * @return the buffer position
- */
- public synchronized int position() {
- return byteBuffer.position();
- }
+ /**
+ * Check if there is data left.
+ * @return true if there is data remaining in the buffer.
+ */
+ public synchronized boolean hasRemaining() {
+ return byteBuffer.hasRemaining();
+ }
- /**
- * Check if there is data left.
- * @return true if there is data remaining in the buffer.
- */
- public synchronized boolean hasRemaining() {
- return byteBuffer.hasRemaining();
- }
+ @Override
+ public synchronized void mark(int readlimit) {
+ LOG.debug("mark at {}", position());
+ byteBuffer.mark();
+ }
- @Override
- public synchronized void mark(int readlimit) {
- LOG.debug("mark at {}", position());
- byteBuffer.mark();
- }
+ @Override
+ public synchronized void reset() throws IOException {
+ LOG.debug("reset");
+ byteBuffer.reset();
+ }
- @Override
- public synchronized void reset() throws IOException {
- LOG.debug("reset");
- byteBuffer.reset();
- }
+ @Override
+ public boolean markSupported() {
+ return true;
+ }
- @Override
- public boolean markSupported() {
- return true;
- }
+ /**
+ * Read in data.
+ * @param b destination buffer
+ * @param offset offset within the buffer
+ * @param length length of bytes to read
+ * @throws EOFException if the position is negative
+ * @throws IndexOutOfBoundsException if there isn't space for the
+ * amount of data requested.
+ * @throws IllegalArgumentException other arguments are invalid.
+ */
+ @SuppressWarnings("NullableProblems")
+ public synchronized int read(byte[] b, int offset, int length)
+ throws IOException {
+ Preconditions.checkArgument(length >= 0, "length is negative");
+ Preconditions.checkArgument(b != null, "Null buffer");
+ if (b.length - offset < length) {
+ throw new IndexOutOfBoundsException(
+ FSExceptionMessages.TOO_MANY_BYTES_FOR_DEST_BUFFER
+ + ": request length =" + length
+ + ", with offset =" + offset
+ + "; buffer capacity =" + (b.length - offset));
+ }
+ verifyOpen();
+ if (!hasRemaining()) {
+ return -1;
+ }
- /**
- * Read in data.
- * @param buffer destination buffer
- * @param offset offset within the buffer
- * @param length length of bytes to read
- * @throws EOFException if the position is negative
- * @throws IndexOutOfBoundsException if there isn't space for the
- * amount of data requested.
- * @throws IllegalArgumentException other arguments are invalid.
- */
- @SuppressWarnings("NullableProblems")
- public synchronized int read(byte[] buffer, int offset, int length)
- throws IOException {
- Preconditions.checkArgument(length >= 0, "length is negative");
- Preconditions.checkArgument(buffer != null, "Null buffer");
- if (buffer.length - offset < length) {
- throw new IndexOutOfBoundsException(
- FSExceptionMessages.TOO_MANY_BYTES_FOR_DEST_BUFFER
- + ": request length =" + length
- + ", with offset =" + offset
- + "; buffer capacity =" + (buffer.length - offset));
+ int toRead = Math.min(length, available());
+ byteBuffer.get(b, offset, toRead);
+ return toRead;
}
- verifyOpen();
- if (!hasRemaining()) {
- return -1;
- }
-
- int toRead = Math.min(length, available());
- byteBuffer.get(buffer, offset, toRead);
- return toRead;
- }
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder(
- "ByteBufferInputStream{");
- sb.append("size=").append(size);
- ByteBuffer buffer = this.byteBuffer;
- if (buffer != null) {
- sb.append(", available=").append(buffer.remaining());
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "ByteBufferInputStream{");
+ sb.append("size=").append(size);
+ ByteBuffer buf = this.byteBuffer;
+ if (buf != null) {
+ sb.append(", available=").append(buf.remaining());
+ }
+ sb.append(", ").append(ByteBufferBlock.super.toString());
+ sb.append('}');
+ return sb.toString();
}
- sb.append('}');
- return sb.toString();
}
}
}
@@ -681,22 +794,29 @@ final class S3ADataBlocks {
}
/**
- * Create a temp file and a block which writes to it.
+ * Create a temp file and a {@link DiskBlock} instance to manage it.
+ *
+ * @param index block index
* @param limit limit of the block.
+ * @param statistics statistics to update
* @return the new block
* @throws IOException IO problems
*/
@Override
- DataBlock create(int limit) throws IOException {
+ DataBlock create(long index,
+ int limit,
+ S3AInstrumentation.OutputStreamStatistics statistics)
+ throws IOException {
File destFile = getOwner()
- .createTmpFileForWrite("s3ablock", limit, getOwner().getConf());
- return new DiskBlock(destFile, limit);
+ .createTmpFileForWrite(String.format("s3ablock-%04d-", index),
+ limit, getOwner().getConf());
+ return new DiskBlock(destFile, limit, index, statistics);
}
}
/**
* Stream to a file.
- * This will stop at the limit; the caller is expected to create a new block
+ * This will stop at the limit; the caller is expected to create a new block.
*/
static class DiskBlock extends DataBlock {
@@ -704,12 +824,17 @@ final class S3ADataBlocks {
private final File bufferFile;
private final int limit;
private BufferedOutputStream out;
- private InputStream uploadStream;
+ private final AtomicBoolean closed = new AtomicBoolean(false);
- DiskBlock(File bufferFile, int limit)
+ DiskBlock(File bufferFile,
+ int limit,
+ long index,
+ S3AInstrumentation.OutputStreamStatistics statistics)
throws FileNotFoundException {
+ super(index, statistics);
this.limit = limit;
this.bufferFile = bufferFile;
+ blockAllocated();
out = new BufferedOutputStream(new FileOutputStream(bufferFile));
}
@@ -738,7 +863,7 @@ final class S3ADataBlocks {
}
@Override
- InputStream startUpload() throws IOException {
+ BlockUploadData startUpload() throws IOException {
super.startUpload();
try {
out.flush();
@@ -746,8 +871,7 @@ final class S3ADataBlocks {
out.close();
out = null;
}
- uploadStream = new FileInputStream(bufferFile);
- return new FileDeletingInputStream(uploadStream);
+ return new BlockUploadData(bufferFile);
}
/**
@@ -755,6 +879,7 @@ final class S3ADataBlocks {
* exists.
* @throws IOException IO problems
*/
+ @SuppressWarnings("UnnecessaryDefault")
@Override
protected void innerClose() throws IOException {
final DestState state = getState();
@@ -763,20 +888,19 @@ final class S3ADataBlocks {
case Writing:
if (bufferFile.exists()) {
// file was not uploaded
- LOG.debug("Deleting buffer file as upload did not start");
- boolean deleted = bufferFile.delete();
- if (!deleted && bufferFile.exists()) {
- LOG.warn("Failed to delete buffer file {}", bufferFile);
- }
+ LOG.debug("Block[{}]: Deleting buffer file as upload did not start",
+ index);
+ closeBlock();
}
break;
case Upload:
- LOG.debug("Buffer file {} exists \u2014close upload stream", bufferFile);
+ LOG.debug("Block[{}]: Buffer file {} exists \u2014close upload stream",
+ index, bufferFile);
break;
case Closed:
- // no-op
+ closeBlock();
break;
default:
@@ -798,7 +922,8 @@ final class S3ADataBlocks {
@Override
public String toString() {
String sb = "FileBlock{"
- + "destFile=" + bufferFile +
+ + "index=" + index
+ + ", destFile=" + bufferFile +
", state=" + getState() +
", dataSize=" + dataSize() +
", limit=" + limit +
@@ -807,31 +932,20 @@ final class S3ADataBlocks {
}
/**
- * An input stream which deletes the buffer file when closed.
+ * Close the block.
+ * This will delete the block's buffer file if the block has
+ * not previously been closed.
*/
- private final class FileDeletingInputStream extends FilterInputStream {
- private final AtomicBoolean closed = new AtomicBoolean(false);
-
- FileDeletingInputStream(InputStream source) {
- super(source);
- }
-
- /**
- * Delete the input file when closed.
- * @throws IOException IO problem
- */
- @Override
- public void close() throws IOException {
- try {
- super.close();
- } finally {
- if (!closed.getAndSet(true)) {
- if (!bufferFile.delete()) {
- LOG.warn("delete({}) returned false",
- bufferFile.getAbsoluteFile());
- }
- }
+ void closeBlock() {
+ LOG.debug("block[{}]: closeBlock()", index);
+ if (!closed.getAndSet(true)) {
+ blockReleased();
+ if (!bufferFile.delete() && bufferFile.exists()) {
+ LOG.warn("delete({}) returned false",
+ bufferFile.getAbsoluteFile());
}
+ } else {
+ LOG.debug("block[{}]: skipping re-entrant closeBlock()", index);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index bc47918..1786e68 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -1022,6 +1022,7 @@ public class S3AFileSystem extends FileSystem {
*/
public PutObjectRequest newPutObjectRequest(String key,
ObjectMetadata metadata, File srcfile) {
+ Preconditions.checkNotNull(srcfile);
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key,
srcfile);
setOptionalPutRequestParameters(putObjectRequest);
@@ -1039,8 +1040,9 @@ public class S3AFileSystem extends FileSystem {
* @param inputStream source data.
* @return the request
*/
- PutObjectRequest newPutObjectRequest(String key,
+ private PutObjectRequest newPutObjectRequest(String key,
ObjectMetadata metadata, InputStream inputStream) {
+ Preconditions.checkNotNull(inputStream);
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key,
inputStream, metadata);
setOptionalPutRequestParameters(putObjectRequest);
@@ -1077,12 +1079,16 @@ public class S3AFileSystem extends FileSystem {
}
/**
- * PUT an object, incrementing the put requests and put bytes
+ * Start a transfer-manager managed async PUT of an object,
+ * incrementing the put requests and put bytes
* counters.
* It does not update the other counters,
* as existing code does that as progress callbacks come in.
* Byte length is calculated from the file length, or, if there is no
* file, from the content length of the header.
+ * Because the operation is async, any stream supplied in the request
+ * must reference data (files, buffers) which stay valid until the upload
+ * completes.
* @param putObjectRequest the request
* @return the upload initiated
*/
@@ -1108,6 +1114,7 @@ public class S3AFileSystem extends FileSystem {
* PUT an object directly (i.e. not via the transfer manager).
* Byte length is calculated from the file length, or, if there is no
* file, from the content length of the header.
+ * <i>Important: this call will close any input stream in the request.</i>
* @param putObjectRequest the request
* @return the upload initiated
* @throws AmazonClientException on problems
@@ -1133,7 +1140,8 @@ public class S3AFileSystem extends FileSystem {
/**
* Upload part of a multi-partition file.
- * Increments the write and put counters
+ * Increments the write and put counters.
+ * <i>Important: this call does not close any input stream in the request.</i>
* @param request request
* @return the result of the operation.
* @throws AmazonClientException on problems
@@ -2309,14 +2317,28 @@ public class S3AFileSystem extends FileSystem {
/**
* Create a {@link PutObjectRequest} request.
- * The metadata is assumed to have been configured with the size of the
- * operation.
+ * If {@code length} is set, the metadata is configured with the size of
+ * the upload.
* @param inputStream source data.
* @param length size, if known. Use -1 for not known
* @return the request
*/
PutObjectRequest newPutRequest(InputStream inputStream, long length) {
- return newPutObjectRequest(key, newObjectMetadata(length), inputStream);
+ PutObjectRequest request = newPutObjectRequest(key,
+ newObjectMetadata(length), inputStream);
+ return request;
+ }
+
+ /**
+ * Create a {@link PutObjectRequest} request to upload a file.
+ * @param sourceFile source file
+ * @return the request
+ */
+ PutObjectRequest newPutRequest(File sourceFile) {
+ int length = (int) sourceFile.length();
+ PutObjectRequest request = newPutObjectRequest(key,
+ newObjectMetadata(length), sourceFile);
+ return request;
}
/**
@@ -2379,6 +2401,8 @@ public class S3AFileSystem extends FileSystem {
Preconditions.checkNotNull(partETags);
Preconditions.checkArgument(!partETags.isEmpty(),
"No partitions have been uploaded");
+ LOG.debug("Completing multipart upload {} with {} parts",
+ uploadId, partETags.size());
return s3.completeMultipartUpload(
new CompleteMultipartUploadRequest(bucket,
key,
@@ -2389,42 +2413,51 @@ public class S3AFileSystem extends FileSystem {
/**
* Abort a multipart upload operation.
* @param uploadId multipart operation Id
- * @return the result
* @throws AmazonClientException on problems.
*/
void abortMultipartUpload(String uploadId) throws AmazonClientException {
+ LOG.debug("Aborting multipart upload {}", uploadId);
s3.abortMultipartUpload(
new AbortMultipartUploadRequest(bucket, key, uploadId));
}
/**
* Create and initialize a part request of a multipart upload.
+ * Exactly one of: {@code uploadStream} or {@code sourceFile}
+ * must be specified.
* @param uploadId ID of ongoing upload
- * @param uploadStream source of data to upload
* @param partNumber current part number of the upload
* @param size amount of data
+ * @param uploadStream source of data to upload
+ * @param sourceFile optional source file.
* @return the request.
*/
UploadPartRequest newUploadPartRequest(String uploadId,
- InputStream uploadStream,
- int partNumber,
- int size) {
+ int partNumber, int size, InputStream uploadStream, File sourceFile) {
Preconditions.checkNotNull(uploadId);
- Preconditions.checkNotNull(uploadStream);
+ // exactly one source must be set; xor verifies this
+ Preconditions.checkArgument((uploadStream != null) ^ (sourceFile != null),
+ "Data source");
Preconditions.checkArgument(size > 0, "Invalid partition size %s", size);
- Preconditions.checkArgument(partNumber> 0 && partNumber <=10000,
+ Preconditions.checkArgument(partNumber > 0 && partNumber <= 10000,
"partNumber must be between 1 and 10000 inclusive, but is %s",
partNumber);
LOG.debug("Creating part upload request for {} #{} size {}",
uploadId, partNumber, size);
- return new UploadPartRequest()
+ UploadPartRequest request = new UploadPartRequest()
.withBucketName(bucket)
.withKey(key)
.withUploadId(uploadId)
- .withInputStream(uploadStream)
.withPartNumber(partNumber)
.withPartSize(size);
+ if (uploadStream != null) {
+ // there's an upload stream. Bind to it.
+ request.setInputStream(uploadStream);
+ } else {
+ request.setFile(sourceFile);
+ }
+ return request;
}
/**
@@ -2439,6 +2472,21 @@ public class S3AFileSystem extends FileSystem {
sb.append('}');
return sb.toString();
}
+
+ /**
+ * PUT an object directly (i.e. not via the transfer manager).
+ * @param putObjectRequest the request
+ * @return the upload initiated
+ * @throws IOException on problems
+ */
+ PutObjectResult putObject(PutObjectRequest putObjectRequest)
+ throws IOException {
+ try {
+ return putObjectDirect(putObjectRequest);
+ } catch (AmazonClientException e) {
+ throw translateException("put", putObjectRequest.getKey(), e);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
index fb8c852..d2e7a88 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
@@ -36,6 +36,7 @@ import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.fs.FileSystem.Statistics;
@@ -428,7 +429,7 @@ public class S3AInstrumentation {
if (gauge != null) {
gauge.decr(count);
} else {
- LOG.debug("No Gauge: " + op);
+ LOG.debug("No Gauge: {}", op);
}
}
@@ -676,6 +677,8 @@ public class S3AInstrumentation {
private final AtomicLong transferDuration = new AtomicLong(0);
private final AtomicLong queueDuration = new AtomicLong(0);
private final AtomicLong exceptionsInMultipartFinalize = new AtomicLong(0);
+ private final AtomicInteger blocksAllocated = new AtomicInteger(0);
+ private final AtomicInteger blocksReleased = new AtomicInteger(0);
private Statistics statistics;
@@ -684,6 +687,20 @@ public class S3AInstrumentation {
}
/**
+ * A block has been allocated.
+ */
+ void blockAllocated() {
+ blocksAllocated.incrementAndGet();
+ }
+
+ /**
+ * A block has been released.
+ */
+ void blockReleased() {
+ blocksReleased.incrementAndGet();
+ }
+
+ /**
* Block is queued for upload.
*/
void blockUploadQueued(int blockSize) {
@@ -778,6 +795,24 @@ public class S3AInstrumentation {
return queueDuration.get() + transferDuration.get();
}
+ public int blocksAllocated() {
+ return blocksAllocated.get();
+ }
+
+ public int blocksReleased() {
+ return blocksReleased.get();
+ }
+
+ /**
+ * Get counters of blocks actively allocated; my be inaccurate
+ * if the numbers change during the (non-synchronized) calculation.
+ * @return the number of actively allocated blocks.
+ */
+ public int blocksActivelyAllocated() {
+ return blocksAllocated.get() - blocksReleased.get();
+ }
+
+
@Override
public String toString() {
final StringBuilder sb = new StringBuilder(
@@ -789,6 +824,9 @@ public class S3AInstrumentation {
sb.append(", blockUploadsFailed=").append(blockUploadsFailed);
sb.append(", bytesPendingUpload=").append(bytesPendingUpload);
sb.append(", bytesUploaded=").append(bytesUploaded);
+ sb.append(", blocksAllocated=").append(blocksAllocated);
+ sb.append(", blocksReleased=").append(blocksReleased);
+ sb.append(", blocksActivelyAllocated=").append(blocksActivelyAllocated());
sb.append(", exceptionsInMultipartFinalize=").append(
exceptionsInMultipartFinalize);
sb.append(", transferDuration=").append(transferDuration).append(" ms");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 5311211..84f3c99 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -733,4 +733,30 @@ public final class S3AUtils {
}
return null;
}
+
+ /**
+ * Close the Closeable objects and <b>ignore</b> any Exception or
+ * null pointers.
+ * (This is the SLF4J equivalent of that in {@code IOUtils}).
+ * @param log the log to log at debug level. Can be null.
+ * @param closeables the objects to close
+ */
+ public static void closeAll(Logger log,
+ java.io.Closeable... closeables) {
+ for (java.io.Closeable c : closeables) {
+ if (c != null) {
+ try {
+ if (log != null) {
+ log.debug("Closing {}", c);
+ }
+ c.close();
+ } catch (Exception e) {
+ if (log != null && log.isDebugEnabled()) {
+ log.debug("Exception in closing {}", c, e);
+ }
+ }
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java
index 74cad00..87f676c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java
@@ -24,9 +24,12 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.io.IOUtils;
+import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
import static org.apache.hadoop.fs.s3a.Constants.*;
@@ -38,6 +41,14 @@ import static org.apache.hadoop.fs.s3a.Constants.*;
* multipart tests are kept in scale tests.
*/
public class ITestS3ABlockOutputArray extends AbstractS3ATestBase {
+ private static final int BLOCK_SIZE = 256 * 1024;
+
+ private static byte[] dataset;
+
+ @BeforeClass
+ public static void setupDataset() {
+ dataset = ContractTestUtils.dataset(BLOCK_SIZE, 0, 256);
+ }
@Override
protected Configuration createConfiguration() {
@@ -65,9 +76,9 @@ public class ITestS3ABlockOutputArray extends AbstractS3ATestBase {
}
@Test(expected = IOException.class)
- public void testDoubleStreamClose() throws Throwable {
- Path dest = path("testDoubleStreamClose");
- describe(" testDoubleStreamClose");
+ public void testWriteAfterStreamClose() throws Throwable {
+ Path dest = path("testWriteAfterStreamClose");
+ describe(" testWriteAfterStreamClose");
FSDataOutputStream stream = getFileSystem().create(dest, true);
byte[] data = ContractTestUtils.dataset(16, 'a', 26);
try {
@@ -79,7 +90,25 @@ public class ITestS3ABlockOutputArray extends AbstractS3ATestBase {
}
}
- public void verifyUpload(String name, int fileSize) throws IOException {
+ @Test
+ public void testBlocksClosed() throws Throwable {
+ Path dest = path("testBlocksClosed");
+ describe(" testBlocksClosed");
+ FSDataOutputStream stream = getFileSystem().create(dest, true);
+ S3AInstrumentation.OutputStreamStatistics statistics
+ = S3ATestUtils.getOutputStreamStatistics(stream);
+ byte[] data = ContractTestUtils.dataset(16, 'a', 26);
+ stream.write(data);
+ LOG.info("closing output stream");
+ stream.close();
+ assertEquals("total allocated blocks in " + statistics,
+ 1, statistics.blocksAllocated());
+ assertEquals("actively allocated blocks in " + statistics,
+ 0, statistics.blocksActivelyAllocated());
+ LOG.info("end of test case");
+ }
+
+ private void verifyUpload(String name, int fileSize) throws IOException {
Path dest = path(name);
describe(name + " upload to " + dest);
ContractTestUtils.createAndVerifyFile(
@@ -87,4 +116,43 @@ public class ITestS3ABlockOutputArray extends AbstractS3ATestBase {
dest,
fileSize);
}
+
+ /**
+ * Create a factory for used in mark/reset tests.
+ * @param fileSystem source FS
+ * @return the factory
+ */
+ protected S3ADataBlocks.BlockFactory createFactory(S3AFileSystem fileSystem) {
+ return new S3ADataBlocks.ArrayBlockFactory(fileSystem);
+ }
+
+ private void markAndResetDatablock(S3ADataBlocks.BlockFactory factory)
+ throws Exception {
+ S3AInstrumentation instrumentation =
+ new S3AInstrumentation(new URI("s3a://example"));
+ S3AInstrumentation.OutputStreamStatistics outstats
+ = instrumentation.newOutputStreamStatistics(null);
+ S3ADataBlocks.DataBlock block = factory.create(1, BLOCK_SIZE, outstats);
+ block.write(dataset, 0, dataset.length);
+ S3ADataBlocks.BlockUploadData uploadData = block.startUpload();
+ InputStream stream = uploadData.getUploadStream();
+ assertNotNull(stream);
+ assertTrue("Mark not supported in " + stream, stream.markSupported());
+ assertEquals(0, stream.read());
+ stream.mark(BLOCK_SIZE);
+ // read a lot
+ long l = 0;
+ while (stream.read() != -1) {
+ // do nothing
+ l++;
+ }
+ stream.reset();
+ assertEquals(1, stream.read());
+ }
+
+ @Test
+ public void testMarkReset() throws Throwable {
+ markAndResetDatablock(createFactory(getFileSystem()));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputByteBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputByteBuffer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputByteBuffer.java
index 504426b..02f3de0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputByteBuffer.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputByteBuffer.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.fs.s3a;
-
/**
* Use {@link Constants#FAST_UPLOAD_BYTEBUFFER} for buffering.
*/
@@ -27,4 +26,8 @@ public class ITestS3ABlockOutputByteBuffer extends ITestS3ABlockOutputArray {
return Constants.FAST_UPLOAD_BYTEBUFFER;
}
+ protected S3ADataBlocks.BlockFactory createFactory(S3AFileSystem fileSystem) {
+ return new S3ADataBlocks.ByteBufferBlockFactory(fileSystem);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java
index 550706d..abe8656 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.fs.s3a;
+import org.junit.Assume;
+
/**
* Use {@link Constants#FAST_UPLOAD_BUFFER_DISK} for buffering.
*/
@@ -27,4 +29,14 @@ public class ITestS3ABlockOutputDisk extends ITestS3ABlockOutputArray {
return Constants.FAST_UPLOAD_BUFFER_DISK;
}
+ /**
+ * The disk stream doesn't support mark/reset; calls
+ * {@code Assume} to skip the test.
+ * @param fileSystem source FS
+ * @return null
+ */
+ protected S3ADataBlocks.BlockFactory createFactory(S3AFileSystem fileSystem) {
+ Assume.assumeTrue("mark/reset nopt supoprted", false);
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 567bacb..9528967 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.s3a;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.junit.Assert;
@@ -544,4 +545,16 @@ public final class S3ATestUtils {
}
Assume.assumeTrue(message, condition);
}
+
+ /**
+ * Get the statistics from a wrapped block output stream.
+ * @param out output stream
+ * @return the (active) stats of the write
+ */
+ public static S3AInstrumentation.OutputStreamStatistics
+ getOutputStreamStatistics(FSDataOutputStream out) {
+ S3ABlockOutputStream blockOutputStream
+ = (S3ABlockOutputStream) out.getWrappedStream();
+ return blockOutputStream.getStatistics();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java
index 9fa95fd..700ef5c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java
@@ -51,9 +51,8 @@ public class TestDataBlocks extends Assert {
new S3ADataBlocks.ByteBufferBlockFactory(null)) {
int limit = 128;
S3ADataBlocks.ByteBufferBlockFactory.ByteBufferBlock block
- = factory.create(limit);
- assertEquals("outstanding buffers in " + factory,
- 1, factory.getOutstandingBufferCount());
+ = factory.create(1, limit, null);
+ assertOutstandingBuffers(factory, 1);
byte[] buffer = ContractTestUtils.toAsciiByteArray("test data");
int bufferLen = buffer.length;
@@ -66,24 +65,23 @@ public class TestDataBlocks extends Assert {
block.hasCapacity(limit - bufferLen));
// now start the write
- S3ADataBlocks.ByteBufferBlockFactory.ByteBufferInputStream
- stream = block.startUpload();
+ S3ADataBlocks.BlockUploadData blockUploadData = block.startUpload();
+ S3ADataBlocks.ByteBufferBlockFactory.ByteBufferBlock.ByteBufferInputStream
+ stream =
+ (S3ADataBlocks.ByteBufferBlockFactory.ByteBufferBlock.ByteBufferInputStream)
+ blockUploadData.getUploadStream();
+ assertTrue("Mark not supported in " + stream, stream.markSupported());
assertTrue("!hasRemaining() in " + stream, stream.hasRemaining());
int expected = bufferLen;
assertEquals("wrong available() in " + stream,
expected, stream.available());
assertEquals('t', stream.read());
+ stream.mark(limit);
expected--;
assertEquals("wrong available() in " + stream,
expected, stream.available());
- // close the block. The buffer must remain outstanding here;
- // the stream manages the lifecycle of it now
- block.close();
- assertEquals("outstanding buffers in " + factory,
- 1, factory.getOutstandingBufferCount());
- block.close();
// read into a byte array with an offset
int offset = 5;
@@ -109,16 +107,31 @@ public class TestDataBlocks extends Assert {
0, stream.available());
assertTrue("hasRemaining() in " + stream, !stream.hasRemaining());
+ // go the mark point
+ stream.reset();
+ assertEquals('e', stream.read());
+
// when the stream is closed, the data should be returned
stream.close();
- assertEquals("outstanding buffers in " + factory,
- 0, factory.getOutstandingBufferCount());
+ assertOutstandingBuffers(factory, 1);
+ block.close();
+ assertOutstandingBuffers(factory, 0);
stream.close();
- assertEquals("outstanding buffers in " + factory,
- 0, factory.getOutstandingBufferCount());
-
+ assertOutstandingBuffers(factory, 0);
}
}
+ /**
+ * Assert the number of buffers active for a block factory.
+ * @param factory factory
+ * @param expectedCount expected count.
+ */
+ private static void assertOutstandingBuffers(
+ S3ADataBlocks.ByteBufferBlockFactory factory,
+ int expectedCount) {
+ assertEquals("outstanding buffers in " + factory,
+ expectedCount, factory.getOutstandingBufferCount());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dab00da1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
index fcb6444..89fae82 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
@@ -34,11 +34,13 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AInstrumentation;
import org.apache.hadoop.fs.s3a.Statistic;
import org.apache.hadoop.util.Progressable;
@@ -159,13 +161,20 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
Statistic putBytesPending = Statistic.OBJECT_PUT_BYTES_PENDING;
ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
-
+ S3AInstrumentation.OutputStreamStatistics streamStatistics;
long blocksPer10MB = blocksPerMB * 10;
ProgressCallback progress = new ProgressCallback(timer);
try (FSDataOutputStream out = fs.create(hugefile,
true,
uploadBlockSize,
progress)) {
+ try {
+ streamStatistics = getOutputStreamStatistics(out);
+ } catch (ClassCastException e) {
+ LOG.info("Wrapped output stream is not block stream: {}",
+ out.getWrappedStream());
+ streamStatistics = null;
+ }
for (long block = 1; block <= blocks; block++) {
out.write(data);
@@ -190,7 +199,8 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
}
}
// now close the file
- LOG.info("Closing file and completing write operation");
+ LOG.info("Closing stream {}", out);
+ LOG.info("Statistics : {}", streamStatistics);
ContractTestUtils.NanoTimer closeTimer
= new ContractTestUtils.NanoTimer();
out.close();
@@ -201,6 +211,7 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
filesizeMB, uploadBlockSize);
logFSState();
bandwidth(timer, filesize);
+ LOG.info("Statistics after stream closed: {}", streamStatistics);
long putRequestCount = storageStatistics.getLong(putRequests);
Long putByteCount = storageStatistics.getLong(putBytes);
LOG.info("PUT {} bytes in {} operations; {} MB/operation",
@@ -214,7 +225,14 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
S3AFileStatus status = fs.getFileStatus(hugefile);
ContractTestUtils.assertIsFile(hugefile, status);
assertEquals("File size in " + status, filesize, status.getLen());
- progress.verifyNoFailures("Put file " + hugefile + " of size " + filesize);
+ if (progress != null) {
+ progress.verifyNoFailures("Put file " + hugefile
+ + " of size " + filesize);
+ }
+ if (streamStatistics != null) {
+ assertEquals("actively allocated blocks in " + streamStatistics,
+ 0, streamStatistics.blocksActivelyAllocated());
+ }
}
/**
@@ -285,7 +303,9 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
void assumeHugeFileExists() throws IOException {
S3AFileSystem fs = getFileSystem();
ContractTestUtils.assertPathExists(fs, "huge file not created", hugefile);
- ContractTestUtils.assertIsFile(fs, hugefile);
+ FileStatus status = fs.getFileStatus(hugefile);
+ ContractTestUtils.assertIsFile(hugefile, status);
+ assertTrue("File " + hugefile + " is empty", status.getLen() > 0);
}
private void logFSState() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDFS-11419. HDFS specific network
topology classes with storage type info included. Contributed by Chen Liang.
Posted by jh...@apache.org.
HDFS-11419. HDFS specific network topology classes with storage type info included. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eeca8b0c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eeca8b0c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eeca8b0c
Branch: refs/heads/YARN-5734
Commit: eeca8b0c4e2804b0fee5b012ea14b58383425ec3
Parents: 4e14ead
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Mar 2 09:21:27 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Mar 2 09:21:27 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/net/InnerNode.java | 2 +-
.../org/apache/hadoop/net/InnerNodeImpl.java | 22 +-
.../org/apache/hadoop/net/NetworkTopology.java | 27 +-
.../apache/hadoop/net/TestClusterTopology.java | 5 +-
.../blockmanagement/DatanodeDescriptor.java | 9 +
.../apache/hadoop/hdfs/DFSNetworkTopology.java | 36 +++
.../apache/hadoop/hdfs/DFSTopologyNodeImpl.java | 253 ++++++++++++++++++
.../hadoop/hdfs/TestBlockStoragePolicy.java | 2 +-
.../hadoop/hdfs/TestDFSNetworkTopology.java | 260 +++++++++++++++++++
.../apache/hadoop/net/TestNetworkTopology.java | 6 +-
10 files changed, 601 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
index d07929b..65eaf24 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
@@ -61,7 +61,7 @@ public interface InnerNode extends Node {
*
* @param leafIndex an indexed leaf of the node
* @param excludedNode an excluded node (can be null)
- * @return
+ * @return the leaf node corresponding to the given index.
*/
Node getLeaf(int leafIndex, Node excludedNode);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
index e6aa0f7..81eaf7f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
@@ -25,9 +25,9 @@ import java.util.Map;
/** InnerNode represents a switch/router of a data center or rack.
* Different from a leaf node, it has non-null children.
*/
-class InnerNodeImpl extends NodeBase implements InnerNode {
- static class Factory implements InnerNode.Factory<InnerNodeImpl> {
- private Factory() {}
+public class InnerNodeImpl extends NodeBase implements InnerNode {
+ protected static class Factory implements InnerNode.Factory<InnerNodeImpl> {
+ protected Factory() {}
@Override
public InnerNodeImpl newInnerNode(String path) {
@@ -37,18 +37,18 @@ class InnerNodeImpl extends NodeBase implements InnerNode {
static final Factory FACTORY = new Factory();
- private final List<Node> children = new ArrayList<>();
- private final Map<String, Node> childrenMap = new HashMap<>();
- private int numOfLeaves;
+ protected final List<Node> children = new ArrayList<>();
+ protected final Map<String, Node> childrenMap = new HashMap<>();
+ protected int numOfLeaves;
/** Construct an InnerNode from a path-like string */
- InnerNodeImpl(String path) {
+ protected InnerNodeImpl(String path) {
super(path);
}
/** Construct an InnerNode
* from its name, its network location, its parent, and its level */
- InnerNodeImpl(String name, String location, InnerNode parent, int level) {
+ protected InnerNodeImpl(String name, String location, InnerNode parent, int level) {
super(name, location, parent, level);
}
@@ -81,7 +81,7 @@ class InnerNodeImpl extends NodeBase implements InnerNode {
* @param n a node
* @return true if this node is an ancestor of <i>n</i>
*/
- boolean isAncestor(Node n) {
+ protected boolean isAncestor(Node n) {
return getPath(this).equals(NodeBase.PATH_SEPARATOR_STR) ||
(n.getNetworkLocation()+NodeBase.PATH_SEPARATOR_STR).
startsWith(getPath(this)+NodeBase.PATH_SEPARATOR_STR);
@@ -92,12 +92,12 @@ class InnerNodeImpl extends NodeBase implements InnerNode {
* @param n a node
* @return true if this node is the parent of <i>n</i>
*/
- boolean isParent(Node n) {
+ protected boolean isParent(Node n) {
return n.getNetworkLocation().equals(getPath(this));
}
/* Return a child name of this node who is an ancestor of node <i>n</i> */
- private String getNextAncestorName(Node n) {
+ protected String getNextAncestorName(Node n) {
if (!isAncestor(n)) {
throw new IllegalArgumentException(
this + "is not an ancestor of " + n);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 38c02f8..051012b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -68,16 +68,32 @@ public class NetworkTopology {
* @return an instance of NetworkTopology
*/
public static NetworkTopology getInstance(Configuration conf){
- return ReflectionUtils.newInstance(
+ return getInstance(conf, InnerNodeImpl.FACTORY);
+ }
+
+ public static NetworkTopology getInstance(Configuration conf,
+ InnerNode.Factory factory) {
+ NetworkTopology nt = ReflectionUtils.newInstance(
conf.getClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
- NetworkTopology.class, NetworkTopology.class), conf);
+ NetworkTopology.class, NetworkTopology.class), conf);
+ return nt.init(factory);
+ }
+
+ protected NetworkTopology init(InnerNode.Factory factory) {
+ if (!factory.equals(this.factory)) {
+ // the constructor has initialized the factory to default. So only init
+ // again if another factory is specified.
+ this.factory = factory;
+ this.clusterMap = factory.newInnerNode(NodeBase.ROOT);
+ }
+ return this;
}
- InnerNode.Factory factory = InnerNodeImpl.FACTORY;
+ InnerNode.Factory factory;
/**
* the root cluster map
*/
- InnerNode clusterMap = factory.newInnerNode(NodeBase.ROOT);
+ InnerNode clusterMap;
/** Depth of all leaf nodes */
private int depthOfAllLeaves = -1;
/** rack counter */
@@ -92,7 +108,10 @@ public class NetworkTopology {
/** the lock used to manage access */
protected ReadWriteLock netlock = new ReentrantReadWriteLock();
+ // keeping the constructor because other components like MR still uses this.
public NetworkTopology() {
+ this.factory = InnerNodeImpl.FACTORY;
+ this.clusterMap = factory.newInnerNode(NodeBase.ROOT);
}
/** Add a leaf node
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
index 72fc5cb..09f0914 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.List;
import org.apache.commons.math3.stat.inference.ChiSquareTest;
+import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
import org.junit.Test;
@@ -80,7 +81,7 @@ public class TestClusterTopology extends Assert {
@Test
public void testCountNumNodes() throws Exception {
// create the topology
- NetworkTopology cluster = new NetworkTopology();
+ NetworkTopology cluster = NetworkTopology.getInstance(new Configuration());
NodeElement node1 = getNewNode("node1", "/d1/r1");
cluster.add(node1);
NodeElement node2 = getNewNode("node2", "/d1/r2");
@@ -128,7 +129,7 @@ public class TestClusterTopology extends Assert {
@Test
public void testChooseRandom() {
// create the topology
- NetworkTopology cluster = new NetworkTopology();
+ NetworkTopology cluster = NetworkTopology.getInstance(new Configuration());
NodeElement node1 = getNewNode("node1", "/d1/r1");
cluster.add(node1);
NodeElement node2 = getNewNode("node2", "/d1/r2");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index c180432..15af6ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -300,6 +301,14 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
}
+ public EnumSet<StorageType> getStorageTypes() {
+ EnumSet<StorageType> storageTypes = EnumSet.noneOf(StorageType.class);
+ for (DatanodeStorageInfo dsi : getStorageInfos()) {
+ storageTypes.add(dsi.getStorageType());
+ }
+ return storageTypes;
+ }
+
public StorageReport[] getStorageReports() {
final DatanodeStorageInfo[] infos = getStorageInfos();
final StorageReport[] reports = new StorageReport[infos.length];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java
new file mode 100644
index 0000000..a6b8c00
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetworkTopology;
+
+/**
+ * The HDFS specific network topology class. The main purpose of doing this
+ * subclassing is to add storage-type-aware chooseRandom method. All the
+ * remaining parts should be the same.
+ *
+ * Currently a placeholder to test storage type info.
+ * TODO : add "chooseRandom with storageType info" function.
+ */
+public class DFSNetworkTopology extends NetworkTopology {
+ public static DFSNetworkTopology getInstance(Configuration conf) {
+ DFSNetworkTopology nt = new DFSNetworkTopology();
+ return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java
new file mode 100644
index 0000000..e746823
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.net.InnerNode;
+import org.apache.hadoop.net.InnerNodeImpl;
+import org.apache.hadoop.net.Node;
+
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.HashMap;
+
+/**
+ * The HDFS-specific representation of a network topology inner node. The
+ * difference is this class includes the information about the storage type
+ * info of this subtree. This info will be used when selecting subtrees
+ * in block placement.
+ */
+public class DFSTopologyNodeImpl extends InnerNodeImpl {
+
+ static final InnerNodeImpl.Factory FACTORY
+ = new DFSTopologyNodeImpl.Factory();
+
+ static final class Factory extends InnerNodeImpl.Factory {
+ private Factory() {}
+
+ @Override
+ public InnerNodeImpl newInnerNode(String path) {
+ return new DFSTopologyNodeImpl(path);
+ }
+ }
+
+ /**
+ * The core data structure of this class. The information about what storage
+ * types this subtree has. Basically, a map whose key is a child
+ * id, value is a enum map including the counts of each storage type. e.g.
+ * DISK type has count 5 means there are 5 leaf datanodes with DISK type
+ * available. This value is set/updated upon datanode joining and leaving.
+ *
+ * NOTE : It might be sufficient to keep only a map from storage type
+ * to count, omitting the child node id. But this might make it hard to keep
+ * consistency when there are updates from children.
+ *
+ * For example, if currently R has two children A and B with storage X, Y, and
+ * A : X=1 Y=1
+ * B : X=2 Y=2
+ * so we store X=3 Y=3 as total on R.
+ *
+ * Now say A has a new X plugged in and becomes X=2 Y=1.
+ *
+ * If we know that "A adds one X", it is easy to update R by +1 on X. However,
+ * if we don't know "A adds one X", but instead got "A now has X=2 Y=1",
+ * (which seems to be the case in current heartbeat) we will not know how to
+ * update R. While if we store on R "A has X=1 and Y=1" then we can simply
+ * update R by completely replacing the A entry and all will be good.
+ */
+ private final HashMap
+ <String, EnumMap<StorageType, Integer>> childrenStorageInfo;
+
+ DFSTopologyNodeImpl(String path) {
+ super(path);
+ childrenStorageInfo = new HashMap<>();
+ }
+
+ DFSTopologyNodeImpl(
+ String name, String location, InnerNode parent, int level) {
+ super(name, location, parent, level);
+ childrenStorageInfo = new HashMap<>();
+ }
+
+ int getNumOfChildren() {
+ return children.size();
+ }
+
+ @Override
+ public boolean add(Node n) {
+ if (!isAncestor(n)) {
+ throw new IllegalArgumentException(n.getName()
+ + ", which is located at " + n.getNetworkLocation()
+ + ", is not a descendant of " + getPath(this));
+ }
+ // In HDFS topology, the leaf node should always be DatanodeDescriptor
+ if (!(n instanceof DatanodeDescriptor)) {
+ throw new IllegalArgumentException("Unexpected node type "
+ + n.getClass().getName());
+ }
+ DatanodeDescriptor dnDescriptor = (DatanodeDescriptor) n;
+ if (isParent(n)) {
+ // this node is the parent of n; add n directly
+ n.setParent(this);
+ n.setLevel(this.level + 1);
+ Node prev = childrenMap.put(n.getName(), n);
+ if (prev != null) {
+ for(int i=0; i<children.size(); i++) {
+ if (children.get(i).getName().equals(n.getName())) {
+ children.set(i, n);
+ return false;
+ }
+ }
+ }
+ children.add(n);
+ numOfLeaves++;
+ synchronized (childrenStorageInfo) {
+ if (!childrenStorageInfo.containsKey(dnDescriptor.getName())) {
+ childrenStorageInfo.put(
+ dnDescriptor.getName(), new EnumMap<>(StorageType.class));
+ }
+ for (StorageType st : dnDescriptor.getStorageTypes()) {
+ childrenStorageInfo.get(dnDescriptor.getName()).put(st, 1);
+ }
+ }
+ return true;
+ } else {
+ // find the next ancestor node
+ String parentName = getNextAncestorName(n);
+ InnerNode parentNode = (InnerNode)childrenMap.get(parentName);
+ if (parentNode == null) {
+ // create a new InnerNode
+ parentNode = createParentNode(parentName);
+ children.add(parentNode);
+ childrenMap.put(parentNode.getName(), parentNode);
+ }
+ // add n to the subtree of the next ancestor node
+ if (parentNode.add(n)) {
+ numOfLeaves++;
+ synchronized (childrenStorageInfo) {
+ if (!childrenStorageInfo.containsKey(parentNode.getName())) {
+ childrenStorageInfo.put(
+ parentNode.getName(), new EnumMap<>(StorageType.class));
+ for (StorageType st : dnDescriptor.getStorageTypes()) {
+ childrenStorageInfo.get(parentNode.getName()).put(st, 1);
+ }
+ } else {
+ EnumMap<StorageType, Integer> currentCount =
+ childrenStorageInfo.get(parentNode.getName());
+ for (StorageType st : dnDescriptor.getStorageTypes()) {
+ if (currentCount.containsKey(st)) {
+ currentCount.put(st, currentCount.get(st) + 1);
+ } else {
+ currentCount.put(st, 1);
+ }
+ }
+ }
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ @VisibleForTesting
+ HashMap <String, EnumMap<StorageType, Integer>> getChildrenStorageInfo() {
+ return childrenStorageInfo;
+ }
+
+
+ private DFSTopologyNodeImpl createParentNode(String parentName) {
+ return new DFSTopologyNodeImpl(
+ parentName, getPath(this), this, this.getLevel() + 1);
+ }
+
+ @Override
+ public boolean remove(Node n) {
+ if (!isAncestor(n)) {
+ throw new IllegalArgumentException(n.getName()
+ + ", which is located at " + n.getNetworkLocation()
+ + ", is not a descendant of " + getPath(this));
+ }
+ // In HDFS topology, the leaf node should always be DatanodeDescriptor
+ if (!(n instanceof DatanodeDescriptor)) {
+ throw new IllegalArgumentException("Unexpected node type "
+ + n.getClass().getName());
+ }
+ DatanodeDescriptor dnDescriptor = (DatanodeDescriptor) n;
+ if (isParent(n)) {
+ // this node is the parent of n; remove n directly
+ if (childrenMap.containsKey(n.getName())) {
+ for (int i=0; i<children.size(); i++) {
+ if (children.get(i).getName().equals(n.getName())) {
+ children.remove(i);
+ childrenMap.remove(n.getName());
+ synchronized (childrenStorageInfo) {
+ childrenStorageInfo.remove(dnDescriptor.getName());
+ }
+ numOfLeaves--;
+ n.setParent(null);
+ return true;
+ }
+ }
+ }
+ return false;
+ } else {
+ // find the next ancestor node: the parent node
+ String parentName = getNextAncestorName(n);
+ DFSTopologyNodeImpl parentNode =
+ (DFSTopologyNodeImpl)childrenMap.get(parentName);
+ if (parentNode == null) {
+ return false;
+ }
+ // remove n from the parent node
+ boolean isRemoved = parentNode.remove(n);
+ if (isRemoved) {
+ // if the parent node has no children, remove the parent node too
+ synchronized (childrenStorageInfo) {
+ EnumMap<StorageType, Integer> currentCount =
+ childrenStorageInfo.get(parentNode.getName());
+ EnumSet<StorageType> toRemove = EnumSet.noneOf(StorageType.class);
+ for (StorageType st : dnDescriptor.getStorageTypes()) {
+ int newCount = currentCount.get(st) - 1;
+ if (newCount == 0) {
+ toRemove.add(st);
+ }
+ currentCount.put(st, newCount);
+ }
+ for (StorageType st : toRemove) {
+ currentCount.remove(st);
+ }
+ }
+ if (parentNode.getNumOfChildren() == 0) {
+ for(int i=0; i < children.size(); i++) {
+ if (children.get(i).getName().equals(parentName)) {
+ children.remove(i);
+ childrenMap.remove(parentName);
+ childrenStorageInfo.remove(parentNode.getName());
+ break;
+ }
+ }
+ }
+ numOfLeaves--;
+ }
+ return isRemoved;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 6d55c46..a3c0aa4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -1268,7 +1268,7 @@ public class TestBlockStoragePolicy {
}
FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
- conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
File baseDir = PathUtils.getTestDir(TestReplicationPolicy.class);
conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
new File(baseDir, "name").getPath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java
new file mode 100644
index 0000000..ac1edf9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.util.EnumMap;
+import java.util.HashMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This class tests the correctness of storage type info stored in
+ * DFSNetworkTopology.
+ */
+public class TestDFSNetworkTopology {
+ private static final Log LOG =
+ LogFactory.getLog(TestDFSNetworkTopology.class);
+ private final static DFSNetworkTopology CLUSTER =
+ DFSNetworkTopology.getInstance(new Configuration());
+ private DatanodeDescriptor[] dataNodes;
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30000);
+
+ @Before
+ public void setupDatanodes() {
+ final String[] racks = {
+ "/l1/d1/r1", "/l1/d1/r1", "/l1/d1/r2", "/l1/d1/r2", "/l1/d1/r2",
+
+ "/l1/d2/r3", "/l1/d2/r3", "/l1/d2/r3",
+
+ "/l2/d3/r1", "/l2/d3/r2", "/l2/d3/r3", "/l2/d3/r4", "/l2/d3/r5",
+
+ "/l2/d4/r1", "/l2/d4/r1", "/l2/d4/r1", "/l2/d4/r1", "/l2/d4/r1",
+ "/l2/d4/r1", "/l2/d4/r1"};
+ final String[] hosts = {
+ "host1", "host2", "host3", "host4", "host5",
+ "host6", "host7", "host8", "host9", "host10",
+ "host11", "host12", "host13", "host14", "host15",
+ "host16", "host17", "host18", "host19", "host20"};
+ final StorageType[] types = {
+ StorageType.ARCHIVE, StorageType.DISK, StorageType.ARCHIVE,
+ StorageType.DISK, StorageType.DISK,
+
+ StorageType.DISK, StorageType.RAM_DISK, StorageType.SSD,
+
+ StorageType.DISK, StorageType.RAM_DISK, StorageType.DISK,
+ StorageType.ARCHIVE, StorageType.ARCHIVE,
+
+ StorageType.DISK, StorageType.DISK, StorageType.RAM_DISK,
+ StorageType.RAM_DISK, StorageType.ARCHIVE, StorageType.ARCHIVE,
+ StorageType.SSD};
+ final DatanodeStorageInfo[] storages =
+ DFSTestUtil.createDatanodeStorageInfos(20, racks, hosts, types);
+ dataNodes = DFSTestUtil.toDatanodeDescriptor(storages);
+ for (int i = 0; i < dataNodes.length; i++) {
+ CLUSTER.add(dataNodes[i]);
+ }
+ dataNodes[9].setDecommissioned();
+ dataNodes[10].setDecommissioned();
+ }
+
+ /**
+ * Test getting the storage type info of subtree.
+ * @throws Exception
+ */
+ @Test
+ public void testGetStorageTypeInfo() throws Exception {
+ // checking level = 2 nodes
+ DFSTopologyNodeImpl d1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1/d1");
+ HashMap<String, EnumMap<StorageType, Integer>> d1info =
+ d1.getChildrenStorageInfo();
+ assertEquals(2, d1info.keySet().size());
+ assertTrue(d1info.get("r1").size() == 2 && d1info.get("r2").size() == 2);
+ assertEquals(1, (int)d1info.get("r1").get(StorageType.DISK));
+ assertEquals(1, (int)d1info.get("r1").get(StorageType.ARCHIVE));
+ assertEquals(2, (int)d1info.get("r2").get(StorageType.DISK));
+ assertEquals(1, (int)d1info.get("r2").get(StorageType.ARCHIVE));
+
+ DFSTopologyNodeImpl d2 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1/d2");
+ HashMap<String, EnumMap<StorageType, Integer>> d2info =
+ d2.getChildrenStorageInfo();
+ assertEquals(1, d2info.keySet().size());
+ assertTrue(d2info.get("r3").size() == 3);
+ assertEquals(1, (int)d2info.get("r3").get(StorageType.DISK));
+ assertEquals(1, (int)d2info.get("r3").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)d2info.get("r3").get(StorageType.SSD));
+
+ DFSTopologyNodeImpl d3 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l2/d3");
+ HashMap<String, EnumMap<StorageType, Integer>> d3info =
+ d3.getChildrenStorageInfo();
+ assertEquals(5, d3info.keySet().size());
+ assertEquals(1, (int)d3info.get("r1").get(StorageType.DISK));
+ assertEquals(1, (int)d3info.get("r2").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)d3info.get("r3").get(StorageType.DISK));
+ assertEquals(1, (int)d3info.get("r4").get(StorageType.ARCHIVE));
+ assertEquals(1, (int)d3info.get("r5").get(StorageType.ARCHIVE));
+
+ DFSTopologyNodeImpl d4 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l2/d4");
+ HashMap<String, EnumMap<StorageType, Integer>> d4info =
+ d4.getChildrenStorageInfo();
+ assertEquals(1, d4info.keySet().size());
+ assertEquals(2, (int)d4info.get("r1").get(StorageType.DISK));
+ assertEquals(2, (int)d4info.get("r1").get(StorageType.RAM_DISK));
+ assertEquals(2, (int)d4info.get("r1").get(StorageType.ARCHIVE));
+ assertEquals(1, (int)d4info.get("r1").get(StorageType.SSD));
+
+ DFSTopologyNodeImpl l1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1");
+ HashMap<String, EnumMap<StorageType, Integer>> l1info =
+ l1.getChildrenStorageInfo();
+ assertEquals(2, l1info.keySet().size());
+ assertTrue(l1info.get("d1").size() == 2
+ && l1info.get("d2").size() == 3);
+ assertEquals(2, (int)l1info.get("d1").get(StorageType.ARCHIVE));
+ assertEquals(3, (int)l1info.get("d1").get(StorageType.DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.SSD));
+
+ // checking level = 1 nodes
+ DFSTopologyNodeImpl l2 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l2");
+ HashMap<String, EnumMap<StorageType, Integer>> l2info =
+ l2.getChildrenStorageInfo();
+ assertTrue(l2info.get("d3").size() == 3
+ && l2info.get("d4").size() == 4);
+ assertEquals(2, l2info.keySet().size());
+ assertEquals(2, (int)l2info.get("d3").get(StorageType.DISK));
+ assertEquals(2, (int)l2info.get("d3").get(StorageType.ARCHIVE));
+ assertEquals(1, (int)l2info.get("d3").get(StorageType.RAM_DISK));
+ assertEquals(2, (int)l2info.get("d4").get(StorageType.DISK));
+ assertEquals(2, (int)l2info.get("d4").get(StorageType.ARCHIVE));
+ assertEquals(2, (int)l2info.get("d4").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)l2info.get("d4").get(StorageType.SSD));
+ }
+
+ /**
+ * Test the correctness of storage type info when nodes are added and removed.
+ * @throws Exception
+ */
+ @Test
+ public void testAddAndRemoveTopology() throws Exception {
+ String[] newRack = {"/l1/d1/r1", "/l1/d1/r3", "/l1/d3/r3", "/l1/d3/r3"};
+ String[] newHost = {"nhost1", "nhost2", "nhost3", "nhost4"};
+ String[] newips = {"30.30.30.30", "31.31.31.31", "32.32.32.32",
+ "33.33.33.33"};
+ StorageType[] newTypes = {StorageType.DISK, StorageType.SSD,
+ StorageType.SSD, StorageType.SSD};
+ DatanodeDescriptor[] newDD = new DatanodeDescriptor[4];
+
+ for (int i = 0; i<4; i++) {
+ DatanodeStorageInfo dsi = DFSTestUtil.createDatanodeStorageInfo(
+ "s" + newHost[i], newips[i], newRack[i], newHost[i],
+ newTypes[i], null);
+ newDD[i] = dsi.getDatanodeDescriptor();
+ CLUSTER.add(newDD[i]);
+ }
+
+ DFSTopologyNodeImpl d1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1/d1");
+ HashMap<String, EnumMap<StorageType, Integer>> d1info =
+ d1.getChildrenStorageInfo();
+ assertEquals(3, d1info.keySet().size());
+ assertTrue(d1info.get("r1").size() == 2 && d1info.get("r2").size() == 2
+ && d1info.get("r3").size() == 1);
+ assertEquals(2, (int)d1info.get("r1").get(StorageType.DISK));
+ assertEquals(1, (int)d1info.get("r1").get(StorageType.ARCHIVE));
+ assertEquals(2, (int)d1info.get("r2").get(StorageType.DISK));
+ assertEquals(1, (int)d1info.get("r2").get(StorageType.ARCHIVE));
+ assertEquals(1, (int)d1info.get("r3").get(StorageType.SSD));
+
+ DFSTopologyNodeImpl d3 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1/d3");
+ HashMap<String, EnumMap<StorageType, Integer>> d3info =
+ d3.getChildrenStorageInfo();
+ assertEquals(1, d3info.keySet().size());
+ assertTrue(d3info.get("r3").size() == 1);
+ assertEquals(2, (int)d3info.get("r3").get(StorageType.SSD));
+
+ DFSTopologyNodeImpl l1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1");
+ HashMap<String, EnumMap<StorageType, Integer>> l1info =
+ l1.getChildrenStorageInfo();
+ assertEquals(3, l1info.keySet().size());
+ assertTrue(l1info.get("d1").size() == 3 &&
+ l1info.get("d2").size() == 3 && l1info.get("d3").size() == 1);
+ assertEquals(4, (int)l1info.get("d1").get(StorageType.DISK));
+ assertEquals(2, (int)l1info.get("d1").get(StorageType.ARCHIVE));
+ assertEquals(1, (int)l1info.get("d1").get(StorageType.SSD));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.SSD));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.DISK));
+ assertEquals(2, (int)l1info.get("d3").get(StorageType.SSD));
+
+
+ for (int i = 0; i<4; i++) {
+ CLUSTER.remove(newDD[i]);
+ }
+
+ // /d1/r3 should've been out, /d1/r1 should've been resumed
+ DFSTopologyNodeImpl nd1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1/d1");
+ HashMap<String, EnumMap<StorageType, Integer>> nd1info =
+ nd1.getChildrenStorageInfo();
+ assertEquals(2, nd1info.keySet().size());
+ assertTrue(nd1info.get("r1").size() == 2 && nd1info.get("r2").size() == 2);
+ assertEquals(1, (int)nd1info.get("r1").get(StorageType.DISK));
+ assertEquals(1, (int)nd1info.get("r1").get(StorageType.ARCHIVE));
+ assertEquals(2, (int)nd1info.get("r2").get(StorageType.DISK));
+ assertEquals(1, (int)nd1info.get("r2").get(StorageType.ARCHIVE));
+
+ // /l1/d3 should've been out, and /l1/d1 should've been resumed
+ DFSTopologyNodeImpl nl1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1");
+ HashMap<String, EnumMap<StorageType, Integer>> nl1info =
+ nl1.getChildrenStorageInfo();
+ assertEquals(2, nl1info.keySet().size());
+ assertTrue(l1info.get("d1").size() == 2
+ && l1info.get("d2").size() == 3);
+ assertEquals(2, (int)nl1info.get("d1").get(StorageType.ARCHIVE));
+ assertEquals(3, (int)nl1info.get("d1").get(StorageType.DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.SSD));
+
+ assertNull(CLUSTER.getNode("/l1/d3"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
index 62bd262..923a73f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
@@ -47,7 +47,8 @@ import org.junit.rules.Timeout;
public class TestNetworkTopology {
private static final Log LOG = LogFactory.getLog(TestNetworkTopology.class);
- private final static NetworkTopology cluster = new NetworkTopology();
+ private final static NetworkTopology cluster =
+ NetworkTopology.getInstance(new Configuration());
private DatanodeDescriptor dataNodes[];
@Rule
@@ -101,7 +102,8 @@ public class TestNetworkTopology {
@Test
public void testCreateInvalidTopology() throws Exception {
- NetworkTopology invalCluster = new NetworkTopology();
+ NetworkTopology invalCluster =
+ NetworkTopology.getInstance(new Configuration());
DatanodeDescriptor invalDataNodes[] = new DatanodeDescriptor[] {
DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1"),
DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1"),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-10506. OIV's ReverseXML
processor cannot reconstruct some snapshot details. Contributed by Akira
Ajisaka.
Posted by jh...@apache.org.
HDFS-10506. OIV's ReverseXML processor cannot reconstruct some snapshot details. Contributed by Akira Ajisaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05391c18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05391c18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05391c18
Branch: refs/heads/YARN-5734
Commit: 05391c1845639d4f01da8e5df966e2dc2682f2ca
Parents: dab00da
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Sat Feb 25 14:38:23 2017 -0800
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Sat Feb 25 14:38:50 2017 -0800
----------------------------------------------------------------------
.../OfflineImageReconstructor.java | 85 ++++++++++++++------
.../offlineImageViewer/PBImageXmlWriter.java | 30 ++++++-
.../TestOfflineImageViewer.java | 15 +++-
3 files changed, 101 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05391c18/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
index a2fa315..ed348d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
@@ -567,6 +567,13 @@ class OfflineImageReconstructor {
private void processFileXml(Node node, INodeSection.INode.Builder inodeBld)
throws IOException {
inodeBld.setType(INodeSection.INode.Type.FILE);
+ INodeSection.INodeFile.Builder bld = createINodeFileBuilder(node);
+ inodeBld.setFile(bld);
+ // Will check remaining keys and serialize in processINodeXml
+ }
+
+ private INodeSection.INodeFile.Builder createINodeFileBuilder(Node node)
+ throws IOException {
INodeSection.INodeFile.Builder bld = INodeSection.INodeFile.newBuilder();
Integer ival = node.removeChildInt(SECTION_REPLICATION);
if (ival != null) {
@@ -595,24 +602,7 @@ class OfflineImageReconstructor {
if (block == null) {
break;
}
- HdfsProtos.BlockProto.Builder blockBld =
- HdfsProtos.BlockProto.newBuilder();
- Long id = block.removeChildLong(SECTION_ID);
- if (id == null) {
- throw new IOException("<block> found without <id>");
- }
- blockBld.setBlockId(id);
- Long genstamp = block.removeChildLong(INODE_SECTION_GEMSTAMP);
- if (genstamp == null) {
- throw new IOException("<block> found without <genstamp>");
- }
- blockBld.setGenStamp(genstamp);
- Long numBytes = block.removeChildLong(INODE_SECTION_NUM_BYTES);
- if (numBytes == null) {
- throw new IOException("<block> found without <numBytes>");
- }
- blockBld.setNumBytes(numBytes);
- bld.addBlocks(blockBld);
+ bld.addBlocks(createBlockBuilder(block));
}
}
Node fileUnderConstruction =
@@ -663,14 +653,43 @@ class OfflineImageReconstructor {
blockType);
}
}
- inodeBld.setFile(bld);
- // Will check remaining keys and serialize in processINodeXml
+ return bld;
+ }
+
+ private HdfsProtos.BlockProto.Builder createBlockBuilder(Node block)
+ throws IOException {
+ HdfsProtos.BlockProto.Builder blockBld =
+ HdfsProtos.BlockProto.newBuilder();
+ Long id = block.removeChildLong(SECTION_ID);
+ if (id == null) {
+ throw new IOException("<block> found without <id>");
+ }
+ blockBld.setBlockId(id);
+ Long genstamp = block.removeChildLong(INODE_SECTION_GEMSTAMP);
+ if (genstamp == null) {
+ throw new IOException("<block> found without <genstamp>");
+ }
+ blockBld.setGenStamp(genstamp);
+ Long numBytes = block.removeChildLong(INODE_SECTION_NUM_BYTES);
+ if (numBytes == null) {
+ throw new IOException("<block> found without <numBytes>");
+ }
+ blockBld.setNumBytes(numBytes);
+ return blockBld;
}
private void processDirectoryXml(Node node,
INodeSection.INode.Builder inodeBld) throws IOException {
inodeBld.setType(INodeSection.INode.Type.DIRECTORY);
INodeSection.INodeDirectory.Builder bld =
+ createINodeDirectoryBuilder(node);
+ inodeBld.setDirectory(bld);
+ // Will check remaining keys and serialize in processINodeXml
+ }
+
+ private INodeSection.INodeDirectory.Builder
+ createINodeDirectoryBuilder(Node node) throws IOException {
+ INodeSection.INodeDirectory.Builder bld =
INodeSection.INodeDirectory.newBuilder();
Long lval = node.removeChildLong(INODE_SECTION_MTIME);
if (lval != null) {
@@ -723,8 +742,7 @@ class OfflineImageReconstructor {
qf.addQuotas(qbld);
}
bld.setTypeQuotas(qf);
- inodeBld.setDirectory(bld);
- // Will check remaining keys and serialize in processINodeXml
+ return bld;
}
private void processSymlinkXml(Node node,
@@ -1368,7 +1386,11 @@ class OfflineImageReconstructor {
if (name != null) {
bld.setName(ByteString.copyFrom(name, "UTF8"));
}
- // TODO: add missing snapshotCopy field to XML
+ Node snapshotCopy = dirDiff.removeChild(
+ SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY);
+ if (snapshotCopy != null) {
+ bld.setSnapshotCopy(createINodeDirectoryBuilder(snapshotCopy));
+ }
Integer expectedCreatedListSize = dirDiff.removeChildInt(
SNAPSHOT_DIFF_SECTION_CREATED_LIST_SIZE);
if (expectedCreatedListSize == null) {
@@ -1467,8 +1489,21 @@ class OfflineImageReconstructor {
if (name != null) {
bld.setName(ByteString.copyFrom(name, "UTF8"));
}
- // TODO: missing snapshotCopy
- // TODO: missing blocks
+ Node snapshotCopy = fileDiff.removeChild(
+ SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY);
+ if (snapshotCopy != null) {
+ bld.setSnapshotCopy(createINodeFileBuilder(snapshotCopy));
+ }
+ Node blocks = fileDiff.removeChild(INODE_SECTION_BLOCKS);
+ if (blocks != null) {
+ while (true) {
+ Node block = blocks.removeChild(INODE_SECTION_BLOCK);
+ if (block == null) {
+ break;
+ }
+ bld.addBlocks(createBlockBuilder(block));
+ }
+ }
fileDiff.verifyNoRemainingKeys("fileDiff");
bld.build().writeDelimitedTo(out);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05391c18/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
index 8df55bd..7f0bf38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
@@ -190,6 +190,8 @@ public final class PBImageXmlWriter {
"childrenSize";
public static final String SNAPSHOT_DIFF_SECTION_IS_SNAPSHOT_ROOT =
"isSnapshotRoot";
+ public static final String SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY =
+ "snapshotCopy";
public static final String SNAPSHOT_DIFF_SECTION_CREATED_LIST_SIZE =
"createdListSize";
public static final String SNAPSHOT_DIFF_SECTION_DELETED_INODE =
@@ -667,6 +669,23 @@ public final class PBImageXmlWriter {
o(SNAPSHOT_DIFF_SECTION_SNAPSHOT_ID, f.getSnapshotId())
.o(SNAPSHOT_DIFF_SECTION_SIZE, f.getFileSize())
.o(SECTION_NAME, f.getName().toStringUtf8());
+ INodeSection.INodeFile snapshotCopy = f.getSnapshotCopy();
+ if (snapshotCopy != null) {
+ out.print("<" + SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY + ">");
+ dumpINodeFile(snapshotCopy);
+ out.print("</" + SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY + ">\n");
+ }
+ if (f.getBlocksCount() > 0) {
+ out.print("<" + INODE_SECTION_BLOCKS + ">");
+ for (BlockProto b : f.getBlocksList()) {
+ out.print("<" + INODE_SECTION_BLOCK + ">");
+ o(SECTION_ID, b.getBlockId())
+ .o(INODE_SECTION_GEMSTAMP, b.getGenStamp())
+ .o(INODE_SECTION_NUM_BYTES, b.getNumBytes());
+ out.print("</" + INODE_SECTION_BLOCK + ">\n");
+ }
+ out.print("</" + INODE_SECTION_BLOCKS + ">\n");
+ }
out.print("</" + SNAPSHOT_DIFF_SECTION_FILE_DIFF + ">\n");
}
}
@@ -679,9 +698,14 @@ public final class PBImageXmlWriter {
o(SNAPSHOT_DIFF_SECTION_SNAPSHOT_ID, d.getSnapshotId())
.o(SNAPSHOT_DIFF_SECTION_CHILDREN_SIZE, d.getChildrenSize())
.o(SNAPSHOT_DIFF_SECTION_IS_SNAPSHOT_ROOT, d.getIsSnapshotRoot())
- .o(SECTION_NAME, d.getName().toStringUtf8())
- .o(SNAPSHOT_DIFF_SECTION_CREATED_LIST_SIZE,
- d.getCreatedListSize());
+ .o(SECTION_NAME, d.getName().toStringUtf8());
+ INodeDirectory snapshotCopy = d.getSnapshotCopy();
+ if (snapshotCopy != null) {
+ out.print("<" + SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY + ">");
+ dumpINodeDirectory(snapshotCopy);
+ out.print("</" + SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY + ">\n");
+ }
+ o(SNAPSHOT_DIFF_SECTION_CREATED_LIST_SIZE, d.getCreatedListSize());
for (long did : d.getDeletedINodeList()) {
o(SNAPSHOT_DIFF_SECTION_DELETED_INODE, did);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05391c18/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index dacbb85..f77911d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -181,14 +181,27 @@ public class TestOfflineImageViewer {
hdfs.mkdirs(src);
dirCount++;
writtenFiles.put(src.toString(), hdfs.getFileStatus(src));
+
+ // Create snapshot and snapshotDiff.
final Path orig = new Path("/src/orig");
hdfs.mkdirs(orig);
+ final Path file1 = new Path("/src/file");
+ FSDataOutputStream o = hdfs.create(file1);
+ o.write(23);
+ o.write(45);
+ o.close();
hdfs.allowSnapshot(src);
hdfs.createSnapshot(src, "snapshot");
final Path dst = new Path("/dst");
+ // Rename a directory in the snapshot directory to add snapshotCopy
+ // field to the dirDiff entry.
hdfs.rename(orig, dst);
dirCount++;
writtenFiles.put(dst.toString(), hdfs.getFileStatus(dst));
+ // Truncate a file in the snapshot directory to add snapshotCopy and
+ // blocks fields to the fileDiff entry.
+ hdfs.truncate(file1, 1);
+ writtenFiles.put(file1.toString(), hdfs.getFileStatus(file1));
// Set XAttrs so the fsimage contains XAttr ops
final Path xattr = new Path("/xattr");
@@ -279,7 +292,7 @@ public class TestOfflineImageViewer {
Matcher matcher = p.matcher(outputString);
assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalFiles = Integer.parseInt(matcher.group(1));
- assertEquals(NUM_DIRS * FILES_PER_DIR, totalFiles);
+ assertEquals(NUM_DIRS * FILES_PER_DIR + 1, totalFiles);
p = Pattern.compile("totalDirectories = (\\d+)\n");
matcher = p.matcher(outputString);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org