You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/07/21 22:11:55 UTC
[01/16] hadoop git commit: HDFS-8344. NameNode doesn't recover lease
for files with missing blocks (raviprak)
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-12111 b41fe3111 -> a3a4578cc
HDFS-8344. NameNode doesn't recover lease for files with missing blocks (raviprak)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4f75626
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4f75626
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4f75626
Branch: refs/heads/HADOOP-12111
Commit: e4f756260f16156179ba4adad974ec92279c2fac
Parents: 98c2bc8
Author: Ravi Prakash <ra...@altiscale.com>
Authored: Mon Jul 20 14:03:34 2015 -0700
Committer: Ravi Prakash <ra...@altiscale.com>
Committed: Mon Jul 20 14:03:34 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +
.../BlockInfoUnderConstruction.java | 19 ++++-
.../server/blockmanagement/BlockManager.java | 14 +++-
.../hdfs/server/namenode/FSNamesystem.java | 10 +++
.../src/main/resources/hdfs-default.xml | 9 +++
.../apache/hadoop/hdfs/TestLeaseRecovery.java | 78 ++++++++++++++++++++
7 files changed, 132 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f75626/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 58491a6..13d9969 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1050,6 +1050,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8778. TestBlockReportRateLimiting#testLeaseExpiration can deadlock.
(Arpit Agarwal)
+ HDFS-8344. NameNode doesn't recover lease for files with missing blocks
+ (raviprak)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f75626/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 0e569f0..210d1e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -440,6 +440,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT = 10 * 1000;
public static final String DFS_BLOCK_INVALIDATE_LIMIT_KEY = "dfs.block.invalidate.limit";
public static final int DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT = 1000;
+ public static final String DFS_BLOCK_UC_MAX_RECOVERY_ATTEMPTS = "dfs.block.uc.max.recovery.attempts";
+ public static final int DFS_BLOCK_UC_MAX_RECOVERY_ATTEMPTS_DEFAULT = 5;
+
public static final String DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY = "dfs.corruptfilesreturned.max";
public static final int DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED = 500;
/* Maximum number of blocks to process for initializing replication queues */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f75626/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
index 9cd3987..28f1633 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -61,6 +60,11 @@ public abstract class BlockInfoUnderConstruction extends BlockInfo {
*/
protected Block truncateBlock;
+ /** The number of times all replicas will be used to attempt recovery before
+ * giving up and marking the block under construction missing.
+ */
+ private int recoveryAttemptsBeforeMarkingBlockMissing;
+
/**
* ReplicaUnderConstruction contains information about replicas while
* they are under construction.
@@ -174,6 +178,8 @@ public abstract class BlockInfoUnderConstruction extends BlockInfo {
"BlockInfoUnderConstruction cannot be in COMPLETE state");
this.blockUCState = state;
setExpectedLocations(targets);
+ this.recoveryAttemptsBeforeMarkingBlockMissing =
+ BlockManager.getMaxBlockUCRecoveries();
}
/** Set expected locations. */
@@ -271,7 +277,7 @@ public abstract class BlockInfoUnderConstruction extends BlockInfo {
if (replicas.size() == 0) {
NameNode.blockStateChangeLog.warn("BLOCK* " +
"BlockInfoUnderConstruction.initLeaseRecovery: " +
- "No blocks found, lease removed.");
+ "No replicas found.");
}
boolean allLiveReplicasTriedAsPrimary = true;
for (int i = 0; i < replicas.size(); i++) {
@@ -283,6 +289,11 @@ public abstract class BlockInfoUnderConstruction extends BlockInfo {
}
}
if (allLiveReplicasTriedAsPrimary) {
+ recoveryAttemptsBeforeMarkingBlockMissing--;
+ NameNode.blockStateChangeLog.info("Tried to recover " + this +" using all"
+ + " replicas. Will try " + recoveryAttemptsBeforeMarkingBlockMissing
+ + " more times");
+
// Just set all the replicas to be chosen whether they are alive or not.
for (int i = 0; i < replicas.size(); i++) {
replicas.get(i).setChosenAsPrimary(false);
@@ -341,6 +352,10 @@ public abstract class BlockInfoUnderConstruction extends BlockInfo {
replicas.add(new ReplicaUnderConstruction(block, storage, rState));
}
+ public int getNumRecoveryAttemptsLeft() {
+ return recoveryAttemptsBeforeMarkingBlockMissing;
+ }
+
/**
* Convert an under construction block to a complete block.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f75626/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 7dce2a8..16dfe45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
import static org.apache.hadoop.util.ExitUtil.terminate;
import java.io.IOException;
@@ -274,6 +273,13 @@ public class BlockManager implements BlockStatsMXBean {
private BlockPlacementPolicy blockplacement;
private final BlockStoragePolicySuite storagePolicySuite;
+ /** The number of times a block under construction's recovery will be
+ * attempted using all known replicas. e.g. if there are 3 replicas, each
+ * node will be tried 5 times (for a total of 15 retries across all nodes)*/
+ private static int maxBlockUCRecoveries =
+ DFSConfigKeys.DFS_BLOCK_UC_MAX_RECOVERY_ATTEMPTS_DEFAULT;
+ public static int getMaxBlockUCRecoveries() { return maxBlockUCRecoveries; }
+
/** Check whether name system is running before terminating */
private boolean checkNSRunning = true;
@@ -282,6 +288,9 @@ public class BlockManager implements BlockStatsMXBean {
this.namesystem = namesystem;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
+ maxBlockUCRecoveries = conf.getInt(
+ DFSConfigKeys.DFS_BLOCK_UC_MAX_RECOVERY_ATTEMPTS,
+ DFSConfigKeys.DFS_BLOCK_UC_MAX_RECOVERY_ATTEMPTS_DEFAULT);
startupDelayBlockDeletionInMs = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
@@ -731,7 +740,8 @@ public class BlockManager implements BlockStatsMXBean {
/**
* Force the given block in the given file to be marked as complete,
* regardless of whether enough replicas are present. This is necessary
- * when tailing edit logs as a Standby.
+ * when tailing edit logs as a Standby or when recovering a lease on a file
+ * with missing blocks.
*/
public BlockInfo forceCompleteBlock(final BlockCollection bc,
final BlockInfoUnderConstruction block) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f75626/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 7c6d6a1..becd05a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3287,6 +3287,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
+ "Removed empty last block and closed file.");
return true;
}
+
+ //If the block's recovery has been attempted enough times, mark the block
+ //complete anyway and recover the lease
+ if(uc.getNumRecoveryAttemptsLeft() == 0) {
+ blockManager.forceCompleteBlock(pendingFile, uc);
+ finalizeINodeFileUnderConstruction(src, pendingFile,
+ iip.getLatestSnapshotId());
+ return true;
+ }
+
// start recovery of the last block for this file
long blockRecoveryId = nextGenerationStamp(blockIdManager.isLegacyBlock(uc));
lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f75626/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 8cb7d5f..dee9111 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -425,6 +425,15 @@
</property>
<property>
+ <name>dfs.block.uc.max.recovery.attempts</name>
+ <value>5</value>
+ <description>The number of times a block under construction's recovery will be
+ attempted using all known replicas. e.g. if there are 3 replicas, each node
+ will be tried 5 times (for a total of 15 retries across all nodes).
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.data.dir</name>
<value>file://${hadoop.tmp.dir}/dfs/data</value>
<description>Determines where on the local filesystem an DFS data node
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f75626/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
index c9f3842..c9448ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
@@ -27,6 +27,7 @@ import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -258,4 +259,81 @@ public class TestLeaseRecovery {
}
}
}
+
+ /**
+ * Test that when a client was writing to a file and died, and before the
+ * lease can be recovered, all the datanodes to which the file was written
+ * also die, after some time (5 * lease recovery times) the file is indeed
+ * closed and lease recovered.
+ * We also check that if the datanode came back after some time, the data
+ * originally written is not truncated
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public void testLeaseRecoveryWithMissingBlocks()
+ throws IOException, InterruptedException {
+ Configuration conf = new HdfsConfiguration();
+
+ //Start a cluster with 3 datanodes
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+ cluster.setLeasePeriod(LEASE_PERIOD, LEASE_PERIOD);
+ cluster.waitActive();
+
+ //create a file (with replication 1)
+ Path file = new Path("/testRecoveryFile");
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ FSDataOutputStream out = dfs.create(file, (short) 1);
+
+ //This keeps count of the number of bytes written (AND is also the data we
+ //are writing)
+ long writtenBytes = 0;
+ while (writtenBytes < 2 * 1024 * 1024) {
+ out.writeLong(writtenBytes);
+ writtenBytes += 8;
+ }
+ System.out.println("Written " + writtenBytes + " bytes");
+ out.hsync();
+ System.out.println("hsynced the data");
+
+ //Kill the datanode to which the file was written.
+ DatanodeInfo dn =
+ ((DFSOutputStream) out.getWrappedStream()).getPipeline()[0];
+ DataNodeProperties dnStopped = cluster.stopDataNode(dn.getName());
+
+ //Wait at most 20 seconds for the lease to be recovered
+ LeaseManager lm = NameNodeAdapter.getLeaseManager(cluster.getNamesystem());
+ int i = 40;
+ while(i-- > 0 && lm.countLease() != 0) {
+ System.out.println("Still got " + lm.countLease() + " lease(s)");
+ Thread.sleep(500);
+ }
+ assertTrue("The lease was not recovered", lm.countLease() == 0);
+ System.out.println("Got " + lm.countLease() + " leases");
+
+ //Make sure we can't read any data because the datanode is dead
+ FSDataInputStream in = dfs.open(file);
+ try {
+ in.readLong();
+ assertTrue("Shouldn't have reached here", false);
+ } catch(BlockMissingException bme) {
+ System.out.println("Correctly got BlockMissingException because datanode"
+ + " is still dead");
+ }
+
+ //Bring the dead datanode back.
+ cluster.restartDataNode(dnStopped);
+ System.out.println("Restart datanode");
+
+ //Make sure we can read all the data back (since we hsync'ed).
+ in = dfs.open(file);
+ int readBytes = 0;
+ while(in.available() != 0) {
+ assertEquals("Didn't read the data we wrote", in.readLong(), readBytes);
+ readBytes += 8;
+ }
+ assertEquals("Didn't get all the data", readBytes, writtenBytes);
+ System.out.println("Read back all the " + readBytes + " bytes");
+ }
+
}
[16/16] hadoop git commit: Merge branch 'trunk' into HADOOP-12111
Posted by aw...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3a4578c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3a4578c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3a4578c
Branch: refs/heads/HADOOP-12111
Commit: a3a4578cc98bbe1119587e88cc02bbdd6d20847f
Parents: b41fe31 5137b38
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jul 21 13:11:45 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Jul 21 13:11:45 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 8 +
.../hadoop/security/UserGroupInformation.java | 3 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 22 +-
.../hadoop-hdfs/src/main/bin/hdfs | 8 -
.../blockmanagement/CorruptReplicasMap.java | 19 +-
.../hdfs/server/namenode/AclTransformation.java | 30 +-
.../src/main/webapps/hdfs/dfshealth.html | 25 ++
.../src/main/webapps/hdfs/dfshealth.js | 8 +
.../markdown/HDFSHighAvailabilityWithNFS.md | 40 ++-
.../markdown/HDFSHighAvailabilityWithQJM.md | 32 +-
.../blockmanagement/TestBlockStatsMXBean.java | 1 -
.../blockmanagement/TestCorruptReplicaInfo.java | 12 +-
.../server/namenode/TestAclTransformation.java | 55 ++-
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../mapreduce/v2/app/job/impl/JobImpl.java | 2 +
.../org/apache/hadoop/tools/HadoopArchives.java | 87 +++--
.../apache/hadoop/tools/TestHadoopArchives.java | 4 +-
.../fs/swift/snative/SwiftNativeFileSystem.java | 9 +
.../fs/swift/TestSwiftFileSystemBasicOps.java | 7 +
.../sls/scheduler/ResourceSchedulerWrapper.java | 10 +
hadoop-yarn-project/CHANGES.txt | 7 +
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../org/apache/hadoop/yarn/client/SCMAdmin.java | 2 +-
.../server/resourcemanager/RMAppManager.java | 20 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 15 +-
.../scheduler/AbstractYarnScheduler.java | 10 +
.../server/resourcemanager/scheduler/Queue.java | 8 +
.../scheduler/SchedulerApplication.java | 22 ++
.../scheduler/SchedulerApplicationAttempt.java | 15 +-
.../scheduler/YarnScheduler.java | 20 ++
.../scheduler/capacity/AbstractCSQueue.java | 7 +
.../scheduler/capacity/CapacityScheduler.java | 73 +++-
.../CapacitySchedulerConfiguration.java | 13 +
.../scheduler/capacity/LeafQueue.java | 19 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 8 +
.../scheduler/event/AppAddedSchedulerEvent.java | 28 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 6 +
.../scheduler/fifo/FifoScheduler.java | 6 +
.../scheduler/policy/FifoComparator.java | 11 +-
.../scheduler/policy/SchedulableEntity.java | 5 +
.../yarn/server/resourcemanager/MockRM.java | 31 +-
.../server/resourcemanager/TestAppManager.java | 1 +
.../TestWorkPreservingRMRestart.java | 2 +-
...pacityPreemptionPolicyForNodePartitions.java | 1 +
.../capacity/TestApplicationLimits.java | 5 +-
.../capacity/TestApplicationPriority.java | 345 +++++++++++++++++++
.../capacity/TestCapacityScheduler.java | 5 +
.../scheduler/policy/MockSchedulableEntity.java | 13 +-
.../security/TestDelegationTokenRenewer.java | 10 +-
.../TestRMWebServicesAppsModification.java | 2 +-
.../src/site/markdown/ResourceManagerRestart.md | 32 +-
51 files changed, 961 insertions(+), 171 deletions(-)
----------------------------------------------------------------------
[03/16] hadoop git commit: HDFS-8794. Improve
CorruptReplicasMap#corruptReplicasMap. (yliu)
Posted by aw...@apache.org.
HDFS-8794. Improve CorruptReplicasMap#corruptReplicasMap. (yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6d58606
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6d58606
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6d58606
Branch: refs/heads/HADOOP-12111
Commit: d6d58606b8adf94b208aed5fc2d054b9dd081db1
Parents: ed01dc7
Author: yliu <yl...@apache.org>
Authored: Tue Jul 21 09:20:22 2015 +0800
Committer: yliu <yl...@apache.org>
Committed: Tue Jul 21 09:20:22 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../blockmanagement/CorruptReplicasMap.java | 19 ++++++++++++++-----
.../blockmanagement/TestCorruptReplicaInfo.java | 12 ++++++------
3 files changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d58606/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cd32c0e..388b553 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -727,6 +727,8 @@ Release 2.8.0 - UNRELEASED
HDFS-7314. When the DFSClient lease cannot be renewed, abort open-for-write
files rather than the entire DFSClient. (mingma)
+ HDFS-8794. Improve CorruptReplicasMap#corruptReplicasMap. (yliu)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d58606/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
index fc2e234..f83cbaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
@@ -17,12 +17,19 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.ipc.Server;
-import java.util.*;
+import com.google.common.annotations.VisibleForTesting;
/**
* Stores information about all corrupt blocks in the File System.
@@ -46,8 +53,8 @@ public class CorruptReplicasMap{
CORRUPTION_REPORTED // client or datanode reported the corruption
}
- private final SortedMap<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap =
- new TreeMap<Block, Map<DatanodeDescriptor, Reason>>();
+ private final Map<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap =
+ new HashMap<Block, Map<DatanodeDescriptor, Reason>>();
/**
* Mark the block belonging to datanode as corrupt.
@@ -181,13 +188,15 @@ public class CorruptReplicasMap{
* @return Up to numExpectedBlocks blocks from startingBlockId if it exists
*
*/
- long[] getCorruptReplicaBlockIds(int numExpectedBlocks,
+ @VisibleForTesting
+ long[] getCorruptReplicaBlockIdsForTesting(int numExpectedBlocks,
Long startingBlockId) {
if (numExpectedBlocks < 0 || numExpectedBlocks > 100) {
return null;
}
- Iterator<Block> blockIt = corruptReplicasMap.keySet().iterator();
+ Iterator<Block> blockIt =
+ new TreeMap<>(corruptReplicasMap).keySet().iterator();
// if the starting block id was specified, iterate over keys until
// we find the matching block. If we find a matching block, break
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d58606/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
index 21fb54e..4bdaaac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
@@ -73,9 +73,9 @@ public class TestCorruptReplicaInfo {
// Make sure initial values are returned correctly
assertEquals("Number of corrupt blocks must initially be 0", 0, crm.size());
- assertNull("Param n cannot be less than 0", crm.getCorruptReplicaBlockIds(-1, null));
- assertNull("Param n cannot be greater than 100", crm.getCorruptReplicaBlockIds(101, null));
- long[] l = crm.getCorruptReplicaBlockIds(0, null);
+ assertNull("Param n cannot be less than 0", crm.getCorruptReplicaBlockIdsForTesting(-1, null));
+ assertNull("Param n cannot be greater than 100", crm.getCorruptReplicaBlockIdsForTesting(101, null));
+ long[] l = crm.getCorruptReplicaBlockIdsForTesting(0, null);
assertNotNull("n = 0 must return non-null", l);
assertEquals("n = 0 must return an empty list", 0, l.length);
@@ -118,14 +118,14 @@ public class TestCorruptReplicaInfo {
assertTrue("First five block ids not returned correctly ",
Arrays.equals(new long[]{0,1,2,3,4},
- crm.getCorruptReplicaBlockIds(5, null)));
+ crm.getCorruptReplicaBlockIdsForTesting(5, null)));
- LOG.info(crm.getCorruptReplicaBlockIds(10, 7L));
+ LOG.info(crm.getCorruptReplicaBlockIdsForTesting(10, 7L));
LOG.info(block_ids.subList(7, 18));
assertTrue("10 blocks after 7 not returned correctly ",
Arrays.equals(new long[]{8,9,10,11,12,13,14,15,16,17},
- crm.getCorruptReplicaBlockIds(10, 7L)));
+ crm.getCorruptReplicaBlockIdsForTesting(10, 7L)));
}
[10/16] hadoop git commit: YARN-3915. scmadmin help message
correction (Bibin A Chundatt via aw)
Posted by aw...@apache.org.
YARN-3915. scmadmin help message correction (Bibin A Chundatt via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/da2d1ac4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/da2d1ac4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/da2d1ac4
Branch: refs/heads/HADOOP-12111
Commit: da2d1ac4bc0bf0812b9a2a1ffbb7748113cdaf6d
Parents: c9507fe
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jul 21 09:44:45 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Jul 21 09:44:45 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../src/main/java/org/apache/hadoop/yarn/client/SCMAdmin.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da2d1ac4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e6a3343..d0829c1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -51,6 +51,8 @@ Trunk - Unreleased
YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var
for a container (Darrell Taylor via aw)
+ YARN-3915. scmadmin help message correction (Bibin A Chundatt via aw)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da2d1ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/SCMAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/SCMAdmin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/SCMAdmin.java
index 1e45c5a..dc6bf48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/SCMAdmin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/SCMAdmin.java
@@ -52,7 +52,7 @@ public class SCMAdmin extends Configured implements Tool {
String summary = "scmadmin is the command to execute shared cache manager" +
"administrative commands.\n" +
"The full syntax is: \n\n" +
- "hadoop scmadmin" +
+ "yarn scmadmin" +
" [-runCleanerTask]" +
" [-help [cmd]]\n";
[14/16] hadoop git commit: HADOOP-11762. Enable swift distcp to
secure HDFS (Chen He via aw)
Posted by aw...@apache.org.
HADOOP-11762. Enable swift distcp to secure HDFS (Chen He via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68d1f4bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68d1f4bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68d1f4bf
Branch: refs/heads/HADOOP-12111
Commit: 68d1f4bfe863761d3c83b79b17ec1194edca3172
Parents: cf74772
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jul 21 11:19:29 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Jul 21 11:19:29 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../hadoop/fs/swift/snative/SwiftNativeFileSystem.java | 9 +++++++++
.../apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java | 7 +++++++
3 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68d1f4bf/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 24709e0..5b51bce 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -495,6 +495,8 @@ Trunk (Unreleased)
HADOOP-12107. long running apps may have a huge number of StatisticsData
instances under FileSystem (Sangjin Lee via Ming Ma)
+ HADOOP-11762. Enable swift distcp to secure HDFS (Chen He via aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68d1f4bf/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
index e9faaf2..7f93c38 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
@@ -222,6 +222,15 @@ public class SwiftNativeFileSystem extends FileSystem {
}
/**
+ * Override getCononicalServiceName because we don't support token in Swift
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ // Does not support Token
+ return null;
+ }
+
+ /**
* Return an array containing hostnames, offset and size of
* portions of the given file. For a nonexistent
* file or regions, null will be returned.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68d1f4bf/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
index c7e8b57..c84be6b 100644
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.swift;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
@@ -286,4 +287,10 @@ public class TestSwiftFileSystemBasicOps extends SwiftFileSystemBaseTest {
}
}
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testGetCanonicalServiceName() {
+ Assert.assertNull(fs.getCanonicalServiceName());
+ }
+
+
}
[07/16] hadoop git commit: HADOOP-12214. Parse 'HadoopArchive'
commandline using cli Options. (Contributed by Vinayakumar B)
Posted by aw...@apache.org.
HADOOP-12214. Parse 'HadoopArchive' commandline using cli Options. (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87f29c6b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87f29c6b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87f29c6b
Branch: refs/heads/HADOOP-12111
Commit: 87f29c6b8acc07cc011713a79554d51945e265ac
Parents: df1e8ce
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jul 21 13:12:46 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Jul 21 13:12:46 2015 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/tools/HadoopArchives.java | 87 +++++++++++++-------
.../apache/hadoop/tools/TestHadoopArchives.java | 4 +-
3 files changed, 64 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f29c6b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ef8e238..24709e0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -708,6 +708,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12081. Fix UserGroupInformation.java to support 64-bit zLinux.
(aajisaka)
+ HADOOP-12214. Parse 'HadoopArchive' commandline using cli Options.
+ (vinayakumarb)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f29c6b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
index f00bb6d..330830b 100644
--- a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
+++ b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
@@ -33,6 +33,11 @@ import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.Parser;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -81,6 +86,10 @@ public class HadoopArchives implements Tool {
private static final Log LOG = LogFactory.getLog(HadoopArchives.class);
private static final String NAME = "har";
+ private static final String ARCHIVE_NAME = "archiveName";
+ private static final String REPLICATION = "r";
+ private static final String PARENT_PATH = "p";
+ private static final String HELP = "help";
static final String SRC_LIST_LABEL = NAME + ".src.list";
static final String DST_DIR_LABEL = NAME + ".dest.path";
static final String TMP_DIR_LABEL = NAME + ".tmp.dir";
@@ -101,9 +110,9 @@ public class HadoopArchives implements Tool {
/** the desired replication degree; default is 10 **/
short repl = 10;
- private static final String usage = "Usage: archive"
- + " -archiveName <NAME>.har -p <parent path> [-r <replication factor>]" +
- "<src>* <dest>" +
+ private static final String usage = "archive"
+ + " <-archiveName <NAME>.har> <-p <parent path>> [-r <replication factor>]" +
+ " <src>* <dest>" +
"\n";
@@ -794,7 +803,17 @@ public class HadoopArchives implements Tool {
}
}
-
+
+ private void printUsage(Options opts, boolean printDetailed) {
+ HelpFormatter helpFormatter = new HelpFormatter();
+ if (printDetailed) {
+ helpFormatter.printHelp(usage.length() + 10, usage, null, opts, null,
+ false);
+ } else {
+ System.out.println(usage);
+ }
+ }
+
/** the main driver for creating the archives
* it takes at least three command line parameters. The parent path,
* The src and the dest. It does an lsr on the source paths.
@@ -804,43 +823,51 @@ public class HadoopArchives implements Tool {
public int run(String[] args) throws Exception {
try {
- Path parentPath = null;
- List<Path> srcPaths = new ArrayList<Path>();
- Path destPath = null;
- String archiveName = null;
- if (args.length < 5) {
- System.out.println(usage);
- throw new IOException("Invalid usage.");
- }
- if (!"-archiveName".equals(args[0])) {
- System.out.println(usage);
+ // Parse CLI options
+ Options options = new Options();
+ options.addOption(ARCHIVE_NAME, true,
+ "Name of the Archive. This is mandatory option");
+ options.addOption(PARENT_PATH, true,
+ "Parent path of sources. This is mandatory option");
+ options.addOption(REPLICATION, true, "Replication factor archive files");
+ options.addOption(HELP, false, "Show the usage");
+ Parser parser = new GnuParser();
+ CommandLine commandLine = parser.parse(options, args, true);
+
+ if (commandLine.hasOption(HELP)) {
+ printUsage(options, true);
+ return 0;
+ }
+ if (!commandLine.hasOption(ARCHIVE_NAME)) {
+ printUsage(options, false);
throw new IOException("Archive Name not specified.");
}
- archiveName = args[1];
+ String archiveName = commandLine.getOptionValue(ARCHIVE_NAME);
if (!checkValidName(archiveName)) {
- System.out.println(usage);
+ printUsage(options, false);
throw new IOException("Invalid name for archives. " + archiveName);
}
- int i = 2;
//check to see if relative parent has been provided or not
//this is a required parameter.
- if (! "-p".equals(args[i])) {
- System.out.println(usage);
+ if (!commandLine.hasOption(PARENT_PATH)) {
+ printUsage(options, false);
throw new IOException("Parent path not specified.");
}
- parentPath = new Path(args[i+1]);
+ Path parentPath = new Path(commandLine.getOptionValue(PARENT_PATH));
if (!parentPath.isAbsolute()) {
- parentPath= parentPath.getFileSystem(getConf()).makeQualified(parentPath);
+ parentPath = parentPath.getFileSystem(getConf()).makeQualified(
+ parentPath);
}
- i+=2;
-
- if ("-r".equals(args[i])) {
- repl = Short.parseShort(args[i+1]);
- i+=2;
+ if (commandLine.hasOption(REPLICATION)) {
+ repl = Short.parseShort(commandLine.getOptionValue(REPLICATION));
}
+ // Remaining args
+ args = commandLine.getArgs();
+ List<Path> srcPaths = new ArrayList<Path>();
+ Path destPath = null;
//read the rest of the paths
- for (; i < args.length; i++) {
+ for (int i = 0; i < args.length; i++) {
if (i == (args.length - 1)) {
destPath = new Path(args[i]);
if (!destPath.isAbsolute()) {
@@ -850,13 +877,17 @@ public class HadoopArchives implements Tool {
else {
Path argPath = new Path(args[i]);
if (argPath.isAbsolute()) {
- System.out.println(usage);
+ printUsage(options, false);
throw new IOException("Source path " + argPath +
" is not relative to "+ parentPath);
}
srcPaths.add(new Path(parentPath, argPath));
}
}
+ if (destPath == null) {
+ printUsage(options, false);
+ throw new IOException("Destination path not specified.");
+ }
if (srcPaths.size() == 0) {
// assuming if the user does not specify path for sources
// the whole parent directory needs to be archived.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f29c6b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
index 101cb06..d8222dc 100644
--- a/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
+++ b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
@@ -753,8 +753,8 @@ public class TestHadoopArchives {
final String harName = "foo.har";
final String fullHarPathStr = prefix + harName;
- final String[] args = { "-archiveName", harName, "-p", inputPathStr,
- "-r 3", "*", archivePath.toString() };
+ final String[] args = { "-archiveName", harName, "-p", inputPathStr, "-r",
+ "3", "*", archivePath.toString() };
System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH,
HADOOP_ARCHIVES_JAR);
final HadoopArchives har = new HadoopArchives(conf);
[09/16] hadoop git commit: HDFS-8800. hdfs --daemon stop namenode
corrupts logs (John Smith via aw)
Posted by aw...@apache.org.
HDFS-8800. hdfs --daemon stop namenode corrupts logs (John Smith via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9507fe6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9507fe6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9507fe6
Branch: refs/heads/HADOOP-12111
Commit: c9507fe6c12491f3aef5cd4142b4d466bd6b71c3
Parents: 29cf887b
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jul 21 09:41:28 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Jul 21 09:41:28 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 8 --------
2 files changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9507fe6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6c91c45..223baaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -343,6 +343,8 @@ Trunk (Unreleased)
HDFS-8657. Update docs for mSNN. (Jesse Yates via atm)
+ HDFS-8800. hdfs --daemon stop namenode corrupts logs (John Smith via aw)
+
Release 2.8.0 - UNRELEASED
NEW FEATURES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9507fe6/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index a996a80..23a08be 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -274,14 +274,6 @@ fi
hadoop_finalize
if [[ -n "${supportdaemonization}" ]]; then
- if [[ "${COMMAND}" == "namenode" ]] &&
- [[ "${HADOOP_DAEMON_MODE}" == "stop" ]]; then
- hadoop_debug "Do checkpoint if necessary before stopping NameNode"
- export CLASSPATH
- "${JAVA}" "-Dproc_dfsadmin" ${HADOOP_OPTS} "org.apache.hadoop.hdfs.tools.DFSAdmin" "-safemode" "enter"
- "${JAVA}" "-Dproc_dfsadmin" ${HADOOP_OPTS} "org.apache.hadoop.hdfs.tools.DFSAdmin" "-saveNamespace" "-beforeShutdown"
- "${JAVA}" "-Dproc_dfsadmin" ${HADOOP_OPTS} "org.apache.hadoop.hdfs.tools.DFSAdmin" "-safemode" "leave"
- fi
if [[ -n "${secure_service}" ]]; then
hadoop_secure_daemon_handler \
"${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\
[13/16] hadoop git commit: MAPREDUCE-5801. Uber mode's log message is
missing a vcore reason (Steven Wong via aw)
Posted by aw...@apache.org.
MAPREDUCE-5801. Uber mode's log message is missing a vcore reason (Steven Wong via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf747720
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf747720
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf747720
Branch: refs/heads/HADOOP-12111
Commit: cf74772064de0ce7cefd541e3f407442f4f0e281
Parents: 3b7ffc4
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jul 21 10:58:52 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Jul 21 10:58:52 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java | 2 ++
2 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf747720/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 31f4eaa..398ffc6 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -270,6 +270,9 @@ Trunk (Unreleased)
MAPREDUCE-6078. native-task: fix gtest build on macosx (Binglin Chang)
+ MAPREDUCE-5801. Uber mode's log message is missing a vcore reason
+ (Steven Wong via aw)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf747720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index 731bcba..4c3b3fe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -1289,6 +1289,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
msg.append(" too much CPU;");
if (!smallMemory)
msg.append(" too much RAM;");
+ if (!smallCpu)
+ msg.append(" too much CPU;");
if (!notChainJob)
msg.append(" chainjob;");
LOG.info(msg.toString());
[15/16] hadoop git commit: Revert "HDFS-8344. NameNode doesn't
recover lease for files with missing blocks (raviprak)"
Posted by aw...@apache.org.
Revert "HDFS-8344. NameNode doesn't recover lease for files with missing blocks (raviprak)"
This reverts commit e4f756260f16156179ba4adad974ec92279c2fac.
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5137b388
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5137b388
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5137b388
Branch: refs/heads/HADOOP-12111
Commit: 5137b388fc9d4d716f780daf6d04292feeb9df96
Parents: 68d1f4b
Author: Ravi Prakash <ra...@altiscale.com>
Authored: Tue Jul 21 11:29:35 2015 -0700
Committer: Ravi Prakash <ra...@altiscale.com>
Committed: Tue Jul 21 11:29:35 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 -
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 -
.../BlockInfoUnderConstruction.java | 19 +----
.../server/blockmanagement/BlockManager.java | 14 +---
.../hdfs/server/namenode/FSNamesystem.java | 10 ---
.../src/main/resources/hdfs-default.xml | 9 ---
.../apache/hadoop/hdfs/TestLeaseRecovery.java | 78 --------------------
7 files changed, 4 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5137b388/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 223baaf..a29a090 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1056,9 +1056,6 @@ Release 2.8.0 - UNRELEASED
HDFS-8778. TestBlockReportRateLimiting#testLeaseExpiration can deadlock.
(Arpit Agarwal)
- HDFS-8344. NameNode doesn't recover lease for files with missing blocks
- (raviprak)
-
HDFS-7582. Enforce maximum number of ACL entries separately per access
and default. (vinayakumarb)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5137b388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 210d1e5..0e569f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -440,9 +440,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT = 10 * 1000;
public static final String DFS_BLOCK_INVALIDATE_LIMIT_KEY = "dfs.block.invalidate.limit";
public static final int DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT = 1000;
- public static final String DFS_BLOCK_UC_MAX_RECOVERY_ATTEMPTS = "dfs.block.uc.max.recovery.attempts";
- public static final int DFS_BLOCK_UC_MAX_RECOVERY_ATTEMPTS_DEFAULT = 5;
-
public static final String DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY = "dfs.corruptfilesreturned.max";
public static final int DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED = 500;
/* Maximum number of blocks to process for initializing replication queues */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5137b388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
index 28f1633..9cd3987 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -60,11 +61,6 @@ public abstract class BlockInfoUnderConstruction extends BlockInfo {
*/
protected Block truncateBlock;
- /** The number of times all replicas will be used to attempt recovery before
- * giving up and marking the block under construction missing.
- */
- private int recoveryAttemptsBeforeMarkingBlockMissing;
-
/**
* ReplicaUnderConstruction contains information about replicas while
* they are under construction.
@@ -178,8 +174,6 @@ public abstract class BlockInfoUnderConstruction extends BlockInfo {
"BlockInfoUnderConstruction cannot be in COMPLETE state");
this.blockUCState = state;
setExpectedLocations(targets);
- this.recoveryAttemptsBeforeMarkingBlockMissing =
- BlockManager.getMaxBlockUCRecoveries();
}
/** Set expected locations. */
@@ -277,7 +271,7 @@ public abstract class BlockInfoUnderConstruction extends BlockInfo {
if (replicas.size() == 0) {
NameNode.blockStateChangeLog.warn("BLOCK* " +
"BlockInfoUnderConstruction.initLeaseRecovery: " +
- "No replicas found.");
+ "No blocks found, lease removed.");
}
boolean allLiveReplicasTriedAsPrimary = true;
for (int i = 0; i < replicas.size(); i++) {
@@ -289,11 +283,6 @@ public abstract class BlockInfoUnderConstruction extends BlockInfo {
}
}
if (allLiveReplicasTriedAsPrimary) {
- recoveryAttemptsBeforeMarkingBlockMissing--;
- NameNode.blockStateChangeLog.info("Tried to recover " + this +" using all"
- + " replicas. Will try " + recoveryAttemptsBeforeMarkingBlockMissing
- + " more times");
-
// Just set all the replicas to be chosen whether they are alive or not.
for (int i = 0; i < replicas.size(); i++) {
replicas.get(i).setChosenAsPrimary(false);
@@ -352,10 +341,6 @@ public abstract class BlockInfoUnderConstruction extends BlockInfo {
replicas.add(new ReplicaUnderConstruction(block, storage, rState));
}
- public int getNumRecoveryAttemptsLeft() {
- return recoveryAttemptsBeforeMarkingBlockMissing;
- }
-
/**
* Convert an under construction block to a complete block.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5137b388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 16dfe45..7dce2a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
import static org.apache.hadoop.util.ExitUtil.terminate;
import java.io.IOException;
@@ -273,13 +274,6 @@ public class BlockManager implements BlockStatsMXBean {
private BlockPlacementPolicy blockplacement;
private final BlockStoragePolicySuite storagePolicySuite;
- /** The number of times a block under construction's recovery will be
- * attempted using all known replicas. e.g. if there are 3 replicas, each
- * node will be tried 5 times (for a total of 15 retries across all nodes)*/
- private static int maxBlockUCRecoveries =
- DFSConfigKeys.DFS_BLOCK_UC_MAX_RECOVERY_ATTEMPTS_DEFAULT;
- public static int getMaxBlockUCRecoveries() { return maxBlockUCRecoveries; }
-
/** Check whether name system is running before terminating */
private boolean checkNSRunning = true;
@@ -288,9 +282,6 @@ public class BlockManager implements BlockStatsMXBean {
this.namesystem = namesystem;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
- maxBlockUCRecoveries = conf.getInt(
- DFSConfigKeys.DFS_BLOCK_UC_MAX_RECOVERY_ATTEMPTS,
- DFSConfigKeys.DFS_BLOCK_UC_MAX_RECOVERY_ATTEMPTS_DEFAULT);
startupDelayBlockDeletionInMs = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
@@ -740,8 +731,7 @@ public class BlockManager implements BlockStatsMXBean {
/**
* Force the given block in the given file to be marked as complete,
* regardless of whether enough replicas are present. This is necessary
- * when tailing edit logs as a Standby or when recovering a lease on a file
- * with missing blocks.
+ * when tailing edit logs as a Standby.
*/
public BlockInfo forceCompleteBlock(final BlockCollection bc,
final BlockInfoUnderConstruction block) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5137b388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index becd05a..7c6d6a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3287,16 +3287,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
+ "Removed empty last block and closed file.");
return true;
}
-
- //If the block's recovery has been attempted enough times, mark the block
- //complete anyway and recover the lease
- if(uc.getNumRecoveryAttemptsLeft() == 0) {
- blockManager.forceCompleteBlock(pendingFile, uc);
- finalizeINodeFileUnderConstruction(src, pendingFile,
- iip.getLatestSnapshotId());
- return true;
- }
-
// start recovery of the last block for this file
long blockRecoveryId = nextGenerationStamp(blockIdManager.isLegacyBlock(uc));
lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5137b388/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index dee9111..8cb7d5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -425,15 +425,6 @@
</property>
<property>
- <name>dfs.block.uc.max.recovery.attempts</name>
- <value>5</value>
- <description>The number of times a block under construction's recovery will be
- attempted using all known replicas. e.g. if there are 3 replicas, each node
- will be tried 5 times (for a total of 15 retries across all nodes).
- </description>
-</property>
-
-<property>
<name>dfs.datanode.data.dir</name>
<value>file://${hadoop.tmp.dir}/dfs/data</value>
<description>Determines where on the local filesystem an DFS data node
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5137b388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
index c9448ac..c9f3842 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
@@ -27,7 +27,6 @@ import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -259,81 +258,4 @@ public class TestLeaseRecovery {
}
}
}
-
- /**
- * Test that when a client was writing to a file and died, and before the
- * lease can be recovered, all the datanodes to which the file was written
- * also die, after some time (5 * lease recovery times) the file is indeed
- * closed and lease recovered.
- * We also check that if the datanode came back after some time, the data
- * originally written is not truncated
- * @throws IOException
- * @throws InterruptedException
- */
- @Test
- public void testLeaseRecoveryWithMissingBlocks()
- throws IOException, InterruptedException {
- Configuration conf = new HdfsConfiguration();
-
- //Start a cluster with 3 datanodes
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
- cluster.setLeasePeriod(LEASE_PERIOD, LEASE_PERIOD);
- cluster.waitActive();
-
- //create a file (with replication 1)
- Path file = new Path("/testRecoveryFile");
- DistributedFileSystem dfs = cluster.getFileSystem();
- FSDataOutputStream out = dfs.create(file, (short) 1);
-
- //This keeps count of the number of bytes written (AND is also the data we
- //are writing)
- long writtenBytes = 0;
- while (writtenBytes < 2 * 1024 * 1024) {
- out.writeLong(writtenBytes);
- writtenBytes += 8;
- }
- System.out.println("Written " + writtenBytes + " bytes");
- out.hsync();
- System.out.println("hsynced the data");
-
- //Kill the datanode to which the file was written.
- DatanodeInfo dn =
- ((DFSOutputStream) out.getWrappedStream()).getPipeline()[0];
- DataNodeProperties dnStopped = cluster.stopDataNode(dn.getName());
-
- //Wait at most 20 seconds for the lease to be recovered
- LeaseManager lm = NameNodeAdapter.getLeaseManager(cluster.getNamesystem());
- int i = 40;
- while(i-- > 0 && lm.countLease() != 0) {
- System.out.println("Still got " + lm.countLease() + " lease(s)");
- Thread.sleep(500);
- }
- assertTrue("The lease was not recovered", lm.countLease() == 0);
- System.out.println("Got " + lm.countLease() + " leases");
-
- //Make sure we can't read any data because the datanode is dead
- FSDataInputStream in = dfs.open(file);
- try {
- in.readLong();
- assertTrue("Shouldn't have reached here", false);
- } catch(BlockMissingException bme) {
- System.out.println("Correctly got BlockMissingException because datanode"
- + " is still dead");
- }
-
- //Bring the dead datanode back.
- cluster.restartDataNode(dnStopped);
- System.out.println("Restart datanode");
-
- //Make sure we can read all the data back (since we hsync'ed).
- in = dfs.open(file);
- int readBytes = 0;
- while(in.available() != 0) {
- assertEquals("Didn't read the data we wrote", in.readLong(), readBytes);
- readBytes += 8;
- }
- assertEquals("Didn't get all the data", readBytes, writtenBytes);
- System.out.println("Read back all the " + readBytes + " bytes");
- }
-
}
[04/16] hadoop git commit: HADOOP-12081. Fix
UserGroupInformation.java to support 64-bit zLinux. (aajisaka)
Posted by aw...@apache.org.
HADOOP-12081. Fix UserGroupInformation.java to support 64-bit zLinux. (aajisaka)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/773c6709
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/773c6709
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/773c6709
Branch: refs/heads/HADOOP-12111
Commit: 773c670943757681feeafb227a2d0c29d48f38f1
Parents: d6d5860
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Jul 21 11:21:49 2015 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Jul 21 11:21:49 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/security/UserGroupInformation.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773c6709/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a23a508..ef8e238 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -705,6 +705,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11893. Mark org.apache.hadoop.security.token.Token as
@InterfaceAudience.Public. (Brahma Reddy Battula via stevel)
+ HADOOP-12081. Fix UserGroupInformation.java to support 64-bit zLinux.
+ (aajisaka)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773c6709/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index be3d60d..80a0898 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -369,7 +369,8 @@ public class UserGroupInformation {
private static final boolean windows =
System.getProperty("os.name").startsWith("Windows");
private static final boolean is64Bit =
- System.getProperty("os.arch").contains("64");
+ System.getProperty("os.arch").contains("64") ||
+ System.getProperty("os.arch").contains("s390x");
private static final boolean aix = System.getProperty("os.name").equals("AIX");
/* Return the OS login module class name */
[05/16] hadoop git commit: Move HDFS-6945 to 2.7.2 section in
CHANGES.txt.
Posted by aw...@apache.org.
Move HDFS-6945 to 2.7.2 section in CHANGES.txt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a628f675
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a628f675
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a628f675
Branch: refs/heads/HADOOP-12111
Commit: a628f675900d2533ddf86fb3d3e601238ecd68c3
Parents: 773c670
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Jul 21 11:45:00 2015 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Jul 21 11:45:00 2015 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a628f675/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 388b553..1293388 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -762,9 +762,6 @@ Release 2.8.0 - UNRELEASED
HDFS-7997. The first non-existing xattr should also throw IOException.
(zhouyingchao via yliu)
- HDFS-6945. BlockManager should remove a block from excessReplicateMap and
- decrement ExcessBlocks metric when the block is removed. (aajisaka)
-
HDFS-7922. ShortCircuitCache#close is not releasing
ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe)
@@ -1072,8 +1069,11 @@ Release 2.7.2 - UNRELEASED
HDFS-8722. Optimize datanode writes for small writes and flushes (kihwal)
BUG FIXES
-
-Release 2.7.1 - 2015-07-06
+
+ HDFS-6945. BlockManager should remove a block from excessReplicateMap and
+ decrement ExcessBlocks metric when the block is removed. (aajisaka)
+
+Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
[06/16] hadoop git commit: HDFS-7483. Display information per tier on
the Namenode UI. Contributed by Benoy Antony and Haohui Mai.
Posted by aw...@apache.org.
HDFS-7483. Display information per tier on the Namenode UI. Contributed by Benoy Antony and Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df1e8ce4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df1e8ce4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df1e8ce4
Branch: refs/heads/HADOOP-12111
Commit: df1e8ce44a4716b2cb4ff3d161d7df8081572290
Parents: a628f67
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Jul 20 20:10:53 2015 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Mon Jul 20 20:13:12 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../src/main/webapps/hdfs/dfshealth.html | 25 ++++++++++++++++++++
.../src/main/webapps/hdfs/dfshealth.js | 8 +++++++
.../blockmanagement/TestBlockStatsMXBean.java | 1 -
4 files changed, 36 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df1e8ce4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1293388..f38a870 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -729,6 +729,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8794. Improve CorruptReplicasMap#corruptReplicasMap. (yliu)
+ HDFS-7483. Display information per tier on the Namenode UI.
+ (Benoy Antony and wheat9 via wheat9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df1e8ce4/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index 5a3a309..8cdff84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -199,6 +199,31 @@
{#failed}{#helper_dir_status type="Failed"/}{/failed}
{/nn.NameDirStatuses}
</table>
+<div class="page-header"><h1>DFS Storage Types</h1></div>
+<small>
+<table class="table">
+ <thead>
+ <tr>
+ <th>Storage Type</th>
+ <th>Configured Capacity</th>
+ <th>Capacity Used</th>
+ <th>Capacity Remaining</th>
+ <th>Block Pool Used</th>
+ <th>Nodes In Service</th>
+ </tr>
+ </thead>
+ {#blockstats.StorageTypeStats}
+ <tr>
+ <td>{key}</td>
+ <td>{value.capacityTotal|fmt_bytes}</td>
+ <td>{value.capacityUsed|fmt_bytes} ({value.capacityUsedPercentage|fmt_percentage})</td>
+ <td>{value.capacityRemaining|fmt_bytes} ({value.capacityRemainingPercentage|fmt_percentage})</td>
+ <td>{value.blockPoolUsed|fmt_bytes}</td>
+ <td>{value.nodesInService}</td>
+ </tr>
+ {/blockstats.StorageTypeStats}
+</table>
+</small>
</script>
<script type="text/x-dust-template" id="tmpl-snapshot">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df1e8ce4/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
index a045e42..1c13493 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
@@ -29,6 +29,7 @@
{"name": "nn", "url": "/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo"},
{"name": "nnstat", "url": "/jmx?qry=Hadoop:service=NameNode,name=NameNodeStatus"},
{"name": "fs", "url": "/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState"},
+ {"name": "blockstats", "url": "/jmx?qry=Hadoop:service=NameNode,name=BlockStats"},
{"name": "mem", "url": "/jmx?qry=java.lang:type=Memory"}
];
@@ -88,6 +89,13 @@
for (var k in d) {
data[k] = k === 'nn' ? workaround(d[k].beans[0]) : d[k].beans[0];
}
+
+ var blockstats = data['blockstats'];
+ for (var k in blockstats.StorageTypeStats) {
+ var b = blockstats.StorageTypeStats[k].value;
+ b.capacityUsedPercentage = b.capacityUsed * 100.0 / b.capacityTotal;
+ b.capacityRemainingPercentage = b.capacityRemaining * 100.0 / b.capacityTotal;
+ }
render();
}),
function (url, jqxhr, text, err) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df1e8ce4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
index 43d983d..2fe8768 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
@@ -106,7 +106,6 @@ public class TestBlockStatsMXBean {
public void testStorageTypeStatsJMX() throws Exception {
URL baseUrl = new URL (cluster.getHttpUri(0));
String result = readOutput(new URL(baseUrl, "/jmx"));
- System.out.println(result);
Map<String, Object> stat = (Map<String, Object>) JSON.parse(result);
Object[] beans =(Object[]) stat.get("beans");
[08/16] hadoop git commit: HDFS-7582. Enforce maximum number of ACL
entries separately per access and default. (Contributed by Vinayakumar B)
Posted by aw...@apache.org.
HDFS-7582. Enforce maximum number of ACL entries separately per access and default. (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29cf887b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29cf887b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29cf887b
Branch: refs/heads/HADOOP-12111
Commit: 29cf887b226f4ab3c336a6e681db5e8e70699d66
Parents: 87f29c6
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jul 21 15:16:52 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Jul 21 15:16:52 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/namenode/AclTransformation.java | 30 +++++++----
.../server/namenode/TestAclTransformation.java | 55 ++++++++++++++++++--
3 files changed, 76 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29cf887b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f38a870..6c91c45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1057,6 +1057,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8344. NameNode doesn't recover lease for files with missing blocks
(raviprak)
+ HDFS-7582. Enforce maximum number of ACL entries separately per access
+ and default. (vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29cf887b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java
index 1474e03..c887e9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java
@@ -271,10 +271,6 @@ final class AclTransformation {
*/
private static List<AclEntry> buildAndValidateAcl(
ArrayList<AclEntry> aclBuilder) throws AclException {
- if (aclBuilder.size() > MAX_ENTRIES) {
- throw new AclException("Invalid ACL: ACL has " + aclBuilder.size() +
- " entries, which exceeds maximum of " + MAX_ENTRIES + ".");
- }
aclBuilder.trimToSize();
Collections.sort(aclBuilder, ACL_ENTRY_COMPARATOR);
// Full iteration to check for duplicates and invalid named entries.
@@ -292,9 +288,12 @@ final class AclTransformation {
}
prevEntry = entry;
}
+
+ ScopedAclEntries scopedEntries = new ScopedAclEntries(aclBuilder);
+ checkMaxEntries(scopedEntries);
+
// Search for the required base access entries. If there is a default ACL,
// then do the same check on the default entries.
- ScopedAclEntries scopedEntries = new ScopedAclEntries(aclBuilder);
for (AclEntryType type: EnumSet.of(USER, GROUP, OTHER)) {
AclEntry accessEntryKey = new AclEntry.Builder().setScope(ACCESS)
.setType(type).build();
@@ -316,6 +315,22 @@ final class AclTransformation {
return Collections.unmodifiableList(aclBuilder);
}
+ // Check the max entries separately on access and default entries
+ // HDFS-7582
+ private static void checkMaxEntries(ScopedAclEntries scopedEntries)
+ throws AclException {
+ List<AclEntry> accessEntries = scopedEntries.getAccessEntries();
+ List<AclEntry> defaultEntries = scopedEntries.getDefaultEntries();
+ if (accessEntries.size() > MAX_ENTRIES) {
+ throw new AclException("Invalid ACL: ACL has " + accessEntries.size()
+ + " access entries, which exceeds maximum of " + MAX_ENTRIES + ".");
+ }
+ if (defaultEntries.size() > MAX_ENTRIES) {
+ throw new AclException("Invalid ACL: ACL has " + defaultEntries.size()
+ + " default entries, which exceeds maximum of " + MAX_ENTRIES + ".");
+ }
+ }
+
/**
* Calculates mask entries required for the ACL. Mask calculation is performed
* separately for each scope: access and default. This method is responsible
@@ -444,11 +459,8 @@ final class AclTransformation {
* @throws AclException if validation fails
*/
public ValidatedAclSpec(List<AclEntry> aclSpec) throws AclException {
- if (aclSpec.size() > MAX_ENTRIES) {
- throw new AclException("Invalid ACL: ACL spec has " + aclSpec.size() +
- " entries, which exceeds maximum of " + MAX_ENTRIES + ".");
- }
Collections.sort(aclSpec, ACL_ENTRY_COMPARATOR);
+ checkMaxEntries(new ScopedAclEntries(aclSpec));
this.aclSpec = aclSpec;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29cf887b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAclTransformation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAclTransformation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAclTransformation.java
index 23a2677..f66bf2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAclTransformation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAclTransformation.java
@@ -31,11 +31,8 @@ import com.google.common.collect.Lists;
import org.junit.Test;
import org.apache.hadoop.fs.permission.AclEntry;
-import org.apache.hadoop.fs.permission.AclEntryScope;
-import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.protocol.AclException;
-import org.apache.hadoop.hdfs.server.namenode.AclTransformation;
/**
* Tests operations that modify ACLs. All tests in this suite have been
@@ -45,10 +42,13 @@ import org.apache.hadoop.hdfs.server.namenode.AclTransformation;
public class TestAclTransformation {
private static final List<AclEntry> ACL_SPEC_TOO_LARGE;
+ private static final List<AclEntry> ACL_SPEC_DEFAULT_TOO_LARGE;
static {
ACL_SPEC_TOO_LARGE = Lists.newArrayListWithCapacity(33);
+ ACL_SPEC_DEFAULT_TOO_LARGE = Lists.newArrayListWithCapacity(33);
for (int i = 0; i < 33; ++i) {
ACL_SPEC_TOO_LARGE.add(aclEntry(ACCESS, USER, "user" + i, ALL));
+ ACL_SPEC_DEFAULT_TOO_LARGE.add(aclEntry(DEFAULT, USER, "user" + i, ALL));
}
}
@@ -351,6 +351,17 @@ public class TestAclTransformation {
filterAclEntriesByAclSpec(existing, ACL_SPEC_TOO_LARGE);
}
+ @Test(expected = AclException.class)
+ public void testFilterDefaultAclEntriesByAclSpecInputTooLarge()
+ throws AclException {
+ List<AclEntry> existing = new ImmutableList.Builder<AclEntry>()
+ .add(aclEntry(DEFAULT, USER, ALL))
+ .add(aclEntry(DEFAULT, GROUP, READ))
+ .add(aclEntry(DEFAULT, OTHER, NONE))
+ .build();
+ filterAclEntriesByAclSpec(existing, ACL_SPEC_DEFAULT_TOO_LARGE);
+ }
+
@Test
public void testFilterDefaultAclEntries() throws AclException {
List<AclEntry> existing = new ImmutableList.Builder<AclEntry>()
@@ -720,6 +731,16 @@ public class TestAclTransformation {
}
@Test(expected=AclException.class)
+ public void testMergeAclDefaultEntriesInputTooLarge() throws AclException {
+ List<AclEntry> existing = new ImmutableList.Builder<AclEntry>()
+ .add(aclEntry(DEFAULT, USER, ALL))
+ .add(aclEntry(DEFAULT, GROUP, READ))
+ .add(aclEntry(DEFAULT, OTHER, NONE))
+ .build();
+ mergeAclEntries(existing, ACL_SPEC_DEFAULT_TOO_LARGE);
+ }
+
+ @Test(expected=AclException.class)
public void testMergeAclEntriesResultTooLarge() throws AclException {
ImmutableList.Builder<AclEntry> aclBuilder =
new ImmutableList.Builder<AclEntry>()
@@ -737,6 +758,24 @@ public class TestAclTransformation {
mergeAclEntries(existing, aclSpec);
}
+ @Test(expected = AclException.class)
+ public void testMergeAclDefaultEntriesResultTooLarge() throws AclException {
+ ImmutableList.Builder<AclEntry> aclBuilder =
+ new ImmutableList.Builder<AclEntry>()
+ .add(aclEntry(DEFAULT, USER, ALL));
+ for (int i = 1; i <= 28; ++i) {
+ aclBuilder.add(aclEntry(DEFAULT, USER, "user" + i, READ));
+ }
+ aclBuilder
+ .add(aclEntry(DEFAULT, GROUP, READ))
+ .add(aclEntry(DEFAULT, MASK, READ))
+ .add(aclEntry(DEFAULT, OTHER, NONE));
+ List<AclEntry> existing = aclBuilder.build();
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, USER, "bruce", READ));
+ mergeAclEntries(existing, aclSpec);
+ }
+
@Test(expected=AclException.class)
public void testMergeAclEntriesDuplicateEntries() throws AclException {
List<AclEntry> existing = new ImmutableList.Builder<AclEntry>()
@@ -1092,6 +1131,16 @@ public class TestAclTransformation {
}
@Test(expected=AclException.class)
+ public void testReplaceAclDefaultEntriesInputTooLarge() throws AclException {
+ List<AclEntry> existing = new ImmutableList.Builder<AclEntry>()
+ .add(aclEntry(DEFAULT, USER, ALL))
+ .add(aclEntry(DEFAULT, GROUP, READ))
+ .add(aclEntry(DEFAULT, OTHER, NONE))
+ .build();
+ replaceAclEntries(existing, ACL_SPEC_DEFAULT_TOO_LARGE);
+ }
+
+ @Test(expected=AclException.class)
public void testReplaceAclEntriesResultTooLarge() throws AclException {
List<AclEntry> existing = new ImmutableList.Builder<AclEntry>()
.add(aclEntry(ACCESS, USER, ALL))
[12/16] hadoop git commit: YARN-3261. rewrite resourcemanager restart
doc to remove roadmap bits (Gururaj Shetty via aw)
Posted by aw...@apache.org.
YARN-3261. rewrite resourcemanager restart doc to remove roadmap bits (Gururaj Shetty via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b7ffc4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b7ffc4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b7ffc4f
Branch: refs/heads/HADOOP-12111
Commit: 3b7ffc4f3f0ffb0fa6c324da6d88803f5b233832
Parents: c39ca54
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jul 21 10:00:20 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Jul 21 10:00:34 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../src/site/markdown/ResourceManagerRestart.md | 32 +++++++++-----------
2 files changed, 16 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b7ffc4f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7259cf2..79e9ae2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -29,6 +29,8 @@ Trunk - Unreleased
YARN-2280. Resource manager web service fields are not accessible
(Krisztian Horvath via aw)
+ YARN-3261. rewrite resourcemanager restart doc to remove roadmap bits (Gururaj Shetty via aw)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b7ffc4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md
index d23505d..ee222c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md
@@ -31,34 +31,30 @@ ResourceManger Restart
Overview
--------
-ResourceManager is the central authority that manages resources and schedules applications running atop of YARN. Hence, it is potentially a single point of failure in a Apache YARN cluster.
-`
-This document gives an overview of ResourceManager Restart, a feature that enhances ResourceManager to keep functioning across restarts and also makes ResourceManager down-time invisible to end-users.
+ResourceManager is the central authority that manages resources and schedules applications running on YARN. Hence, it is potentially a single point of failure in an Apache YARN cluster. This document gives an overview of ResourceManager Restart, a feature that enhances ResourceManager to keep functioning across restarts and also makes ResourceManager down-time invisible to end-users.
-ResourceManager Restart feature is divided into two phases:
+There are two types of restart for ResourceManager:
-* **ResourceManager Restart Phase 1 (Non-work-preserving RM restart)**: Enhance RM to persist application/attempt state and other credentials information in a pluggable state-store. RM will reload this information from state-store upon restart and re-kick the previously running applications. Users are not required to re-submit the applications.
+* **Non-work-preserving RM restart**: This restart enhances RM to persist application/attempt state and other credentials information in a pluggable state-store. RM will reload this information from state-store on restart and re-kick the previously running applications. Users are not required to re-submit the applications.
-* **ResourceManager Restart Phase 2 (Work-preserving RM restart)**: Focus on re-constructing the running state of ResourceManager by combining the container statuses from NodeManagers and container requests from ApplicationMasters upon restart. The key difference from phase 1 is that previously running applications will not be killed after RM restarts, and so applications won't lose its work because of RM outage.
+* **Work-preserving RM restart**: This focuses on re-constructing the running state of RM by combining the container status from NodeManagers and container requests from ApplicationMasters on restart. The key difference from Non-work-preserving RM restart is that previously running applications will not be killed after RM restarts, and so applications will not lose its work because of RM outage.
Feature
-------
-* **Phase 1: Non-work-preserving RM restart**
+* **Non-work-preserving RM restart**
- As of Hadoop 2.4.0 release, only ResourceManager Restart Phase 1 is implemented which is described below.
+ In non-work-preserving RM restart, RM will save the application metadata (i.e. ApplicationSubmissionContext) in a pluggable state-store when client submits an application and also saves the final status of the application such as the completion state (failed, killed, or finished) and diagnostics when the application completes. Besides, RM also saves the credentials like security keys, tokens to work in a secure environment. When RM shuts down, as long as the required information (i.e.application metadata and the alongside credentials if running in a secure environment) is available in the state-store, then when RM restarts, it can pick up the application metadata from the state-store and re-submit the application. RM won't re-submit the applications if they were already completed (i.e. failed, killed, or finished) before RM went down.
- The overall concept is that RM will persist the application metadata (i.e. ApplicationSubmissionContext) in a pluggable state-store when client submits an application and also saves the final status of the application such as the completion state (failed, killed, finished) and diagnostics when the application completes. Besides, RM also saves the credentials like security keys, tokens to work in a secure environment. Any time RM shuts down, as long as the required information (i.e.application metadata and the alongside credentials if running in a secure environment) is available in the state-store, when RM restarts, it can pick up the application metadata from the state-store and re-submit the application. RM won't re-submit the applications if they were already completed (i.e. failed, killed, finished) before RM went down.
+ NodeManagers and clients during the down-time of RM will keep polling RM until RM comes up. When RM comes up, it will send a re-sync command to all the NodeManagers and ApplicationMasters it was talking to via heartbeats. The NMs will kill all its managed containers and re-register with RM. These re-registered NodeManagers are similar to the newly joining NMs. AMs (e.g. MapReduce AM) are expected to shutdown when they receive the re-sync command. After RM restarts and loads all the application metadata, credentials from state-store and populates them into memory, it will create a new attempt (i.e. ApplicationMaster) for each application that was not yet completed and re-kick that application as usual. As described before, the previously running applications' work is lost in this manner since they are essentially killed by RM via the re-sync command on restart.
- NodeManagers and clients during the down-time of RM will keep polling RM until RM comes up. When RM becomes alive, it will send a re-sync command to all the NodeManagers and ApplicationMasters it was talking to via heartbeats. As of Hadoop 2.4.0 release, the behaviors for NodeManagers and ApplicationMasters to handle this command are: NMs will kill all its managed containers and re-register with RM. From the RM's perspective, these re-registered NodeManagers are similar to the newly joining NMs. AMs(e.g. MapReduce AM) are expected to shutdown when they receive the re-sync command. After RM restarts and loads all the application metadata, credentials from state-store and populates them into memory, it will create a new attempt (i.e. ApplicationMaster) for each application that was not yet completed and re-kick that application as usual. As described before, the previously running applications' work is lost in this manner since they are essentially killed by RM via the re-sync co
mmand on restart.
-* **Phase 2: Work-preserving RM restart**
+* **Work-preserving RM restart**
- As of Hadoop 2.6.0, we further enhanced RM restart feature to address the problem to not kill any applications running on YARN cluster if RM restarts.
+ In work-preserving RM restart, RM ensures the persistency of application state and reload that state on recovery, this restart primarily focuses on re-constructing the entire running state of YARN cluster, the majority of which is the state of the central scheduler inside RM which keeps track of all containers' life-cycle, applications' headroom and resource requests, queues' resource usage and so on. In this way, RM need not kill the AM and re-run the application from scratch as it is done in non-work-preserving RM restart. Applications can simply re-sync back with RM and resume from where it were left off.
- Beyond all the groundwork that has been done in Phase 1 to ensure the persistency of application state and reload that state on recovery, Phase 2 primarily focuses on re-constructing the entire running state of YARN cluster, the majority of which is the state of the central scheduler inside RM which keeps track of all containers' life-cycle, applications' headroom and resource requests, queues' resource usage etc. In this way, RM doesn't need to kill the AM and re-run the application from scratch as it is done in Phase 1. Applications can simply re-sync back with RM and resume from where it were left off.
+ RM recovers its running state by taking advantage of the container status sent from all NMs. NM will not kill the containers when it re-syncs with the restarted RM. It continues managing the containers and sends the container status across to RM when it re-registers. RM reconstructs the container instances and the associated applications' scheduling status by absorbing these containers' information. In the meantime, AM needs to re-send the outstanding resource requests to RM because RM may lose the unfulfilled requests when it shuts down. Application writers using AMRMClient library to communicate with RM do not need to worry about the part of AM re-sending resource requests to RM on re-sync, as it is automatically taken care by the library itself.
- RM recovers its runing state by taking advantage of the container statuses sent from all NMs. NM will not kill the containers when it re-syncs with the restarted RM. It continues managing the containers and send the container statuses across to RM when it re-registers. RM reconstructs the container instances and the associated applications' scheduling status by absorbing these containers' information. In the meantime, AM needs to re-send the outstanding resource requests to RM because RM may lose the unfulfilled requests when it shuts down. Application writers using AMRMClient library to communicate with RM do not need to worry about the part of AM re-sending resource requests to RM on re-sync, as it is automatically taken care by the library itself.
Configurations
--------------
@@ -103,7 +99,7 @@ This section describes the configurations involved to enable RM Restart feature.
| `yarn.resourcemanager.fs.state-store.retry-policy-spec` | Hadoop FileSystem client retry policy specification. Hadoop FileSystem client retry is always enabled. Specified in pairs of sleep-time and number-of-retries i.e. (t0, n0), (t1, n1), ..., the first n0 retries sleep t0 milliseconds on average, the following n1 retries sleep t1 milliseconds on average, and so on. Default value is (2000, 500) |
### Configurations for ZooKeeper based state-store implementation
-
+
* Configure the ZooKeeper server address and the root path where the RM state is stored.
| Property | Description |
@@ -145,7 +141,7 @@ ContainerId string format is changed if RM restarts with work-preserving recover
It is now changed to:
`Container_`**e{epoch}**`_{clusterTimestamp}_{appId}_{attemptId}_{containerId}`, e.g. `Container_`**e17**`_1410901177871_0001_01_000005`.
-
+
Here, the additional epoch number is a monotonically increasing integer which starts from 0 and is increased by 1 each time RM restarts. If epoch number is 0, it is omitted and the containerId string format stays the same as before.
Sample Configurations
@@ -155,12 +151,12 @@ Below is a minimum set of configurations for enabling RM work-preserving restart
<property>
- <description>Enable RM to recover state after starting. If true, then
+ <description>Enable RM to recover state after starting. If true, then
yarn.resourcemanager.store.class must be specified</description>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
-
+
<property>
<description>The class to use as the persistent store.</description>
<name>yarn.resourcemanager.store.class</name>
[02/16] hadoop git commit: HDFS-8657. Update docs for mSNN.
Contributed by Jesse Yates.
Posted by aw...@apache.org.
HDFS-8657. Update docs for mSNN. Contributed by Jesse Yates.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed01dc70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed01dc70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed01dc70
Branch: refs/heads/HADOOP-12111
Commit: ed01dc70b2f4ff4bdcaf71c19acf244da0868a82
Parents: e4f7562
Author: Aaron T. Myers <at...@apache.org>
Authored: Mon Jul 20 16:40:06 2015 -0700
Committer: Aaron T. Myers <at...@apache.org>
Committed: Mon Jul 20 16:40:06 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../markdown/HDFSHighAvailabilityWithNFS.md | 40 +++++++++++---------
.../markdown/HDFSHighAvailabilityWithQJM.md | 32 ++++++++++------
3 files changed, 45 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed01dc70/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 13d9969..cd32c0e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -341,6 +341,8 @@ Trunk (Unreleased)
HDFS-8627. NPE thrown if unable to fetch token from Namenode
(J.Andreina via vinayakumarb)
+ HDFS-8657. Update docs for mSNN. (Jesse Yates via atm)
+
Release 2.8.0 - UNRELEASED
NEW FEATURES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed01dc70/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
index 626a473..cc53a38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
@@ -65,18 +65,18 @@ This impacted the total availability of the HDFS cluster in two major ways:
* Planned maintenance events such as software or hardware upgrades on the
NameNode machine would result in windows of cluster downtime.
-The HDFS High Availability feature addresses the above problems by providing the option of running two redundant NameNodes in the same cluster in an Active/Passive configuration with a hot standby. This allows a fast failover to a new NameNode in the case that a machine crashes, or a graceful administrator-initiated failover for the purpose of planned maintenance.
+The HDFS High Availability feature addresses the above problems by providing the option of running two (or more, as of Hadoop 3.0.0) redundant NameNodes in the same cluster in an Active/Passive configuration with a hot standby(s). This allows a fast failover to a new NameNode in the case that a machine crashes, or a graceful administrator-initiated failover for the purpose of planned maintenance.
Architecture
------------
-In a typical HA cluster, two separate machines are configured as NameNodes. At any point in time, exactly one of the NameNodes is in an *Active* state, and the other is in a *Standby* state. The Active NameNode is responsible for all client operations in the cluster, while the Standby is simply acting as a slave, maintaining enough state to provide a fast failover if necessary.
+In a typical HA cluster, two or more separate machines are configured as NameNodes. At any point in time, exactly one of the NameNodes is in an *Active* state, and the others are in a *Standby* state. The Active NameNode is responsible for all client operations in the cluster, while the Standby is simply acting as a slave, maintaining enough state to provide a fast failover if necessary.
-In order for the Standby node to keep its state synchronized with the Active node, the current implementation requires that the two nodes both have access to a directory on a shared storage device (eg an NFS mount from a NAS). This restriction will likely be relaxed in future versions.
+In order for the Standby nodes to keep their state synchronized with the Active node, the current implementation requires that the nodes have access to a directory on a shared storage device (eg an NFS mount from a NAS). This restriction will likely be relaxed in future versions.
-When any namespace modification is performed by the Active node, it durably logs a record of the modification to an edit log file stored in the shared directory. The Standby node is constantly watching this directory for edits, and as it sees the edits, it applies them to its own namespace. In the event of a failover, the Standby will ensure that it has read all of the edits from the shared storage before promoting itself to the Active state. This ensures that the namespace state is fully synchronized before a failover occurs.
+When any namespace modification is performed by the Active node, it durably logs a record of the modification to an edit log file stored in the shared directory. The Standby nodes are constantly watching this directory for edits, and as it sees the edits, it applies them to its own namespace. In the event of a failover, the Standby will ensure that it has read all of the edits from the shared storage before promoting itself to the Active state. This ensures that the namespace state is fully synchronized before a failover occurs.
-In order to provide a fast failover, it is also necessary that the Standby node have up-to-date information regarding the location of blocks in the cluster. In order to achieve this, the DataNodes are configured with the location of both NameNodes, and send block location information and heartbeats to both.
+In order to provide a fast failover, it is also necessary that the Standby nodes have up-to-date information regarding the location of blocks in the cluster. In order to achieve this, the DataNodes are configured with the location of all NameNodes, and send block location information and heartbeats to all the NameNodes.
It is vital for the correct operation of an HA cluster that only one of the NameNodes be Active at a time. Otherwise, the namespace state would quickly diverge between the two, risking data loss or other incorrect results. In order to ensure this property and prevent the so-called "split-brain scenario," the administrator must configure at least one *fencing method* for the shared storage. During a failover, if it cannot be verified that the previous Active node has relinquished its Active state, the fencing process is responsible for cutting off the previous Active's access to the shared edits storage. This prevents it from making any further edits to the namespace, allowing the new Active to safely proceed with failover.
@@ -87,9 +87,9 @@ In order to deploy an HA cluster, you should prepare the following:
* **NameNode machines** - the machines on which you run the Active and Standby NameNodes should have equivalent hardware to each other, and equivalent hardware to what would be used in a non-HA cluster.
-* **Shared storage** - you will need to have a shared directory which both NameNode machines can have read/write access to. Typically this is a remote filer which supports NFS and is mounted on each of the NameNode machines. Currently only a single shared edits directory is supported. Thus, the availability of the system is limited by the availability of this shared edits directory, and therefore in order to remove all single points of failure there needs to be redundancy for the shared edits directory. Specifically, multiple network paths to the storage, and redundancy in the storage itself (disk, network, and power). Beacuse of this, it is recommended that the shared storage server be a high-quality dedicated NAS appliance rather than a simple Linux server.
+* **Shared storage** - you will need to have a shared directory which the NameNode machines have read/write access to. Typically this is a remote filer which supports NFS and is mounted on each of the NameNode machines. Currently only a single shared edits directory is supported. Thus, the availability of the system is limited by the availability of this shared edits directory, and therefore in order to remove all single points of failure there needs to be redundancy for the shared edits directory. Specifically, multiple network paths to the storage, and redundancy in the storage itself (disk, network, and power). Beacuse of this, it is recommended that the shared storage server be a high-quality dedicated NAS appliance rather than a simple Linux server.
-Note that, in an HA cluster, the Standby NameNode also performs checkpoints of the namespace state, and thus it is not necessary to run a Secondary NameNode, CheckpointNode, or BackupNode in an HA cluster. In fact, to do so would be an error. This also allows one who is reconfiguring a non-HA-enabled HDFS cluster to be HA-enabled to reuse the hardware which they had previously dedicated to the Secondary NameNode.
+Note that, in an HA cluster, the Standby NameNodes also perform checkpoints of the namespace state, and thus it is not necessary to run a Secondary NameNode, CheckpointNode, or BackupNode in an HA cluster. In fact, to do so would be an error. This also allows one who is reconfiguring a non-HA-enabled HDFS cluster to be HA-enabled to reuse the hardware which they had previously dedicated to the Secondary NameNode.
Deployment
----------
@@ -124,17 +124,15 @@ The order in which you set these configurations is unimportant, but the values y
Configure with a list of comma-separated NameNode IDs. This will be used by
DataNodes to determine all the NameNodes in the cluster. For example, if you
- used "mycluster" as the nameservice ID previously, and you wanted to use "nn1"
- and "nn2" as the individual IDs of the NameNodes, you would configure this as
+ used "mycluster" as the nameservice ID previously, and you wanted to use "nn1","nn2" and "nn3" as the individual IDs of the NameNodes, you would configure this as
such:
<property>
<name>dfs.ha.namenodes.mycluster</name>
- <value>nn1,nn2</value>
+ <value>nn1,nn2,nn3</value>
</property>
- **Note:** Currently, only a maximum of two NameNodes may be configured per
- nameservice.
+ **Note:** The minimum number of NameNodes for HA is two, but you can configure more. Its suggested to not exceed 5 - with a recommended 3 NameNodes - due to communication overheads.
* **dfs.namenode.rpc-address.[nameservice ID].[name node ID]** - the fully-qualified RPC address for each NameNode to listen on
@@ -150,6 +148,10 @@ The order in which you set these configurations is unimportant, but the values y
<name>dfs.namenode.rpc-address.mycluster.nn2</name>
<value>machine2.example.com:8020</value>
</property>
+ <property>
+ <name>dfs.namenode.rpc-address.mycluster.nn3</name>
+ <value>machine3.example.com:8020</value>
+ </property>
**Note:** You may similarly configure the "**servicerpc-address**" setting if
you so desire.
@@ -167,6 +169,10 @@ The order in which you set these configurations is unimportant, but the values y
<name>dfs.namenode.http-address.mycluster.nn2</name>
<value>machine2.example.com:50070</value>
</property>
+ <property>
+ <name>dfs.namenode.http-address.mycluster.nn3</name>
+ <value>machine3.example.com:50070</value>
+ </property>
**Note:** If you have Hadoop's security features enabled, you should also set
the *https-address* similarly for each NameNode.
@@ -174,9 +180,9 @@ The order in which you set these configurations is unimportant, but the values y
* **dfs.namenode.shared.edits.dir** - the location of the shared storage directory
This is where one configures the path to the remote shared edits directory
- which the Standby NameNode uses to stay up-to-date with all the file system
+ which the Standby NameNodes use to stay up-to-date with all the file system
changes the Active NameNode makes. **You should only configure one of these
- directories.** This directory should be mounted r/w on both NameNode machines.
+ directories.** This directory should be mounted r/w on the NameNode machines.
The value of this setting should be the absolute path to this directory on the
NameNode machines. For example:
@@ -203,7 +209,7 @@ The order in which you set these configurations is unimportant, but the values y
It is critical for correctness of the system that only one NameNode be in the
Active state at any given time. Thus, during a failover, we first ensure that
the Active NameNode is either in the Standby state, or the process has
- terminated, before transitioning the other NameNode to the Active state. In
+ terminated, before transitioning another NameNode to the Active state. In
order to do this, you must configure at least one **fencing method.** These are
configured as a carriage-return-separated list, which will be attempted in order
until one indicates that fencing has succeeded. There are two methods which
@@ -320,7 +326,7 @@ After all of the necessary configuration options have been set, one must initial
* If you have already formatted the NameNode, or are converting a
non-HA-enabled cluster to be HA-enabled, you should now copy over the
contents of your NameNode metadata directories to the other, unformatted
- NameNode by running the command "*hdfs namenode -bootstrapStandby*" on the
+ NameNodes by running the command "*hdfs namenode -bootstrapStandby*" on the
unformatted NameNode. Running this command will also ensure that the shared
edits directory (as configured by **dfs.namenode.shared.edits.dir**) contains
sufficient edits transactions to be able to start both NameNodes.
@@ -329,7 +335,7 @@ After all of the necessary configuration options have been set, one must initial
command "*hdfs -initializeSharedEdits*", which will initialize the shared
edits directory with the edits data from the local NameNode edits directories.
-At this point you may start both of your HA NameNodes as you normally would start a NameNode.
+At this point you may start all of your HA NameNodes as you normally would start a NameNode.
You can visit each of the NameNodes' web pages separately by browsing to their configured HTTP addresses. You should notice that next to the configured address will be the HA state of the NameNode (either "standby" or "active".) Whenever an HA NameNode starts, it is initially in the Standby state.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed01dc70/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
index 06e1bb1..d9d9a67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
@@ -65,16 +65,16 @@ This impacted the total availability of the HDFS cluster in two major ways:
* Planned maintenance events such as software or hardware upgrades on the
NameNode machine would result in windows of cluster downtime.
-The HDFS High Availability feature addresses the above problems by providing the option of running two redundant NameNodes in the same cluster in an Active/Passive configuration with a hot standby. This allows a fast failover to a new NameNode in the case that a machine crashes, or a graceful administrator-initiated failover for the purpose of planned maintenance.
+The HDFS High Availability feature addresses the above problems by providing the option of running two (and as of 3.0.0 more than two) redundant NameNodes in the same cluster in an Active/Passive configuration with a hot standby. This allows a fast failover to a new NameNode in the case that a machine crashes, or a graceful administrator-initiated failover for the purpose of planned maintenance.
Architecture
------------
-In a typical HA cluster, two separate machines are configured as NameNodes. At any point in time, exactly one of the NameNodes is in an *Active* state, and the other is in a *Standby* state. The Active NameNode is responsible for all client operations in the cluster, while the Standby is simply acting as a slave, maintaining enough state to provide a fast failover if necessary.
+In a typical HA cluster, two or more separate machines are configured as NameNodes. At any point in time, exactly one of the NameNodes is in an *Active* state, and the others are in a *Standby* state. The Active NameNode is responsible for all client operations in the cluster, while the Standbys are simply acting as slaves, maintaining enough state to provide a fast failover if necessary.
In order for the Standby node to keep its state synchronized with the Active node, both nodes communicate with a group of separate daemons called "JournalNodes" (JNs). When any namespace modification is performed by the Active node, it durably logs a record of the modification to a majority of these JNs. The Standby node is capable of reading the edits from the JNs, and is constantly watching them for changes to the edit log. As the Standby Node sees the edits, it applies them to its own namespace. In the event of a failover, the Standby will ensure that it has read all of the edits from the JounalNodes before promoting itself to the Active state. This ensures that the namespace state is fully synchronized before a failover occurs.
-In order to provide a fast failover, it is also necessary that the Standby node have up-to-date information regarding the location of blocks in the cluster. In order to achieve this, the DataNodes are configured with the location of both NameNodes, and send block location information and heartbeats to both.
+In order to provide a fast failover, it is also necessary that the Standby node have up-to-date information regarding the location of blocks in the cluster. In order to achieve this, the DataNodes are configured with the location of all NameNodes, and send block location information and heartbeats to all.
It is vital for the correct operation of an HA cluster that only one of the NameNodes be Active at a time. Otherwise, the namespace state would quickly diverge between the two, risking data loss or other incorrect results. In order to ensure this property and prevent the so-called "split-brain scenario," the JournalNodes will only ever allow a single NameNode to be a writer at a time. During a failover, the NameNode which is to become active will simply take over the role of writing to the JournalNodes, which will effectively prevent the other NameNode from continuing in the Active state, allowing the new Active to safely proceed with failover.
@@ -99,7 +99,7 @@ In order to deploy an HA cluster, you should prepare the following:
running with N JournalNodes, the system can tolerate at most (N - 1) / 2
failures and continue to function normally.
-Note that, in an HA cluster, the Standby NameNode also performs checkpoints of the namespace state, and thus it is not necessary to run a Secondary NameNode, CheckpointNode, or BackupNode in an HA cluster. In fact, to do so would be an error. This also allows one who is reconfiguring a non-HA-enabled HDFS cluster to be HA-enabled to reuse the hardware which they had previously dedicated to the Secondary NameNode.
+Note that, in an HA cluster, the Standby NameNodes also performs checkpoints of the namespace state, and thus it is not necessary to run a Secondary NameNode, CheckpointNode, or BackupNode in an HA cluster. In fact, to do so would be an error. This also allows one who is reconfiguring a non-HA-enabled HDFS cluster to be HA-enabled to reuse the hardware which they had previously dedicated to the Secondary NameNode.
Deployment
----------
@@ -136,16 +136,16 @@ The order in which you set these configurations is unimportant, but the values y
Configure with a list of comma-separated NameNode IDs. This will be used by
DataNodes to determine all the NameNodes in the cluster. For example, if you
- used "mycluster" as the nameservice ID previously, and you wanted to use "nn1"
- and "nn2" as the individual IDs of the NameNodes, you would configure this as
+ used "mycluster" as the nameservice ID previously, and you wanted to use "nn1",
+ "nn2" and "nn3" as the individual IDs of the NameNodes, you would configure this as
such:
<property>
<name>dfs.ha.namenodes.mycluster</name>
- <value>nn1,nn2</value>
+ <value>nn1,nn2, nn3</value>
</property>
- **Note:** Currently, only a maximum of two NameNodes may be configured per nameservice.
+ **Note:** The minimum number of NameNodes for HA is two, but you can configure more. Its suggested to not exceed 5 - with a recommended 3 NameNodes - due to communication overheads.
* **dfs.namenode.rpc-address.[nameservice ID].[name node ID]** - the fully-qualified RPC address for each NameNode to listen on
@@ -161,6 +161,10 @@ The order in which you set these configurations is unimportant, but the values y
<name>dfs.namenode.rpc-address.mycluster.nn2</name>
<value>machine2.example.com:8020</value>
</property>
+ <property>
+ <name>dfs.namenode.rpc-address.mycluster.nn3</name>
+ <value>machine3.example.com:8020</value>
+ </property>
**Note:** You may similarly configure the "**servicerpc-address**" setting if you so desire.
@@ -177,6 +181,10 @@ The order in which you set these configurations is unimportant, but the values y
<name>dfs.namenode.http-address.mycluster.nn2</name>
<value>machine2.example.com:50070</value>
</property>
+ <property>
+ <name>dfs.namenode.http-address.mycluster.nn3</name>
+ <value>machine3.example.com:50070</value>
+ </property>
**Note:** If you have Hadoop's security features enabled, you should also set
the *https-address* similarly for each NameNode.
@@ -365,8 +373,8 @@ Once the JournalNodes have been started, one must initially synchronize the two
* If you have already formatted the NameNode, or are converting a
non-HA-enabled cluster to be HA-enabled, you should now copy over the
contents of your NameNode metadata directories to the other, unformatted
- NameNode by running the command "*hdfs namenode -bootstrapStandby*" on the
- unformatted NameNode. Running this command will also ensure that the
+ NameNode(s) by running the command "*hdfs namenode -bootstrapStandby*" on the
+ unformatted NameNode(s). Running this command will also ensure that the
JournalNodes (as configured by **dfs.namenode.shared.edits.dir**) contain
sufficient edits transactions to be able to start both NameNodes.
@@ -374,7 +382,7 @@ Once the JournalNodes have been started, one must initially synchronize the two
command "*hdfs namenode -initializeSharedEdits*", which will initialize the
JournalNodes with the edits data from the local NameNode edits directories.
-At this point you may start both of your HA NameNodes as you normally would start a NameNode.
+At this point you may start all your HA NameNodes as you normally would start a NameNode.
You can visit each of the NameNodes' web pages separately by browsing to their configured HTTP addresses. You should notice that next to the configured address will be the HA state of the NameNode (either "standby" or "active".) Whenever an HA NameNode starts, it is initially in the Standby state.
@@ -443,7 +451,7 @@ Apache ZooKeeper is a highly available service for maintaining small amounts of
* **Failure detection** - each of the NameNode machines in the cluster
maintains a persistent session in ZooKeeper. If the machine crashes, the
- ZooKeeper session will expire, notifying the other NameNode that a failover
+ ZooKeeper session will expire, notifying the other NameNode(s) that a failover
should be triggered.
* **Active NameNode election** - ZooKeeper provides a simple mechanism to
[11/16] hadoop git commit: YARN-2003. Support for Application
priority : Changes in RM and Capacity Scheduler. (Sunil G via wangda)
Posted by aw...@apache.org.
YARN-2003. Support for Application priority : Changes in RM and Capacity Scheduler. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c39ca541
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c39ca541
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c39ca541
Branch: refs/heads/HADOOP-12111
Commit: c39ca541f498712133890961598bbff50d89d68b
Parents: da2d1ac
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Jul 21 09:56:59 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Jul 21 09:57:23 2015 -0700
----------------------------------------------------------------------
.../sls/scheduler/ResourceSchedulerWrapper.java | 10 +
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../server/resourcemanager/RMAppManager.java | 20 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 15 +-
.../scheduler/AbstractYarnScheduler.java | 10 +
.../server/resourcemanager/scheduler/Queue.java | 8 +
.../scheduler/SchedulerApplication.java | 22 ++
.../scheduler/SchedulerApplicationAttempt.java | 15 +-
.../scheduler/YarnScheduler.java | 20 ++
.../scheduler/capacity/AbstractCSQueue.java | 7 +
.../scheduler/capacity/CapacityScheduler.java | 73 +++-
.../CapacitySchedulerConfiguration.java | 13 +
.../scheduler/capacity/LeafQueue.java | 19 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 8 +
.../scheduler/event/AppAddedSchedulerEvent.java | 28 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 6 +
.../scheduler/fifo/FifoScheduler.java | 6 +
.../scheduler/policy/FifoComparator.java | 11 +-
.../scheduler/policy/SchedulableEntity.java | 5 +
.../yarn/server/resourcemanager/MockRM.java | 31 +-
.../server/resourcemanager/TestAppManager.java | 1 +
.../TestWorkPreservingRMRestart.java | 2 +-
...pacityPreemptionPolicyForNodePartitions.java | 1 +
.../capacity/TestApplicationLimits.java | 5 +-
.../capacity/TestApplicationPriority.java | 345 +++++++++++++++++++
.../capacity/TestCapacityScheduler.java | 5 +
.../scheduler/policy/MockSchedulableEntity.java | 13 +-
.../security/TestDelegationTokenRenewer.java | 10 +-
.../TestRMWebServicesAppsModification.java | 2 +-
30 files changed, 664 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
index 08cb1e6..14e2645 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -949,4 +950,13 @@ final public class ResourceSchedulerWrapper
ContainerStatus containerStatus, RMContainerEventType event) {
// do nothing
}
+
+ @Override
+ public Priority checkAndGetApplicationPriority(Priority priority,
+ String user, String queueName, ApplicationId applicationId)
+ throws YarnException {
+ // TODO Dummy implementation.
+ return Priority.newInstance(0);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d0829c1..7259cf2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -139,6 +139,9 @@ Release 2.8.0 - UNRELEASED
YARN-3116. RM notifies NM whether a container is an AM container or normal
task container. (Giovanni Matteo Fumarola via zjshen)
+ YARN-2003. Support for Application priority : Changes in RM and Capacity
+ Scheduler. (Sunil G via wangda)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6b660f7..060635f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1928,6 +1928,11 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_NODELABEL_CONFIGURATION_TYPE =
CENTALIZED_NODELABEL_CONFIGURATION_TYPE;
+ public static final String MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY =
+ YARN_PREFIX + "cluster.max-application-priority";
+
+ public static final int DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY = 0;
+
@Private
public static boolean isDistributedNodeLabelConfiguration(Configuration conf) {
return DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE.equals(conf.get(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 2d9431d..6fd1838 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -329,14 +330,19 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
ResourceRequest amReq =
validateAndCreateResourceRequest(submissionContext, isRecovery);
+ // Verify and get the update application priority and set back to
+ // submissionContext
+ Priority appPriority = rmContext.getScheduler()
+ .checkAndGetApplicationPriority(submissionContext.getPriority(), user,
+ submissionContext.getQueue(), applicationId);
+ submissionContext.setPriority(appPriority);
+
// Create RMApp
- RMAppImpl application =
- new RMAppImpl(applicationId, rmContext, this.conf,
- submissionContext.getApplicationName(), user,
- submissionContext.getQueue(),
- submissionContext, this.scheduler, this.masterService,
- submitTime, submissionContext.getApplicationType(),
- submissionContext.getApplicationTags(), amReq);
+ RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf,
+ submissionContext.getApplicationName(), user,
+ submissionContext.getQueue(), submissionContext, this.scheduler,
+ this.masterService, submitTime, submissionContext.getApplicationType(),
+ submissionContext.getApplicationTags(), amReq);
// Concurrent app submissions with same applicationId will fail here
// Concurrent app submissions with different applicationIds will not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 62d5555..d480c24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -924,17 +924,15 @@ public class RMAppImpl implements RMApp, Recoverable {
// No existent attempts means the attempt associated with this app was not
// started or started but not yet saved.
if (app.attempts.isEmpty()) {
- app.scheduler.handle(new AppAddedSchedulerEvent(app.applicationId,
- app.submissionContext.getQueue(), app.user,
- app.submissionContext.getReservationID()));
+ app.scheduler.handle(new AppAddedSchedulerEvent(app.user,
+ app.submissionContext, false));
return RMAppState.SUBMITTED;
}
// Add application to scheduler synchronously to guarantee scheduler
// knows applications before AM or NM re-registers.
- app.scheduler.handle(new AppAddedSchedulerEvent(app.applicationId,
- app.submissionContext.getQueue(), app.user, true,
- app.submissionContext.getReservationID()));
+ app.scheduler.handle(new AppAddedSchedulerEvent(app.user,
+ app.submissionContext, true));
// recover attempts
app.recoverAppAttempts();
@@ -960,9 +958,8 @@ public class RMAppImpl implements RMApp, Recoverable {
RMAppTransition {
@Override
public void transition(RMAppImpl app, RMAppEvent event) {
- app.handler.handle(new AppAddedSchedulerEvent(app.applicationId,
- app.submissionContext.getQueue(), app.user,
- app.submissionContext.getReservationID()));
+ app.handler.handle(new AppAddedSchedulerEvent(app.user,
+ app.submissionContext, false));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index aad76fd..094f77d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -691,4 +692,13 @@ public abstract class AbstractYarnScheduler
}
return null;
}
+
+ @Override
+ public Priority checkAndGetApplicationPriority(Priority priorityFromContext,
+ String user, String queueName, ApplicationId applicationId)
+ throws YarnException {
+ // Dummy Implementation till Application Priority changes are done in
+ // specific scheduler.
+ return Priority.newInstance(0);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java
index 02003c1..8646381 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -110,4 +111,11 @@ public interface Queue {
* new resource asked
*/
public void decPendingResource(String nodeLabel, Resource resourceToDec);
+
+ /**
+ * Get the Default Application Priority for this queue
+ *
+ * @return default application priority
+ */
+ public Priority getDefaultApplicationPriority();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
index 2c788aa..519de98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@Private
@@ -28,10 +29,18 @@ public class SchedulerApplication<T extends SchedulerApplicationAttempt> {
private Queue queue;
private final String user;
private T currentAttempt;
+ private volatile Priority priority;
public SchedulerApplication(Queue queue, String user) {
this.queue = queue;
this.user = user;
+ this.priority = null;
+ }
+
+ public SchedulerApplication(Queue queue, String user, Priority priority) {
+ this.queue = queue;
+ this.user = user;
+ this.priority = priority;
}
public Queue getQueue() {
@@ -58,4 +67,17 @@ public class SchedulerApplication<T extends SchedulerApplicationAttempt> {
queue.getMetrics().finishApp(user, rmAppFinalState);
}
+ public Priority getPriority() {
+ return priority;
+ }
+
+ public void setPriority(Priority priority) {
+ this.priority = priority;
+
+ // Also set priority in current running attempt
+ if (null != currentAttempt) {
+ currentAttempt.setPriority(priority);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 475f2c7..cf543bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -97,7 +97,9 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private boolean unmanagedAM = true;
private boolean amRunning = false;
private LogAggregationContext logAggregationContext;
-
+
+ private Priority appPriority = null;
+
protected ResourceUsage attemptResourceUsage = new ResourceUsage();
private AtomicLong firstAllocationRequestSentTime = new AtomicLong(0);
private AtomicLong firstContainerAllocatedTime = new AtomicLong(0);
@@ -726,7 +728,16 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
public ResourceUsage getAppAttemptResourceUsage() {
return this.attemptResourceUsage;
}
-
+
+ @Override
+ public Priority getPriority() {
+ return appPriority;
+ }
+
+ public void setPriority(Priority appPriority) {
+ this.appPriority = appPriority;
+ }
+
@Override
public String getId() {
return getApplicationId().toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index b99b217..f629579 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -286,4 +287,23 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
* @return an EnumSet containing the resource types
*/
public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes();
+
+ /**
+ *
+ * Verify whether a submitted application priority is valid as per configured
+ * Queue
+ *
+ * @param priorityFromContext
+ * Submitted Application priority.
+ * @param user
+ * User who submitted the Application
+ * @param queueName
+ * Name of the Queue
+ * @param applicationId
+ * Application ID
+ * @return Updated Priority from scheduler
+ */
+ public Priority checkAndGetApplicationPriority(Priority priorityFromContext,
+ String user, String queueName, ApplicationId applicationId)
+ throws YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index cd5bd8d..7f8e164 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
@@ -574,4 +575,10 @@ public abstract class AbstractCSQueue implements CSQueue {
// sorry, you cannot access
return false;
}
+
+ @Override
+ public Priority getDefaultApplicationPriority() {
+ // TODO add dummy implementation
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 559dfc6..5a20f8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -159,6 +160,9 @@ public class CapacityScheduler extends
new Comparator<FiCaSchedulerApp>() {
@Override
public int compare(FiCaSchedulerApp a1, FiCaSchedulerApp a2) {
+ if (!a1.getPriority().equals(a2.getPriority())) {
+ return a1.getPriority().compareTo(a2.getPriority());
+ }
return a1.getApplicationId().compareTo(a2.getApplicationId());
}
};
@@ -226,6 +230,7 @@ public class CapacityScheduler extends
private RMNodeLabelsManager labelManager;
private SchedulerHealth schedulerHealth = new SchedulerHealth();
long lastNodeUpdateTime;
+ private Priority maxClusterLevelAppPriority;
/**
* EXPERT
*/
@@ -326,6 +331,9 @@ public class CapacityScheduler extends
if (scheduleAsynchronously) {
asyncSchedulerThread = new AsyncScheduleThread(this);
}
+ maxClusterLevelAppPriority = Priority.newInstance(yarnConf.getInt(
+ YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY,
+ YarnConfiguration.DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY));
LOG.info("Initialized CapacityScheduler with " +
"calculator=" + getResourceCalculator().getClass() + ", " +
@@ -692,7 +700,7 @@ public class CapacityScheduler extends
}
private synchronized void addApplication(ApplicationId applicationId,
- String queueName, String user, boolean isAppRecovering) {
+ String queueName, String user, boolean isAppRecovering, Priority priority) {
if (mappings != null && mappings.size() > 0) {
try {
@@ -761,7 +769,7 @@ public class CapacityScheduler extends
// update the metrics
queue.getMetrics().submitApp(user);
SchedulerApplication<FiCaSchedulerApp> application =
- new SchedulerApplication<FiCaSchedulerApp>(queue, user);
+ new SchedulerApplication<FiCaSchedulerApp>(queue, user, priority);
applications.put(applicationId, application);
LOG.info("Accepted application " + applicationId + " from user: " + user
+ ", in queue: " + queueName);
@@ -783,9 +791,9 @@ public class CapacityScheduler extends
applications.get(applicationAttemptId.getApplicationId());
CSQueue queue = (CSQueue) application.getQueue();
- FiCaSchedulerApp attempt =
- new FiCaSchedulerApp(applicationAttemptId, application.getUser(),
- queue, queue.getActiveUsersManager(), rmContext);
+ FiCaSchedulerApp attempt = new FiCaSchedulerApp(applicationAttemptId,
+ application.getUser(), queue, queue.getActiveUsersManager(), rmContext,
+ application.getPriority());
if (transferStateFromPreviousAttempt) {
attempt.transferStateFromPreviousAttempt(application
.getCurrentAppAttempt());
@@ -1307,7 +1315,8 @@ public class CapacityScheduler extends
addApplication(appAddedEvent.getApplicationId(),
queueName,
appAddedEvent.getUser(),
- appAddedEvent.getIsAppRecovering());
+ appAddedEvent.getIsAppRecovering(),
+ appAddedEvent.getApplicatonPriority());
}
}
break;
@@ -1833,4 +1842,56 @@ public class CapacityScheduler extends
private synchronized void setLastNodeUpdateTime(long time) {
this.lastNodeUpdateTime = time;
}
+
+ @Override
+ public Priority checkAndGetApplicationPriority(Priority priorityFromContext,
+ String user, String queueName, ApplicationId applicationId)
+ throws YarnException {
+ Priority appPriority = null;
+
+ // ToDo: Verify against priority ACLs
+
+ // Verify the scenario where priority is null from submissionContext.
+ if (null == priorityFromContext) {
+ // Get the default priority for the Queue. If Queue is non-existent, then
+ // use default priority
+ priorityFromContext = getDefaultPriorityForQueue(queueName);
+
+ LOG.info("Application '" + applicationId
+ + "' is submitted without priority "
+ + "hence considering default queue/cluster priority:"
+ + priorityFromContext.getPriority());
+ }
+
+ // Verify whether submitted priority is lesser than max priority
+ // in the cluster. If it is out of found, defining a max cap.
+ if (priorityFromContext.compareTo(getMaxClusterLevelAppPriority()) < 0) {
+ priorityFromContext = Priority
+ .newInstance(getMaxClusterLevelAppPriority().getPriority());
+ }
+
+ appPriority = priorityFromContext;
+
+ LOG.info("Priority '" + appPriority.getPriority()
+ + "' is acceptable in queue :" + queueName + "for application:"
+ + applicationId + "for the user: " + user);
+
+ return appPriority;
+ }
+
+ private Priority getDefaultPriorityForQueue(String queueName) {
+ Queue queue = getQueue(queueName);
+ if (null == queue) {
+ // Return with default application priority
+ return Priority.newInstance(CapacitySchedulerConfiguration
+ .DEFAULT_CONFIGURATION_APPLICATION_PRIORITY);
+ }
+
+ return Priority.newInstance(queue.getDefaultApplicationPriority()
+ .getPriority());
+ }
+
+ public Priority getMaxClusterLevelAppPriority() {
+ return maxClusterLevelAppPriority;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 563643c..be5e6dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -207,6 +207,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String QUEUE_PREEMPTION_DISABLED = "disable_preemption";
@Private
+ public static final String DEFAULT_APPLICATION_PRIORITY = "default-application-priority";
+
+ @Private
+ public static final Integer DEFAULT_CONFIGURATION_APPLICATION_PRIORITY = 0;
+
+ @Private
public static class QueueMapping {
public enum MappingType {
@@ -947,4 +953,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return configuredNodeLabels;
}
+
+ public Integer getDefaultApplicationPriorityConfPerQueue(String queue) {
+ Integer defaultPriority = getInt(getQueuePrefix(queue)
+ + DEFAULT_APPLICATION_PRIORITY,
+ DEFAULT_CONFIGURATION_APPLICATION_PRIORITY);
+ return defaultPriority;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 598f279..0ce4d68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -95,9 +95,11 @@ public class LeafQueue extends AbstractCSQueue {
private int nodeLocalityDelay;
- Map<ApplicationAttemptId, FiCaSchedulerApp> applicationAttemptMap =
+ Map<ApplicationAttemptId, FiCaSchedulerApp> applicationAttemptMap =
new HashMap<ApplicationAttemptId, FiCaSchedulerApp>();
-
+
+ private Priority defaultAppPriorityPerQueue;
+
Set<FiCaSchedulerApp> pendingApplications;
private float minimumAllocationFactor;
@@ -220,6 +222,9 @@ public class LeafQueue extends AbstractCSQueue {
}
}
+ defaultAppPriorityPerQueue = Priority.newInstance(conf
+ .getDefaultApplicationPriorityConfPerQueue(getQueuePath()));
+
LOG.info("Initializing " + queueName + "\n" +
"capacity = " + queueCapacities.getCapacity() +
" [= (float) configuredCapacity / 100 ]" + "\n" +
@@ -265,7 +270,8 @@ public class LeafQueue extends AbstractCSQueue {
"nodeLocalityDelay = " + nodeLocalityDelay + "\n" +
"reservationsContinueLooking = " +
reservationsContinueLooking + "\n" +
- "preemptionDisabled = " + getPreemptionDisabled() + "\n");
+ "preemptionDisabled = " + getPreemptionDisabled() + "\n" +
+ "defaultAppPriorityPerQueue = " + defaultAppPriorityPerQueue);
}
@Override
@@ -2060,7 +2066,12 @@ public class LeafQueue extends AbstractCSQueue {
);
this.orderingPolicy = orderingPolicy;
}
-
+
+ @Override
+ public Priority getDefaultApplicationPriority() {
+ return defaultAppPriorityPerQueue;
+ }
+
/*
* Holds shared values used by all applications in
* the queue to calculate headroom on demand
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 3085d93..dfeb30f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -72,6 +72,13 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId,
String user, Queue queue, ActiveUsersManager activeUsersManager,
RMContext rmContext) {
+ this(applicationAttemptId, user, queue, activeUsersManager, rmContext,
+ Priority.newInstance(0));
+ }
+
+ public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId,
+ String user, Queue queue, ActiveUsersManager activeUsersManager,
+ RMContext rmContext, Priority appPriority) {
super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
RMApp rmApp = rmContext.getRMApps().get(getApplicationId());
@@ -87,6 +94,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
}
setAMResource(amResource);
+ setPriority(appPriority);
}
synchronized public boolean containerCompleted(RMContainer rmContainer,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
index a54e4bf..89d2f66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationId;
public class AppAddedSchedulerEvent extends SchedulerEvent {
@@ -28,25 +30,35 @@ public class AppAddedSchedulerEvent extends SchedulerEvent {
private final String user;
private final ReservationId reservationID;
private final boolean isAppRecovering;
+ private final Priority appPriority;
- public AppAddedSchedulerEvent(
- ApplicationId applicationId, String queue, String user) {
- this(applicationId, queue, user, false, null);
+ public AppAddedSchedulerEvent(ApplicationId applicationId, String queue,
+ String user) {
+ this(applicationId, queue, user, false, null, Priority.newInstance(0));
}
public AppAddedSchedulerEvent(ApplicationId applicationId, String queue,
- String user, ReservationId reservationID) {
- this(applicationId, queue, user, false, reservationID);
+ String user, ReservationId reservationID, Priority appPriority) {
+ this(applicationId, queue, user, false, reservationID, appPriority);
+ }
+
+ public AppAddedSchedulerEvent(String user,
+ ApplicationSubmissionContext submissionContext, boolean isAppRecovering) {
+ this(submissionContext.getApplicationId(), submissionContext.getQueue(),
+ user, isAppRecovering, submissionContext.getReservationID(),
+ submissionContext.getPriority());
}
public AppAddedSchedulerEvent(ApplicationId applicationId, String queue,
- String user, boolean isAppRecovering, ReservationId reservationID) {
+ String user, boolean isAppRecovering, ReservationId reservationID,
+ Priority appPriority) {
super(SchedulerEventType.APP_ADDED);
this.applicationId = applicationId;
this.queue = queue;
this.user = user;
this.reservationID = reservationID;
this.isAppRecovering = isAppRecovering;
+ this.appPriority = appPriority;
}
public ApplicationId getApplicationId() {
@@ -68,4 +80,8 @@ public class AppAddedSchedulerEvent extends SchedulerEvent {
public ReservationId getReservationID() {
return reservationID;
}
+
+ public Priority getApplicatonPriority() {
+ return appPriority;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index e488c76..713bdca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -331,6 +331,12 @@ public abstract class FSQueue implements Queue, Schedulable {
public void decPendingResource(String nodeLabel, Resource resourceToDec) {
}
+ @Override
+ public Priority getDefaultApplicationPriority() {
+ // TODO add implementation for FSParentQueue
+ return null;
+ }
+
public boolean fitsInMaxShare(Resource additionalResource) {
Resource usagePlusAddition =
Resources.add(getResourceUsage(), additionalResource);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index e66c02c..6b77ceb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -210,6 +210,12 @@ public class FifoScheduler extends
@Override
public void decPendingResource(String nodeLabel, Resource resourceToDec) {
}
+
+ @Override
+ public Priority getDefaultApplicationPriority() {
+ // TODO add implementation for FIFO scheduler
+ return null;
+ }
};
public FifoScheduler() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
index b92b264..1045386 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy;
import java.util.*;
+
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
@@ -29,9 +30,13 @@ public class FifoComparator
implements Comparator<SchedulableEntity> {
@Override
- public int compare(SchedulableEntity r1, SchedulableEntity r2) {
- int res = r1.compareInputOrderTo(r2);
- return res;
+ public int compare(SchedulableEntity r1, SchedulableEntity r2) {
+ if (r1.getPriority() != null
+ && !r1.getPriority().equals(r2.getPriority())) {
+ return r1.getPriority().compareTo(r2.getPriority());
}
+ int res = r1.compareInputOrderTo(r2);
+ return res;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/SchedulableEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/SchedulableEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/SchedulableEntity.java
index 9b9d73d..2ccb1cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/SchedulableEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/SchedulableEntity.java
@@ -48,4 +48,9 @@ public interface SchedulableEntity {
*/
public ResourceUsage getSchedulingResourceUsage();
+ /**
+ * Get the priority of the application
+ */
+ public Priority getPriority();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index d068a94..5080355 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -289,6 +290,15 @@ public class MockRM extends ResourceManager {
return submitApp(masterMemory, false);
}
+ public RMApp submitApp(int masterMemory, Priority priority) throws Exception {
+ Resource resource = Resource.newInstance(masterMemory, 0);
+ return submitApp(resource, "", UserGroupInformation.getCurrentUser()
+ .getShortUserName(), null, false, null,
+ super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true,
+ false, false, null, 0, null, true, priority);
+ }
+
public RMApp submitApp(int masterMemory, boolean unmanaged)
throws Exception {
return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser()
@@ -327,7 +337,7 @@ public class MockRM extends ResourceManager {
return submitApp(resource, name, user, acls, false, queue,
super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null,
- true, false, false, null, 0, null, true);
+ true, false, false, null, 0, null, true, null);
}
public RMApp submitApp(int masterMemory, String name, String user,
@@ -370,18 +380,19 @@ public class MockRM extends ResourceManager {
resource.setMemory(masterMemory);
return submitApp(resource, name, user, acls, unmanaged, queue,
maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
- false, null, 0, null, true);
+ false, null, 0, null, true, Priority.newInstance(0));
}
public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval)
throws Exception {
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(masterMemory);
+ Priority priority = Priority.newInstance(0);
return submitApp(resource, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null,
super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
- false, null, attemptFailuresValidityInterval, null, true);
+ false, null, attemptFailuresValidityInterval, null, true, priority);
}
public RMApp submitApp(int masterMemory, String name, String user,
@@ -391,20 +402,22 @@ public class MockRM extends ResourceManager {
ApplicationId applicationId) throws Exception {
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(masterMemory);
+ Priority priority = Priority.newInstance(0);
return submitApp(resource, name, user, acls, unmanaged, queue,
maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
- isAppIdProvided, applicationId, 0, null, true);
+ isAppIdProvided, applicationId, 0, null, true, priority);
}
public RMApp submitApp(int masterMemory,
LogAggregationContext logAggregationContext) throws Exception {
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(masterMemory);
+ Priority priority = Priority.newInstance(0);
return submitApp(resource, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null,
super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
- false, null, 0, logAggregationContext, true);
+ false, null, 0, logAggregationContext, true, priority);
}
public RMApp submitApp(Resource capability, String name, String user,
@@ -412,7 +425,8 @@ public class MockRM extends ResourceManager {
int maxAppAttempts, Credentials ts, String appType,
boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
ApplicationId applicationId, long attemptFailuresValidityInterval,
- LogAggregationContext logAggregationContext, boolean cancelTokensWhenComplete)
+ LogAggregationContext logAggregationContext,
+ boolean cancelTokensWhenComplete, Priority priority)
throws Exception {
ApplicationId appId = isAppIdProvided ? applicationId : null;
ApplicationClientProtocol client = getClientRMService();
@@ -429,12 +443,15 @@ public class MockRM extends ResourceManager {
sub.setApplicationId(appId);
sub.setApplicationName(name);
sub.setMaxAppAttempts(maxAppAttempts);
- if(unmanaged) {
+ if (unmanaged) {
sub.setUnmanagedAM(true);
}
if (queue != null) {
sub.setQueue(queue);
}
+ if (priority != null) {
+ sub.setPriority(priority);
+ }
sub.setApplicationType(appType);
ContainerLaunchContext clc = Records
.newRecord(ContainerLaunchContext.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 3db8b7c..f073763 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -219,6 +219,7 @@ public class TestAppManager{
rmContext = mockRMContext(1, now - 10);
ResourceScheduler scheduler = mockResourceScheduler();
+ ((RMContextImpl)rmContext).setScheduler(scheduler);
Configuration conf = new Configuration();
ApplicationMasterService masterService =
new ApplicationMasterService(rmContext, scheduler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index 32743c9..b556335 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -1056,7 +1056,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation
.getCurrentUser().getShortUserName(), null, false, null,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true,
- false, null, 0, null, true);
+ false, null, 0, null, true, null);
MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);
am0.allocate("127.0.0.1", 1000, 2, new ArrayList<ContainerId>());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
index b3ac79b..d6f64bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
@@ -1025,6 +1025,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
when(app.getReservedContainers()).thenReturn(reservedContainers);
when(app.getApplicationAttemptId()).thenReturn(appAttemptId);
when(app.getApplicationId()).thenReturn(appId);
+ when(app.getPriority()).thenReturn(Priority.newInstance(0));
// add to LeafQueue
LeafQueue queue = (LeafQueue) nameToCSQueues.get(queueName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 484090d..1afebb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -155,7 +155,8 @@ public class TestApplicationLimits {
doReturn(applicationAttemptId). when(application).getApplicationAttemptId();
doReturn(user).when(application).getUser();
doReturn(amResource).when(application).getAMResource();
- when(application.compareInputOrderTo(any(FiCaSchedulerApp.class))).thenCallRealMethod();
+ doReturn(Priority.newInstance(0)).when(application).getPriority();
+ when(application.compareInputOrderTo(any(FiCaSchedulerApp.class))).thenCallRealMethod();
return application;
}
@@ -175,7 +176,7 @@ public class TestApplicationLimits {
ActiveUsersManager activeUsersManager = mock(ActiveUsersManager.class);
when(queue.getActiveUsersManager()).thenReturn(activeUsersManager);
-
+
assertEquals(Resource.newInstance(8 * GB, 1), queue.getAMResourceLimit());
assertEquals(Resource.newInstance(4 * GB, 1),
queue.getUserAMResourceLimit());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
new file mode 100644
index 0000000..80eff06
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -0,0 +1,345 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestApplicationPriority {
+ private static final Log LOG = LogFactory
+ .getLog(TestApplicationPriority.class);
+ private final int GB = 1024;
+
+ private YarnConfiguration conf;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ }
+
+ @Test
+ public void testApplicationOrderingWithPriority() throws Exception {
+
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ MockRM rm = new MockRM(conf);
+ rm.start();
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue q = (LeafQueue) cs.getQueue("default");
+ Assert.assertNotNull(q);
+
+ String host = "127.0.0.1";
+ RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(16 * GB), 1,
+ host);
+ cs.handle(new NodeAddedSchedulerEvent(node));
+
+ // add app 1 start
+ ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
+ ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(
+ appId1, 1);
+
+ RMAppAttemptMetrics attemptMetric1 = new RMAppAttemptMetrics(appAttemptId1,
+ rm.getRMContext());
+ RMAppImpl app1 = mock(RMAppImpl.class);
+ when(app1.getApplicationId()).thenReturn(appId1);
+ RMAppAttemptImpl attempt1 = mock(RMAppAttemptImpl.class);
+ when(attempt1.getAppAttemptId()).thenReturn(appAttemptId1);
+ when(attempt1.getRMAppAttemptMetrics()).thenReturn(attemptMetric1);
+ when(app1.getCurrentAppAttempt()).thenReturn(attempt1);
+
+ rm.getRMContext().getRMApps().put(appId1, app1);
+
+ SchedulerEvent addAppEvent1 = new AppAddedSchedulerEvent(appId1, "default",
+ "user", null, Priority.newInstance(5));
+ cs.handle(addAppEvent1);
+ SchedulerEvent addAttemptEvent1 = new AppAttemptAddedSchedulerEvent(
+ appAttemptId1, false);
+ cs.handle(addAttemptEvent1);
+ // add app1 end
+
+ // add app2 begin
+ ApplicationId appId2 = BuilderUtils.newApplicationId(100, 2);
+ ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(
+ appId2, 1);
+
+ RMAppAttemptMetrics attemptMetric2 = new RMAppAttemptMetrics(appAttemptId2,
+ rm.getRMContext());
+ RMAppImpl app2 = mock(RMAppImpl.class);
+ when(app2.getApplicationId()).thenReturn(appId2);
+ RMAppAttemptImpl attempt2 = mock(RMAppAttemptImpl.class);
+ when(attempt2.getAppAttemptId()).thenReturn(appAttemptId2);
+ when(attempt2.getRMAppAttemptMetrics()).thenReturn(attemptMetric2);
+ when(app2.getCurrentAppAttempt()).thenReturn(attempt2);
+
+ rm.getRMContext().getRMApps().put(appId2, app2);
+
+ SchedulerEvent addAppEvent2 = new AppAddedSchedulerEvent(appId2, "default",
+ "user", null, Priority.newInstance(8));
+ cs.handle(addAppEvent2);
+ SchedulerEvent addAttemptEvent2 = new AppAttemptAddedSchedulerEvent(
+ appAttemptId2, false);
+ cs.handle(addAttemptEvent2);
+ // add app end
+
+ // Now, the first assignment will be for app2 since app2 is of highest
+ // priority
+ assertEquals(q.getApplications().size(), 2);
+ assertEquals(q.getApplications().iterator().next()
+ .getApplicationAttemptId(), appAttemptId2);
+
+ rm.stop();
+ }
+
+ @Test
+ public void testApplicationPriorityAllocation() throws Exception {
+
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ // Set Max Application Priority as 10
+ conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
+ MockRM rm = new MockRM(conf);
+ rm.start();
+
+ Priority appPriority1 = Priority.newInstance(5);
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB);
+ RMApp app1 = rm.submitApp(1 * GB, appPriority1);
+
+ // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1
+ MockAM am1 = MockRM.launchAM(app1, rm, nm1);
+ am1.registerAppAttempt();
+
+ // add request for containers
+ am1.addRequests(new String[]{"127.0.0.1", "127.0.0.2"}, 2 * GB, 1, 7);
+ AllocateResponse alloc1Response = am1.schedule(); // send the request
+
+ // kick the scheduler, 7 containers will be allocated for App1
+ nm1.nodeHeartbeat(true);
+ while (alloc1Response.getAllocatedContainers().size() < 1) {
+ LOG.info("Waiting for containers to be created for app 1...");
+ Thread.sleep(100);
+ alloc1Response = am1.schedule();
+ }
+
+ List<Container> allocated1 = alloc1Response.getAllocatedContainers();
+ Assert.assertEquals(7, allocated1.size());
+ Assert.assertEquals(2 * GB, allocated1.get(0).getResource().getMemory());
+
+ // check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
+ SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
+ nm1.getNodeId());
+ Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemory());
+ Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemory());
+
+ // Submit the second app App2 with priority 8 (Higher than App1)
+ Priority appPriority2 = Priority.newInstance(8);
+ RMApp app2 = rm.submitApp(1 * GB, appPriority2);
+
+ // kick the scheduler, 1 GB which was free is given to AM of App2
+ nm1.nodeHeartbeat(true);
+ MockAM am2 = rm.sendAMLaunched(app2.getCurrentAppAttempt()
+ .getAppAttemptId());
+ am2.registerAppAttempt();
+
+ // check node report, 16 GB used and 0 GB available
+ report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
+ Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemory());
+ Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemory());
+
+ // get scheduler
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ // get scheduler app
+ FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications()
+ .get(app1.getApplicationId()).getCurrentAppAttempt();
+
+ // kill 2 containers to free up some space
+ int counter = 0;
+ for (Container c : allocated1) {
+ if (++counter > 2) {
+ break;
+ }
+ cs.killContainer(schedulerAppAttempt.getRMContainer(c.getId()));
+ }
+
+ // check node report, 12 GB used and 4 GB available
+ report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
+ Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemory());
+ Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemory());
+
+ // add request for containers App1
+ am1.addRequests(new String[]{"127.0.0.1", "127.0.0.2"}, 2 * GB, 1, 10);
+ am1.schedule(); // send the request for App1
+
+ // add request for containers App2
+ am2.addRequests(new String[]{"127.0.0.1", "127.0.0.2"}, 2 * GB, 1, 3);
+ AllocateResponse alloc1Response4 = am2.schedule(); // send the request
+
+ // kick the scheduler, since App2 priority is more than App1, it will get
+ // remaining cluster space.
+ nm1.nodeHeartbeat(true);
+ while (alloc1Response4.getAllocatedContainers().size() < 1) {
+ LOG.info("Waiting for containers to be created for app 2...");
+ Thread.sleep(100);
+ alloc1Response4 = am2.schedule();
+ }
+
+ // check node report, 16 GB used and 0 GB available
+ report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
+ Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemory());
+ Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemory());
+
+ rm.stop();
+ }
+
+ @Test
+ public void testPriorityWithPendingApplications() throws Exception {
+
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ // Set Max Application Priority as 10
+ conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
+ MockRM rm = new MockRM(conf);
+ rm.start();
+
+ Priority appPriority1 = Priority.newInstance(5);
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 8 * GB);
+ RMApp app1 = rm.submitApp(1 * GB, appPriority1);
+
+ // kick the scheduler, 1 GB given to AM1, remaining 7GB on nm1
+ MockAM am1 = MockRM.launchAM(app1, rm, nm1);
+ am1.registerAppAttempt();
+
+ // add request for containers
+ am1.addRequests(new String[]{"127.0.0.1", "127.0.0.2"}, 1 * GB, 1, 7);
+ AllocateResponse alloc1Response = am1.schedule(); // send the request
+
+ // kick the scheduler, 7 containers will be allocated for App1
+ nm1.nodeHeartbeat(true);
+ while (alloc1Response.getAllocatedContainers().size() < 1) {
+ LOG.info("Waiting for containers to be created for app 1...");
+ Thread.sleep(100);
+ alloc1Response = am1.schedule();
+ }
+
+ List<Container> allocated1 = alloc1Response.getAllocatedContainers();
+ Assert.assertEquals(7, allocated1.size());
+ Assert.assertEquals(1 * GB, allocated1.get(0).getResource().getMemory());
+
+ // check node report, 8 GB used (1 AM and 7 containers) and 0 GB available
+ SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
+ nm1.getNodeId());
+ Assert.assertEquals(8 * GB, report_nm1.getUsedResource().getMemory());
+ Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemory());
+
+ // Submit the second app App2 with priority 7
+ Priority appPriority2 = Priority.newInstance(7);
+ RMApp app2 = rm.submitApp(1 * GB, appPriority2);
+
+ // Submit the third app App3 with priority 8
+ Priority appPriority3 = Priority.newInstance(8);
+ RMApp app3 = rm.submitApp(1 * GB, appPriority3);
+
+ // Submit the second app App4 with priority 6
+ Priority appPriority4 = Priority.newInstance(6);
+ RMApp app4 = rm.submitApp(1 * GB, appPriority4);
+
+ // Only one app can run as AM resource limit restricts it. Kill app1,
+ // If app3 (highest priority among rest) gets active, it indicates that
+ // priority is working with pendingApplications.
+ rm.killApp(app1.getApplicationId());
+
+ // kick the scheduler, app3 (high among pending) gets free space
+ nm1.nodeHeartbeat(true);
+ MockAM am3 = rm.sendAMLaunched(app3.getCurrentAppAttempt()
+ .getAppAttemptId());
+ am3.registerAppAttempt();
+
+ // check node report, 1 GB used and 7 GB available
+ report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
+ Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemory());
+ Assert.assertEquals(7 * GB, report_nm1.getAvailableResource().getMemory());
+
+ rm.stop();
+ }
+
+ @Test
+ public void testMaxPriorityValidation() throws Exception {
+
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ // Set Max Application Priority as 10
+ conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
+ Priority maxPriority = Priority.newInstance(10);
+ MockRM rm = new MockRM(conf);
+ rm.start();
+
+ Priority appPriority1 = Priority.newInstance(15);
+ rm.registerNode("127.0.0.1:1234", 8 * GB);
+ RMApp app1 = rm.submitApp(1 * GB, appPriority1);
+
+ // Application submission should be successful and verify priority
+ Assert.assertEquals(app1.getApplicationSubmissionContext().getPriority(),
+ maxPriority);
+ rm.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index e8afab2..a8bbac3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -129,6 +129,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedule
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
@@ -898,13 +899,17 @@ public class TestCapacityScheduler {
ApplicationId id1 = ApplicationId.newInstance(1, 1);
ApplicationId id2 = ApplicationId.newInstance(1, 2);
ApplicationId id3 = ApplicationId.newInstance(2, 1);
+ Priority priority = Priority.newInstance(0);
//same clusterId
FiCaSchedulerApp app1 = Mockito.mock(FiCaSchedulerApp.class);
when(app1.getApplicationId()).thenReturn(id1);
+ when(app1.getPriority()).thenReturn(priority);
FiCaSchedulerApp app2 = Mockito.mock(FiCaSchedulerApp.class);
when(app2.getApplicationId()).thenReturn(id2);
+ when(app2.getPriority()).thenReturn(priority);
FiCaSchedulerApp app3 = Mockito.mock(FiCaSchedulerApp.class);
when(app3.getApplicationId()).thenReturn(id3);
+ when(app3.getPriority()).thenReturn(priority);
assertTrue(appComparator.compare(app1, app2) < 0);
//different clusterId
assertTrue(appComparator.compare(app1, app3) < 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/MockSchedulableEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/MockSchedulableEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/MockSchedulableEntity.java
index fe8c455..bf4c98a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/MockSchedulableEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/MockSchedulableEntity.java
@@ -31,7 +31,8 @@ public class MockSchedulableEntity implements SchedulableEntity {
private String id;
private long serial = 0;
-
+ private Priority priority;
+
public MockSchedulableEntity() { }
public void setId(String id) {
@@ -74,5 +75,13 @@ public class MockSchedulableEntity implements SchedulableEntity {
}
return 1;//let other types go before this, if any
}
-
+
+ @Override
+ public Priority getPriority() {
+ return priority;
+ }
+
+ public void setApplicationPriority(Priority priority) {
+ this.priority = priority;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
index 49c7bf9..d85e928 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
@@ -1048,13 +1048,13 @@ public class TestDelegationTokenRenewer {
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(200);
RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2,
- credentials, null, true, false, false, null, 0, null, false);
+ credentials, null, true, false, false, null, 0, null, false, null);
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
// submit app2 with the same token, set cancelTokenWhenComplete to true;
RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2,
- credentials, null, true, false, false, null, 0, null, true);
+ credentials, null, true, false, false, null, 0, null, true, null);
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2);
@@ -1114,7 +1114,7 @@ public class TestDelegationTokenRenewer {
resource.setMemory(200);
RMApp app1 =
rm.submitApp(resource, "name", "user", null, false, null, 2, credentials,
- null, true, false, false, null, 0, null, true);
+ null, true, false, false, null, 0, null, true, null);
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
@@ -1122,7 +1122,7 @@ public class TestDelegationTokenRenewer {
Assert.assertNotNull(dttr);
Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2,
- credentials, null, true, false, false, null, 0, null, true);
+ credentials, null, true, false, false, null, 0, null, true, null);
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
@@ -1139,7 +1139,7 @@ public class TestDelegationTokenRenewer {
Assert.assertFalse(Renewer.cancelled);
RMApp app3 = rm.submitApp(resource, "name", "user", null, false, null, 2,
- credentials, null, true, false, false, null, 0, null, true);
+ credentials, null, true, false, false, null, 0, null, true, null);
MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm1);
rm.waitForState(app3.getApplicationId(), RMAppState.RUNNING);
Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ca541/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index 8e5e601..de4d116 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -759,10 +759,10 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo();
appInfo.setApplicationId(appId);
appInfo.setApplicationName(appName);
- appInfo.setPriority(3);
appInfo.setMaxAppAttempts(2);
appInfo.setQueue(queueName);
appInfo.setApplicationType(appType);
+ appInfo.setPriority(0);
HashMap<String, LocalResourceInfo> lr = new HashMap<>();
LocalResourceInfo y = new LocalResourceInfo();
y.setUrl(new URI("http://www.test.com/file.txt"));