You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2014/09/23 01:47:50 UTC
[01/50] [abbrv] git commit: KMS: Support for multiple Kerberos
principals. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/YARN-1051 f00e7af8a -> e82461be5
KMS: Support for multiple Kerberos principals. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fad4cd85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fad4cd85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fad4cd85
Branch: refs/heads/YARN-1051
Commit: fad4cd85b313a1d2378adcf03cad67e946a12cd5
Parents: 52945a3
Author: Alejandro Abdelnur <tu...@apache.org>
Authored: Thu Sep 18 16:03:38 2014 -0700
Committer: Alejandro Abdelnur <tu...@apache.org>
Committed: Thu Sep 18 16:03:38 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../crypto/key/kms/KMSClientProvider.java | 3 +++
.../hadoop-kms/src/site/apt/index.apt.vm | 26 +++++++++++++++++++-
3 files changed, 30 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4cd85/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2e2d569..f21771b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -834,6 +834,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11105. MetricsSystemImpl could leak memory in registered callbacks.
(Chuan Liu via cnauroth)
+ KMS: Support for multiple Kerberos principals. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4cd85/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 899b6c4..a97463a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,6 +45,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -400,6 +401,8 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
+ } catch (UndeclaredThrowableException ex) {
+ throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4cd85/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index b2755a1..cf7a557 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,7 +602,31 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- TBD
+ When KMS instances are behind a load-balancer or VIP, clients will use the
+ hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
+ URL is used to construct the Kerberos service name of the server,
+ <<<HTTP/#HOSTNAME#>>>. This means that all KMS instances must have a Kerberos
+ service name with the load-balancer or VIP hostname.
+
+ In order to be able to access directly a specific KMS instance, the KMS
+ instance must also have Keberos service name with its own hostname. This is
+ required for monitoring and admin purposes.
+
+ Both Kerberos service principal credentials (for the load-balancer/VIP
+ hostname and for the actual KMS instance hostname) must be in the keytab file
+ configured for authentication. And the principal name specified in the
+ configuration must be '*'. For example:
+
++---+
+ <property>
+ <name>hadoop.kms.authentication.kerberos.principal</name>
+ <value>*</value>
+ </property>
++---+
+
+ <<NOTE:>> If using HTTPS, the SSL certificate used by the KMS instance must
+ be configured to support multiple hostnames (see Java 7
+ <<<keytool>> SAN extension support for details on how to do this).
*** HTTP Authentication Signature
[19/50] [abbrv] git commit: HADOOP-11112. TestKMSWithZK does not use
KEY_PROVIDER_URI. (tucu via wang)
Posted by cu...@apache.org.
HADOOP-11112. TestKMSWithZK does not use KEY_PROVIDER_URI. (tucu via wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6ceef90
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6ceef90
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6ceef90
Branch: refs/heads/YARN-1051
Commit: b6ceef90e52c05fd95bfb40bff61fa7e1802787f
Parents: 9e35571
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Sep 19 17:42:00 2014 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Sep 19 17:42:00 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../org/apache/hadoop/crypto/key/kms/server/TestKMSWithZK.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ceef90/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0603304..70579c3 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -591,6 +591,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10833. Remove unused cache in UserProvider. (Benoy Antony)
+ HADOOP-11112. TestKMSWithZK does not use KEY_PROVIDER_URI. (tucu via wang)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ceef90/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSWithZK.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSWithZK.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSWithZK.java
index 59b0002..3a02a0a 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSWithZK.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSWithZK.java
@@ -76,7 +76,7 @@ public class TestKMSWithZK {
protected Configuration createBaseKMSConf(File keyStoreDir) throws Exception {
Configuration conf = new Configuration(false);
- conf.set("hadoop.security.key.provider.path",
+ conf.set(KMSConfiguration.KEY_PROVIDER_URI,
"jceks://file@" + new Path(keyStoreDir.getAbsolutePath(),
"kms.keystore").toUri());
conf.set("hadoop.kms.authentication.type", "simple");
[06/50] [abbrv] git commit: HDFS-6970. Move startFile EDEK retries to
the DFSClient. (wang)
Posted by cu...@apache.org.
HDFS-6970. Move startFile EDEK retries to the DFSClient. (wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20a076ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20a076ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20a076ba
Branch: refs/heads/YARN-1051
Commit: 20a076bafce548298729bab4fb81d12f829e8f7e
Parents: fe38d2e
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Sep 18 17:35:24 2014 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Sep 18 17:35:24 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 64 +++++++---
.../hdfs/server/namenode/FSNamesystem.java | 126 ++++++++-----------
.../namenode/RetryStartFileException.java | 17 ++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 2 +-
5 files changed, 121 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a076ba/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c99207b..c7339f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -562,6 +562,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6727. Refresh data volumes on DataNode based on configuration changes
(Lei Xu via cmccabe)
+ HDFS-6970. Move startFile EDEK retries to the DFSClient. (wang)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a076ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index d368f4e..94a1ddc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.fs.CanSetDropBehind;
@@ -76,6 +77,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
+import org.apache.hadoop.hdfs.server.namenode.RetryStartFileException;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.IOUtils;
@@ -126,6 +128,13 @@ import com.google.common.cache.RemovalNotification;
public class DFSOutputStream extends FSOutputSummer
implements Syncable, CanSetDropBehind {
private static final int MAX_PACKETS = 80; // each packet 64K, total 5MB
+ /**
+ * Number of times to retry creating a file when there are transient
+ * errors (typically related to encryption zones and KeyProvider operations).
+ */
+ @VisibleForTesting
+ public static final int CREATE_RETRY_COUNT = 10;
+
private final DFSClient dfsClient;
private final long dfsclientSlowLogThresholdMs;
private Socket s;
@@ -1648,23 +1657,46 @@ public class DFSOutputStream extends FSOutputSummer
short replication, long blockSize, Progressable progress, int buffersize,
DataChecksum checksum, String[] favoredNodes,
List<CipherSuite> cipherSuites) throws IOException {
- final HdfsFileStatus stat;
- try {
- stat = dfsClient.namenode.create(src, masked, dfsClient.clientName,
- new EnumSetWritable<CreateFlag>(flag), createParent, replication,
- blockSize, cipherSuites);
- } catch(RemoteException re) {
- throw re.unwrapRemoteException(AccessControlException.class,
- DSQuotaExceededException.class,
- FileAlreadyExistsException.class,
- FileNotFoundException.class,
- ParentNotDirectoryException.class,
- NSQuotaExceededException.class,
- SafeModeException.class,
- UnresolvedPathException.class,
- SnapshotAccessControlException.class,
- UnknownCipherSuiteException.class);
+ HdfsFileStatus stat = null;
+
+ // Retry the create if we get a RetryStartFileException up to a maximum
+ // number of times
+ boolean shouldRetry = true;
+ int retryCount = CREATE_RETRY_COUNT;
+ while (shouldRetry) {
+ shouldRetry = false;
+ try {
+ stat = dfsClient.namenode.create(src, masked, dfsClient.clientName,
+ new EnumSetWritable<CreateFlag>(flag), createParent, replication,
+ blockSize, cipherSuites);
+ break;
+ } catch (RemoteException re) {
+ IOException e = re.unwrapRemoteException(
+ AccessControlException.class,
+ DSQuotaExceededException.class,
+ FileAlreadyExistsException.class,
+ FileNotFoundException.class,
+ ParentNotDirectoryException.class,
+ NSQuotaExceededException.class,
+ RetryStartFileException.class,
+ SafeModeException.class,
+ UnresolvedPathException.class,
+ SnapshotAccessControlException.class,
+ UnknownCipherSuiteException.class);
+ if (e instanceof RetryStartFileException) {
+ if (retryCount > 0) {
+ shouldRetry = true;
+ retryCount--;
+ } else {
+ throw new IOException("Too many retries because of encryption" +
+ " zone operations", e);
+ }
+ } else {
+ throw e;
+ }
+ }
}
+ Preconditions.checkNotNull(stat, "HdfsFileStatus should not be null!");
final DFSOutputStream out = new DFSOutputStream(dfsClient, src, stat,
flag, progress, checksum, favoredNodes);
out.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a076ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index fa7f34f..d0a1af6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2490,84 +2490,66 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
waitForLoadingFSImage();
- /*
- * We want to avoid holding any locks while doing KeyProvider operations,
- * since they can be very slow. Since the path can
- * flip flop between being in an encryption zone and not in the meantime,
- * we need to recheck the preconditions and redo KeyProvider operations
- * in some situations.
- *
- * A special RetryStartFileException is used to indicate that we should
- * retry creation of a FileEncryptionInfo.
+ /**
+ * If the file is in an encryption zone, we optimistically create an
+ * EDEK for the file by calling out to the configured KeyProvider.
+ * Since this typically involves doing an RPC, we take the readLock
+ * initially, then drop it to do the RPC.
+ *
+ * Since the path can flip-flop between being in an encryption zone and not
+ * in the meantime, we need to recheck the preconditions when we retake the
+ * lock to do the create. If the preconditions are not met, we throw a
+ * special RetryStartFileException to ask the DFSClient to try the create
+ * again later.
*/
- BlocksMapUpdateInfo toRemoveBlocks = null;
+ CipherSuite suite = null;
+ String ezKeyName = null;
+ readLock();
try {
- boolean shouldContinue = true;
- int iters = 0;
- while (shouldContinue) {
- skipSync = false;
- if (iters >= 10) {
- throw new IOException("Too many retries because of encryption zone " +
- "operations, something might be broken!");
- }
- shouldContinue = false;
- iters++;
-
- // Optimistically determine CipherSuite and ezKeyName if the path is
- // currently within an encryption zone
- CipherSuite suite = null;
- String ezKeyName = null;
- readLock();
- try {
- src = resolvePath(src, pathComponents);
- INodesInPath iip = dir.getINodesInPath4Write(src);
- // Nothing to do if the path is not within an EZ
- if (dir.isInAnEZ(iip)) {
- suite = chooseCipherSuite(iip, cipherSuites);
- if (suite != null) {
- Preconditions.checkArgument(!suite.equals(CipherSuite.UNKNOWN),
- "Chose an UNKNOWN CipherSuite!");
- }
- ezKeyName = dir.getKeyName(iip);
- Preconditions.checkState(ezKeyName != null);
- }
- } finally {
- readUnlock();
+ src = resolvePath(src, pathComponents);
+ INodesInPath iip = dir.getINodesInPath4Write(src);
+ // Nothing to do if the path is not within an EZ
+ if (dir.isInAnEZ(iip)) {
+ suite = chooseCipherSuite(iip, cipherSuites);
+ if (suite != null) {
+ Preconditions.checkArgument(!suite.equals(CipherSuite.UNKNOWN),
+ "Chose an UNKNOWN CipherSuite!");
}
+ ezKeyName = dir.getKeyName(iip);
+ Preconditions.checkState(ezKeyName != null);
+ }
+ } finally {
+ readUnlock();
+ }
- Preconditions.checkState(
- (suite == null && ezKeyName == null) ||
+ Preconditions.checkState(
+ (suite == null && ezKeyName == null) ||
(suite != null && ezKeyName != null),
- "Both suite and ezKeyName should both be null or not null");
- // Generate EDEK if necessary while not holding the lock
- EncryptedKeyVersion edek =
- generateEncryptedDataEncryptionKey(ezKeyName);
- EncryptionFaultInjector.getInstance().startFileAfterGenerateKey();
- // Try to create the file with the computed cipher suite and EDEK
- writeLock();
- try {
- checkOperation(OperationCategory.WRITE);
- checkNameNodeSafeMode("Cannot create file" + src);
- src = resolvePath(src, pathComponents);
- toRemoveBlocks = startFileInternal(pc, src, permissions, holder,
- clientMachine, create, overwrite, createParent, replication,
- blockSize, suite, edek, logRetryCache);
- stat = dir.getFileInfo(src, false,
- FSDirectory.isReservedRawName(srcArg), false);
- } catch (StandbyException se) {
- skipSync = true;
- throw se;
- } catch (RetryStartFileException e) {
- shouldContinue = true;
- if (LOG.isTraceEnabled()) {
- LOG.trace("Preconditions failed, retrying creation of " +
- "FileEncryptionInfo", e);
- }
- } finally {
- writeUnlock();
- }
- }
+ "Both suite and ezKeyName should both be null or not null");
+
+ // Generate EDEK if necessary while not holding the lock
+ EncryptedKeyVersion edek =
+ generateEncryptedDataEncryptionKey(ezKeyName);
+ EncryptionFaultInjector.getInstance().startFileAfterGenerateKey();
+
+ // Proceed with the create, using the computed cipher suite and
+ // generated EDEK
+ BlocksMapUpdateInfo toRemoveBlocks = null;
+ writeLock();
+ try {
+ checkOperation(OperationCategory.WRITE);
+ checkNameNodeSafeMode("Cannot create file" + src);
+ src = resolvePath(src, pathComponents);
+ toRemoveBlocks = startFileInternal(pc, src, permissions, holder,
+ clientMachine, create, overwrite, createParent, replication,
+ blockSize, suite, edek, logRetryCache);
+ stat = dir.getFileInfo(src, false,
+ FSDirectory.isReservedRawName(srcArg), false);
+ } catch (StandbyException se) {
+ skipSync = true;
+ throw se;
} finally {
+ writeUnlock();
// There might be transactions logged while trying to recover the lease.
// They need to be sync'ed even when an exception was thrown.
if (!skipSync) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a076ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java
index a5758a7..0bdd2a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java
@@ -17,5 +17,20 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-public class RetryStartFileException extends Exception {
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class RetryStartFileException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ public RetryStartFileException() {
+ super("Preconditions for creating a file failed because of a " +
+ "transient error, retry create later.");
+ }
+
+ public RetryStartFileException(String s) {
+ super(s);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a076ba/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index ff28200..52ca942 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -940,7 +940,7 @@ public class TestEncryptionZones {
Future<?> future = executor.submit(new CreateFileTask(fsWrapper, file));
// Flip-flop between two EZs to repeatedly fail
- for (int i=0; i<10; i++) {
+ for (int i=0; i<DFSOutputStream.CREATE_RETRY_COUNT+1; i++) {
injector.ready.await();
fsWrapper.delete(zone1, true);
fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true);
[31/50] [abbrv] git commit: MAPREDUCE-6095. Enable DistributedCache
for uber-mode Jobs. Contributed by Gera Shegalov
Posted by cu...@apache.org.
MAPREDUCE-6095. Enable DistributedCache for uber-mode Jobs. Contributed by Gera Shegalov
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7039b98e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7039b98e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7039b98e
Branch: refs/heads/YARN-1051
Commit: 7039b98e1c459e9e0d8caa28cdaa2868e2bde2eb
Parents: 9721e2c
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Sep 22 15:20:59 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Sep 22 15:20:59 2014 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../org/apache/hadoop/mapred/YarnChild.java | 63 +-------------------
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 1 +
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 61 +++++++++++++++++--
.../apache/hadoop/mapreduce/v2/TestUberAM.java | 7 ---
5 files changed, 62 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7039b98e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index eb4c251..815b3fb 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -360,6 +360,9 @@ Release 2.6.0 - UNRELEASED
ApplicationNotFoundException if the job rolled off the RM view (Sangjin
Lee via jlowe)
+ MAPREDUCE-6095. Enable DistributedCache for uber-mode Jobs (Gera Shegalov
+ via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7039b98e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
index 4ba1991..92bbc4a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
@@ -23,15 +23,11 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
-import java.net.URI;
import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
@@ -43,7 +39,6 @@ import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.counters.Limits;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
@@ -307,7 +302,7 @@ class YarnChild {
task.localizeConfiguration(job);
// Set up the DistributedCache related configs
- setupDistributedCacheConfig(job);
+ MRApps.setupDistributedCacheLocal(job);
// Overwrite the localized task jobconf which is linked to in the current
// work-dir.
@@ -317,62 +312,6 @@ class YarnChild {
task.setConf(job);
}
- /**
- * Set up the DistributedCache related configs to make
- * {@link DistributedCache#getLocalCacheFiles(Configuration)}
- * and
- * {@link DistributedCache#getLocalCacheArchives(Configuration)}
- * working.
- * @param job
- * @throws IOException
- */
- private static void setupDistributedCacheConfig(final JobConf job)
- throws IOException {
-
- String localWorkDir = System.getenv("PWD");
- // ^ ^ all symlinks are created in the current work-dir
-
- // Update the configuration object with localized archives.
- URI[] cacheArchives = DistributedCache.getCacheArchives(job);
- if (cacheArchives != null) {
- List<String> localArchives = new ArrayList<String>();
- for (int i = 0; i < cacheArchives.length; ++i) {
- URI u = cacheArchives[i];
- Path p = new Path(u);
- Path name =
- new Path((null == u.getFragment()) ? p.getName()
- : u.getFragment());
- String linkName = name.toUri().getPath();
- localArchives.add(new Path(localWorkDir, linkName).toUri().getPath());
- }
- if (!localArchives.isEmpty()) {
- job.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils
- .arrayToString(localArchives.toArray(new String[localArchives
- .size()])));
- }
- }
-
- // Update the configuration object with localized files.
- URI[] cacheFiles = DistributedCache.getCacheFiles(job);
- if (cacheFiles != null) {
- List<String> localFiles = new ArrayList<String>();
- for (int i = 0; i < cacheFiles.length; ++i) {
- URI u = cacheFiles[i];
- Path p = new Path(u);
- Path name =
- new Path((null == u.getFragment()) ? p.getName()
- : u.getFragment());
- String linkName = name.toUri().getPath();
- localFiles.add(new Path(localWorkDir, linkName).toUri().getPath());
- }
- if (!localFiles.isEmpty()) {
- job.set(MRJobConfig.CACHE_LOCALFILES,
- StringUtils.arrayToString(localFiles
- .toArray(new String[localFiles.size()])));
- }
- }
- }
-
private static final FsPermission urw_gr =
FsPermission.createImmutable((short) 0640);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7039b98e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 59e7249..1cf8b29 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -826,6 +826,7 @@ public class MRAppMaster extends CompositeService {
@Override
protected void serviceStart() throws Exception {
if (job.isUber()) {
+ MRApps.setupDistributedCacheLocal(getConfig());
this.containerAllocator = new LocalContainerAllocator(
this.clientService, this.context, nmHost, nmPort, nmHttpPort
, containerID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7039b98e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
index 3bd8414..113b445 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
@@ -26,12 +26,11 @@ import java.net.URISyntaxException;
import java.security.AccessController;
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
@@ -58,8 +57,6 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.util.ApplicationClassLoader;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.ContainerLogAppender;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -469,6 +466,62 @@ public class MRApps extends Apps {
DistributedCache.getFileVisibilities(conf));
}
+ /**
+ * Set up the DistributedCache related configs to make
+ * {@link DistributedCache#getLocalCacheFiles(Configuration)}
+ * and
+ * {@link DistributedCache#getLocalCacheArchives(Configuration)}
+ * working.
+ * @param conf
+ * @throws java.io.IOException
+ */
+ public static void setupDistributedCacheLocal(Configuration conf)
+ throws IOException {
+
+ String localWorkDir = System.getenv("PWD");
+ // ^ ^ all symlinks are created in the current work-dir
+
+ // Update the configuration object with localized archives.
+ URI[] cacheArchives = DistributedCache.getCacheArchives(conf);
+ if (cacheArchives != null) {
+ List<String> localArchives = new ArrayList<String>();
+ for (int i = 0; i < cacheArchives.length; ++i) {
+ URI u = cacheArchives[i];
+ Path p = new Path(u);
+ Path name =
+ new Path((null == u.getFragment()) ? p.getName()
+ : u.getFragment());
+ String linkName = name.toUri().getPath();
+ localArchives.add(new Path(localWorkDir, linkName).toUri().getPath());
+ }
+ if (!localArchives.isEmpty()) {
+ conf.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils
+ .arrayToString(localArchives.toArray(new String[localArchives
+ .size()])));
+ }
+ }
+
+ // Update the configuration object with localized files.
+ URI[] cacheFiles = DistributedCache.getCacheFiles(conf);
+ if (cacheFiles != null) {
+ List<String> localFiles = new ArrayList<String>();
+ for (int i = 0; i < cacheFiles.length; ++i) {
+ URI u = cacheFiles[i];
+ Path p = new Path(u);
+ Path name =
+ new Path((null == u.getFragment()) ? p.getName()
+ : u.getFragment());
+ String linkName = name.toUri().getPath();
+ localFiles.add(new Path(localWorkDir, linkName).toUri().getPath());
+ }
+ if (!localFiles.isEmpty()) {
+ conf.set(MRJobConfig.CACHE_LOCALFILES,
+ StringUtils.arrayToString(localFiles
+ .toArray(new String[localFiles.size()])));
+ }
+ }
+ }
+
private static String getResourceDescription(LocalResourceType type) {
if(type == LocalResourceType.ARCHIVE || type == LocalResourceType.PATTERN) {
return "cache archive (" + MRJobConfig.CACHE_ARCHIVES + ") ";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7039b98e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java
index 32199e5..e89a919 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java
@@ -191,11 +191,4 @@ public class TestUberAM extends TestMRJobs {
throws IOException, InterruptedException, ClassNotFoundException {
super.testSleepJobWithSecurityOn();
}
-
- // Add a test for distcache when uber mode is enabled. TODO
- @Override
- @Test
- public void testDistributedCache() throws Exception {
- //
- }
}
[13/50] [abbrv] git commit: YARN-2565. Fixed RM to not use
FileSystemApplicationHistoryStore unless explicitly set. Contributed by
Zhijie Shen
Posted by cu...@apache.org.
YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless explicitly set. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/444acf8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/444acf8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/444acf8e
Branch: refs/heads/YARN-1051
Commit: 444acf8ea795e4bc782f1ce3b5ef7a1a47d1d27d
Parents: 25fd69a
Author: Jian He <ji...@apache.org>
Authored: Fri Sep 19 11:26:29 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Fri Sep 19 11:26:29 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../ApplicationHistoryServer.java | 4 +-
.../ahs/RMApplicationHistoryWriter.java | 49 +++++++++++---------
.../ahs/TestRMApplicationHistoryWriter.java | 18 +++++++
4 files changed, 50 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8f717ae..7175b14 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -400,6 +400,9 @@ Release 2.6.0 - UNRELEASED
YARN-2568. Fixed the potential test failures due to race conditions when RM
work-preserving recovery is enabled. (Jian He via zjshen)
+ YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless
+ explicitly set. (Zhijie Shen via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index 204e118..f52ab07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -182,7 +182,9 @@ public class ApplicationHistoryServer extends CompositeService {
// APPLICATION_HISTORY_STORE is neither null nor empty, it means that the
// user has enabled it explicitly.
if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null ||
- conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0) {
+ conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0 ||
+ conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).equals(
+ NullApplicationHistoryStore.class.getName())) {
return new ApplicationHistoryManagerOnTimelineStore(
timelineDataManager, aclsManager);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
index 58d2e3d..bd328ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
@@ -52,6 +52,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* <p>
* {@link ResourceManager} uses this class to write the information of
@@ -71,8 +73,10 @@ public class RMApplicationHistoryWriter extends CompositeService {
.getLog(RMApplicationHistoryWriter.class);
private Dispatcher dispatcher;
- private ApplicationHistoryWriter writer;
- private boolean historyServiceEnabled;
+ @VisibleForTesting
+ ApplicationHistoryWriter writer;
+ @VisibleForTesting
+ boolean historyServiceEnabled;
public RMApplicationHistoryWriter() {
super(RMApplicationHistoryWriter.class.getName());
@@ -80,13 +84,18 @@ public class RMApplicationHistoryWriter extends CompositeService {
@Override
protected synchronized void serviceInit(Configuration conf) throws Exception {
-
historyServiceEnabled =
conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED);
+ if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null ||
+ conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0 ||
+ conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).equals(
+ NullApplicationHistoryStore.class.getName())) {
+ historyServiceEnabled = false;
+ }
- // Only create the services when the history service is enabled, preventing
- // wasting the system resources.
+ // Only create the services when the history service is enabled and not
+ // using the null store, preventing wasting the system resources.
if (historyServiceEnabled) {
writer = createApplicationHistoryStore(conf);
addIfService(writer);
@@ -112,25 +121,19 @@ public class RMApplicationHistoryWriter extends CompositeService {
protected ApplicationHistoryStore createApplicationHistoryStore(
Configuration conf) {
- // If the history writer is not enabled, a dummy store will be used to
- // write nothing
- if (historyServiceEnabled) {
- try {
- Class<? extends ApplicationHistoryStore> storeClass =
- conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
- FileSystemApplicationHistoryStore.class,
+ try {
+ Class<? extends ApplicationHistoryStore> storeClass =
+ conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
+ NullApplicationHistoryStore.class,
ApplicationHistoryStore.class);
- return storeClass.newInstance();
- } catch (Exception e) {
- String msg =
- "Could not instantiate ApplicationHistoryWriter: "
- + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE,
- FileSystemApplicationHistoryStore.class.getName());
- LOG.error(msg, e);
- throw new YarnRuntimeException(msg, e);
- }
- } else {
- return new NullApplicationHistoryStore();
+ return storeClass.newInstance();
+ } catch (Exception e) {
+ String msg =
+ "Could not instantiate ApplicationHistoryWriter: "
+ + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE,
+ NullApplicationHistoryStore.class.getName());
+ LOG.error(msg, e);
+ throw new YarnRuntimeException(msg, e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
index e83a6b9..1c90ad2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
@@ -78,6 +78,8 @@ public class TestRMApplicationHistoryWriter {
store = new MemoryApplicationHistoryStore();
Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
+ conf.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
+ MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
writer = new RMApplicationHistoryWriter() {
@Override
@@ -175,6 +177,22 @@ public class TestRMApplicationHistoryWriter {
}
@Test
+ public void testDefaultStoreSetup() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
+ RMApplicationHistoryWriter writer = new RMApplicationHistoryWriter();
+ writer.init(conf);
+ writer.start();
+ try {
+ Assert.assertFalse(writer.historyServiceEnabled);
+ Assert.assertNull(writer.writer);
+ } finally {
+ writer.stop();
+ writer.close();
+ }
+ }
+
+ @Test
public void testWriteApplication() throws Exception {
RMApp app = createRMApp(ApplicationId.newInstance(0, 1));
[35/50] [abbrv] git commit: YARN-1372. Ensure all completed
containers are reported to the AMs across RM restart. Contributed by Anubhav
Dhoot
Posted by cu...@apache.org.
YARN-1372. Ensure all completed containers are reported to the AMs across RM restart. Contributed by Anubhav Dhoot
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a641496
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a641496
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a641496
Branch: refs/heads/YARN-1051
Commit: 0a641496c706fc175e7bf66d69ebf71c7d078e84
Parents: 376233c
Author: Jian He <ji...@apache.org>
Authored: Mon Sep 22 10:30:53 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 22 10:30:53 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../protocolrecords/NodeHeartbeatResponse.java | 5 +
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 67 ++++++
.../yarn_server_common_service_protos.proto | 1 +
.../yarn/server/nodemanager/NodeManager.java | 2 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 80 ++++---
.../nodemanager/TestNodeManagerResync.java | 24 +-
.../nodemanager/TestNodeStatusUpdater.java | 234 ++++++++++++++-----
.../resourcemanager/ResourceTrackerService.java | 7 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 12 +-
.../rmapp/attempt/RMAppAttempt.java | 23 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 133 +++++++++--
.../RMAppAttemptContainerFinishedEvent.java | 8 +-
.../rmcontainer/RMContainerImpl.java | 13 +-
.../resourcemanager/rmnode/RMNodeEventType.java | 3 +
...RMNodeFinishedContainersPulledByAMEvent.java | 41 ++++
.../resourcemanager/rmnode/RMNodeImpl.java | 48 +++-
.../TestResourceTrackerService.java | 8 +-
.../applicationsmanager/TestAMRestart.java | 24 +-
.../attempt/TestRMAppAttemptTransitions.java | 143 ++++++++++--
.../security/TestAMRMTokens.java | 2 +-
21 files changed, 701 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5c9cafd..7e5d0f9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -238,6 +238,9 @@ Release 2.6.0 - UNRELEASED
YARN-2001. Added a time threshold for RM to wait before starting container
allocations after restart/failover. (Jian He via vinodkv)
+ YARN-1372. Ensure all completed containers are reported to the AMs across
+ RM restart. (Anubhav Dhoot via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
index 38dfa58..9887acc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
@@ -30,6 +30,7 @@ public interface NodeHeartbeatResponse {
NodeAction getNodeAction();
List<ContainerId> getContainersToCleanup();
+ List<ContainerId> getFinishedContainersPulledByAM();
List<ApplicationId> getApplicationsToCleanup();
@@ -43,6 +44,10 @@ public interface NodeHeartbeatResponse {
void setNMTokenMasterKey(MasterKey secretKey);
void addAllContainersToCleanup(List<ContainerId> containers);
+
+ // This tells NM to remove finished containers only after the AM
+ // has actually received it in a previous allocate response
+ void addFinishedContainersPulledByAM(List<ContainerId> containers);
void addAllApplicationsToCleanup(List<ApplicationId> applications);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index 775f95a..e9296f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -46,6 +46,7 @@ public class NodeHeartbeatResponsePBImpl extends ProtoBase<NodeHeartbeatResponse
boolean viaProto = false;
private List<ContainerId> containersToCleanup = null;
+ private List<ContainerId> finishedContainersPulledByAM = null;
private List<ApplicationId> applicationsToCleanup = null;
private MasterKey containerTokenMasterKey = null;
private MasterKey nmTokenMasterKey = null;
@@ -73,6 +74,9 @@ public class NodeHeartbeatResponsePBImpl extends ProtoBase<NodeHeartbeatResponse
if (this.applicationsToCleanup != null) {
addApplicationsToCleanupToProto();
}
+ if (this.finishedContainersPulledByAM != null) {
+ addFinishedContainersPulledByAMToProto();
+ }
if (this.containerTokenMasterKey != null) {
builder.setContainerTokenMasterKey(
convertToProtoFormat(this.containerTokenMasterKey));
@@ -199,6 +203,12 @@ public class NodeHeartbeatResponsePBImpl extends ProtoBase<NodeHeartbeatResponse
return this.containersToCleanup;
}
+ @Override
+ public List<ContainerId> getFinishedContainersPulledByAM() {
+ initFinishedContainersPulledByAM();
+ return this.finishedContainersPulledByAM;
+ }
+
private void initContainersToCleanup() {
if (this.containersToCleanup != null) {
return;
@@ -212,6 +222,19 @@ public class NodeHeartbeatResponsePBImpl extends ProtoBase<NodeHeartbeatResponse
}
}
+ private void initFinishedContainersPulledByAM() {
+ if (this.finishedContainersPulledByAM != null) {
+ return;
+ }
+ NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
+ List<ContainerIdProto> list = p.getFinishedContainersPulledByAmList();
+ this.finishedContainersPulledByAM = new ArrayList<ContainerId>();
+
+ for (ContainerIdProto c : list) {
+ this.finishedContainersPulledByAM.add(convertFromProtoFormat(c));
+ }
+ }
+
@Override
public void addAllContainersToCleanup(
final List<ContainerId> containersToCleanup) {
@@ -221,6 +244,15 @@ public class NodeHeartbeatResponsePBImpl extends ProtoBase<NodeHeartbeatResponse
this.containersToCleanup.addAll(containersToCleanup);
}
+ @Override
+ public void addFinishedContainersPulledByAM(
+ final List<ContainerId> finishedContainersPulledByAM) {
+ if (finishedContainersPulledByAM == null)
+ return;
+ initFinishedContainersPulledByAM();
+ this.finishedContainersPulledByAM.addAll(finishedContainersPulledByAM);
+ }
+
private void addContainersToCleanupToProto() {
maybeInitBuilder();
builder.clearContainersToCleanup();
@@ -256,6 +288,41 @@ public class NodeHeartbeatResponsePBImpl extends ProtoBase<NodeHeartbeatResponse
builder.addAllContainersToCleanup(iterable);
}
+ private void addFinishedContainersPulledByAMToProto() {
+ maybeInitBuilder();
+ builder.clearFinishedContainersPulledByAm();
+ if (finishedContainersPulledByAM == null)
+ return;
+ Iterable<ContainerIdProto> iterable = new Iterable<ContainerIdProto>() {
+
+ @Override
+ public Iterator<ContainerIdProto> iterator() {
+ return new Iterator<ContainerIdProto>() {
+
+ Iterator<ContainerId> iter = finishedContainersPulledByAM.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public ContainerIdProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
+ builder.addAllFinishedContainersPulledByAm(iterable);
+ }
+
@Override
public List<ApplicationId> getApplicationsToCleanup() {
initApplicationsToCleanup();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 29cd64e..600f54d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -58,6 +58,7 @@ message NodeHeartbeatResponseProto {
repeated ApplicationIdProto applications_to_cleanup = 6;
optional int64 nextHeartBeatInterval = 7;
optional string diagnostics_message = 8;
+ repeated ContainerIdProto finished_containers_pulled_by_am = 9;
}
message NMContainerStatusProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index a479be2..43770c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -311,7 +311,7 @@ public class NodeManager extends CompositeService
public static class NMContext implements Context {
private NodeId nodeId = null;
- private final ConcurrentMap<ApplicationId, Application> applications =
+ protected final ConcurrentMap<ApplicationId, Application> applications =
new ConcurrentHashMap<ApplicationId, Application>();
protected final ConcurrentMap<ContainerId, Container> containers =
new ConcurrentSkipListMap<ContainerId, Container>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index b52b0fb..b9feacb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -104,11 +104,6 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
// Duration for which to track recently stopped container.
private long durationToTrackStoppedContainers;
- // This is used to track the current completed containers when nodeheartBeat
- // is called. These completed containers will be removed from NM context after
- // nodeHeartBeat succeeds and the response from the nodeHeartBeat is
- // processed.
- private final Set<ContainerId> previousCompletedContainers;
private final NodeHealthCheckerService healthChecker;
private final NodeManagerMetrics metrics;
@@ -125,7 +120,6 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
this.metrics = metrics;
this.recentlyStoppedContainers =
new LinkedHashMap<ContainerId, Long>();
- this.previousCompletedContainers = new HashSet<ContainerId>();
}
@Override
@@ -331,7 +325,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
return appList;
}
- private NodeStatus getNodeStatus(int responseId) {
+ private NodeStatus getNodeStatus(int responseId) throws IOException {
NodeHealthStatus nodeHealthStatus = this.context.getNodeHealthStatus();
nodeHealthStatus.setHealthReport(healthChecker.getHealthReport());
@@ -352,11 +346,18 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
// Iterate through the NMContext and clone and get all the containers'
// statuses. If it's a completed container, add into the
- // recentlyStoppedContainers and previousCompletedContainers collections.
+ // recentlyStoppedContainers collections.
@VisibleForTesting
- protected List<ContainerStatus> getContainerStatuses() {
+ protected List<ContainerStatus> getContainerStatuses() throws IOException {
List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
for (Container container : this.context.getContainers().values()) {
+ ContainerId containerId = container.getContainerId();
+ ApplicationId applicationId = container.getContainerId()
+ .getApplicationAttemptId().getApplicationId();
+ if (!this.context.getApplications().containsKey(applicationId)) {
+ context.getContainers().remove(containerId);
+ continue;
+ }
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus =
container.cloneAndGetContainerStatus();
containerStatuses.add(containerStatus);
@@ -381,10 +382,17 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
}
// These NMContainerStatus are sent on NM registration and used by YARN only.
- private List<NMContainerStatus> getNMContainerStatuses() {
+ private List<NMContainerStatus> getNMContainerStatuses() throws IOException {
List<NMContainerStatus> containerStatuses =
new ArrayList<NMContainerStatus>();
for (Container container : this.context.getContainers().values()) {
+ ContainerId containerId = container.getContainerId();
+ ApplicationId applicationId = container.getContainerId()
+ .getApplicationAttemptId().getApplicationId();
+ if (!this.context.getApplications().containsKey(applicationId)) {
+ context.getContainers().remove(containerId);
+ continue;
+ }
NMContainerStatus status =
container.getNMContainerStatus();
containerStatuses.add(status);
@@ -402,26 +410,30 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
@Override
public void addCompletedContainer(ContainerId containerId) {
- synchronized (previousCompletedContainers) {
- previousCompletedContainers.add(containerId);
- }
synchronized (recentlyStoppedContainers) {
removeVeryOldStoppedContainersFromCache();
- recentlyStoppedContainers.put(containerId,
- System.currentTimeMillis() + durationToTrackStoppedContainers);
+ if (!recentlyStoppedContainers.containsKey(containerId)) {
+ recentlyStoppedContainers.put(containerId,
+ System.currentTimeMillis() + durationToTrackStoppedContainers);
+ }
}
}
- private void removeCompletedContainersFromContext() {
- synchronized (previousCompletedContainers) {
- if (!previousCompletedContainers.isEmpty()) {
- for (ContainerId containerId : previousCompletedContainers) {
- this.context.getContainers().remove(containerId);
- }
- LOG.info("Removed completed containers from NM context: "
- + previousCompletedContainers);
- previousCompletedContainers.clear();
- }
+ @VisibleForTesting
+ @Private
+ public void removeCompletedContainersFromContext(
+ List<ContainerId>containerIds) throws IOException {
+ Set<ContainerId> removedContainers = new HashSet<ContainerId>();
+
+ // If the AM has pulled the completedContainer it can be removed
+ for (ContainerId containerId : containerIds) {
+ context.getContainers().remove(containerId);
+ removedContainers.add(containerId);
+ }
+
+ if (!removedContainers.isEmpty()) {
+ LOG.info("Removed completed containers from NM context: " +
+ removedContainers);
}
}
@@ -454,7 +466,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
return recentlyStoppedContainers.containsKey(containerId);
}
}
-
+
@Override
public void clearFinishedContainersFromCache() {
synchronized (recentlyStoppedContainers) {
@@ -472,11 +484,13 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
while (i.hasNext()) {
ContainerId cid = i.next();
if (recentlyStoppedContainers.get(cid) < currentTime) {
- i.remove();
- try {
- context.getNMStateStore().removeContainer(cid);
- } catch (IOException e) {
- LOG.error("Unable to remove container " + cid + " in store", e);
+ if (!context.getContainers().containsKey(cid)) {
+ i.remove();
+ try {
+ context.getNMStateStore().removeContainer(cid);
+ } catch (IOException e) {
+ LOG.error("Unable to remove container " + cid + " in store", e);
+ }
}
} else {
break;
@@ -542,7 +556,9 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
// don't want to remove the completed containers before resync
// because these completed containers will be reported back to RM
// when NM re-registers with RM.
- removeCompletedContainersFromContext();
+ // Only remove the cleanedup containers that are acked
+ removeCompletedContainersFromContext(response
+ .getFinishedContainersPulledByAM());
lastHeartBeatID = response.getResponseId();
List<ContainerId> containersToCleanup = response
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
index acda2a9..85bafb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequ
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -247,6 +248,10 @@ public class TestNodeManagerResync {
// put the completed container into the context
getNMContext().getContainers().put(
testCompleteContainer.getContainerId(), container);
+ getNMContext().getApplications().put(
+ testCompleteContainer.getContainerId()
+ .getApplicationAttemptId().getApplicationId(),
+ mock(Application.class));
} else {
// second register contains the completed container info.
List<NMContainerStatus> statuses =
@@ -382,9 +387,17 @@ public class TestNodeManagerResync {
if (containersShouldBePreserved) {
Assert.assertFalse(containers.isEmpty());
Assert.assertTrue(containers.containsKey(existingCid));
+ Assert.assertEquals(ContainerState.RUNNING,
+ containers.get(existingCid)
+ .cloneAndGetContainerStatus().getState());
} else {
- // ensure that containers are empty before restart nodeStatusUpdater
- Assert.assertTrue(containers.isEmpty());
+ // ensure that containers are empty or are completed before
+ // restart nodeStatusUpdater
+ if (!containers.isEmpty()) {
+ Assert.assertEquals(ContainerState.COMPLETE,
+ containers.get(existingCid)
+ .cloneAndGetContainerStatus().getState());
+ }
}
super.rebootNodeStatusUpdaterAndRegisterWithRM();
}
@@ -465,7 +478,12 @@ public class TestNodeManagerResync {
try {
// ensure that containers are empty before restart nodeStatusUpdater
- Assert.assertTrue(containers.isEmpty());
+ if (!containers.isEmpty()) {
+ for (Container container: containers.values()) {
+ Assert.assertEquals(ContainerState.COMPLETE,
+ container.cloneAndGetContainerStatus().getState());
+ }
+ }
super.rebootNodeStatusUpdaterAndRegisterWithRM();
// After this point new containers are free to be launched, except
// containers from previous RM
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index f2a3a4a..8fb51a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.client.RMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -180,7 +181,7 @@ public class TestNodeStatusUpdater {
Map<ApplicationId, List<ContainerStatus>> map =
new HashMap<ApplicationId, List<ContainerStatus>>();
for (ContainerStatus cs : containers) {
- ApplicationId applicationId =
+ ApplicationId applicationId =
cs.getContainerId().getApplicationAttemptId().getApplicationId();
List<ContainerStatus> appContainers = map.get(applicationId);
if (appContainers == null) {
@@ -205,10 +206,10 @@ public class TestNodeStatusUpdater {
nodeStatus.setResponseId(heartBeatID++);
Map<ApplicationId, List<ContainerStatus>> appToContainers =
getAppToContainerStatusMap(nodeStatus.getContainersStatuses());
-
+
ApplicationId appId1 = ApplicationId.newInstance(0, 1);
ApplicationId appId2 = ApplicationId.newInstance(0, 2);
-
+
if (heartBeatID == 1) {
Assert.assertEquals(0, nodeStatus.getContainersStatuses().size());
@@ -419,7 +420,7 @@ public class TestNodeStatusUpdater {
}
private class MyNodeManager extends NodeManager {
-
+
private MyNodeStatusUpdater3 nodeStatusUpdater;
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
@@ -433,7 +434,7 @@ public class TestNodeStatusUpdater {
return this.nodeStatusUpdater;
}
}
-
+
private class MyNodeManager2 extends NodeManager {
public boolean isStopped = false;
private NodeStatusUpdater nodeStatusUpdater;
@@ -467,7 +468,7 @@ public class TestNodeStatusUpdater {
syncBarrier.await(10000, TimeUnit.MILLISECONDS);
}
}
- //
+ //
private class MyResourceTracker2 implements ResourceTracker {
public NodeAction heartBeatNodeAction = NodeAction.NORMAL;
public NodeAction registerNodeAction = NodeAction.NORMAL;
@@ -478,7 +479,7 @@ public class TestNodeStatusUpdater {
public RegisterNodeManagerResponse registerNodeManager(
RegisterNodeManagerRequest request) throws YarnException,
IOException {
-
+
RegisterNodeManagerResponse response = recordFactory
.newRecordInstance(RegisterNodeManagerResponse.class);
response.setNodeAction(registerNodeAction );
@@ -493,7 +494,7 @@ public class TestNodeStatusUpdater {
throws YarnException, IOException {
NodeStatus nodeStatus = request.getNodeStatus();
nodeStatus.setResponseId(heartBeatID++);
-
+
NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils.
newNodeHeartbeatResponse(heartBeatID, heartBeatNodeAction, null,
null, null, null, 1000L);
@@ -501,7 +502,7 @@ public class TestNodeStatusUpdater {
return nhResponse;
}
}
-
+
private class MyResourceTracker3 implements ResourceTracker {
public NodeAction heartBeatNodeAction = NodeAction.NORMAL;
public NodeAction registerNodeAction = NodeAction.NORMAL;
@@ -513,7 +514,7 @@ public class TestNodeStatusUpdater {
MyResourceTracker3(Context context) {
this.context = context;
}
-
+
@Override
public RegisterNodeManagerResponse registerNodeManager(
RegisterNodeManagerRequest request) throws YarnException,
@@ -564,6 +565,14 @@ public class TestNodeStatusUpdater {
public NodeAction registerNodeAction = NodeAction.NORMAL;
public NodeAction heartBeatNodeAction = NodeAction.NORMAL;
private Context context;
+ private final ContainerStatus containerStatus2 =
+ createContainerStatus(2, ContainerState.RUNNING);
+ private final ContainerStatus containerStatus3 =
+ createContainerStatus(3, ContainerState.COMPLETE);
+ private final ContainerStatus containerStatus4 =
+ createContainerStatus(4, ContainerState.RUNNING);
+ private final ContainerStatus containerStatus5 =
+ createContainerStatus(5, ContainerState.COMPLETE);
public MyResourceTracker4(Context context) {
this.context = context;
@@ -583,6 +592,8 @@ public class TestNodeStatusUpdater {
@Override
public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
throws YarnException, IOException {
+ List<ContainerId> finishedContainersPulledByAM = new ArrayList
+ <ContainerId>();
try {
if (heartBeatID == 0) {
Assert.assertEquals(request.getNodeStatus().getContainersStatuses()
@@ -594,10 +605,6 @@ public class TestNodeStatusUpdater {
Assert.assertEquals(statuses.size(), 2);
Assert.assertEquals(context.getContainers().size(), 2);
- ContainerStatus containerStatus2 =
- createContainerStatus(2, ContainerState.RUNNING);
- ContainerStatus containerStatus3 =
- createContainerStatus(3, ContainerState.COMPLETE);
boolean container2Exist = false, container3Exist = false;
for (ContainerStatus status : statuses) {
if (status.getContainerId().equals(
@@ -619,23 +626,14 @@ public class TestNodeStatusUpdater {
// nodeStatusUpdaterRunnable, otherwise nm just shuts down and the
// test passes.
throw new YarnRuntimeException("Lost the heartbeat response");
- } else if (heartBeatID == 2) {
+ } else if (heartBeatID == 2 || heartBeatID == 3) {
List<ContainerStatus> statuses =
request.getNodeStatus().getContainersStatuses();
Assert.assertEquals(statuses.size(), 4);
Assert.assertEquals(context.getContainers().size(), 4);
- ContainerStatus containerStatus2 =
- createContainerStatus(2, ContainerState.RUNNING);
- ContainerStatus containerStatus3 =
- createContainerStatus(3, ContainerState.COMPLETE);
- ContainerStatus containerStatus4 =
- createContainerStatus(4, ContainerState.RUNNING);
- ContainerStatus containerStatus5 =
- createContainerStatus(5, ContainerState.COMPLETE);
-
- boolean container2Exist = false, container3Exist = false, container4Exist =
- false, container5Exist = false;
+ boolean container2Exist = false, container3Exist = false,
+ container4Exist = false, container5Exist = false;
for (ContainerStatus status : statuses) {
if (status.getContainerId().equals(
containerStatus2.getContainerId())) {
@@ -664,6 +662,24 @@ public class TestNodeStatusUpdater {
}
Assert.assertTrue(container2Exist && container3Exist
&& container4Exist && container5Exist);
+
+ if (heartBeatID == 3) {
+ finishedContainersPulledByAM.add(containerStatus3.getContainerId());
+ }
+ } else if (heartBeatID == 4) {
+ List<ContainerStatus> statuses =
+ request.getNodeStatus().getContainersStatuses();
+ Assert.assertEquals(statuses.size(), 3);
+ Assert.assertEquals(context.getContainers().size(), 3);
+
+ boolean container3Exist = false;
+ for (ContainerStatus status : statuses) {
+ if (status.getContainerId().equals(
+ containerStatus3.getContainerId())) {
+ container3Exist = true;
+ }
+ }
+ Assert.assertFalse(container3Exist);
}
} catch (AssertionError error) {
error.printStackTrace();
@@ -676,6 +692,7 @@ public class TestNodeStatusUpdater {
NodeHeartbeatResponse nhResponse =
YarnServerBuilderUtils.newNodeHeartbeatResponse(heartBeatID,
heartBeatNodeAction, null, null, null, null, 1000L);
+ nhResponse.addFinishedContainersPulledByAM(finishedContainersPulledByAM);
return nhResponse;
}
}
@@ -686,7 +703,7 @@ public class TestNodeStatusUpdater {
public RegisterNodeManagerResponse registerNodeManager(
RegisterNodeManagerRequest request) throws YarnException,
IOException {
-
+
RegisterNodeManagerResponse response = recordFactory
.newRecordInstance(RegisterNodeManagerResponse.class);
response.setNodeAction(registerNodeAction );
@@ -694,7 +711,7 @@ public class TestNodeStatusUpdater {
response.setNMTokenMasterKey(createMasterKey());
return response;
}
-
+
@Override
public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
throws YarnException, IOException {
@@ -767,11 +784,11 @@ public class TestNodeStatusUpdater {
lfs.delete(new Path(basedir.getPath()), true);
}
- @Test(timeout = 90000)
- public void testRecentlyFinishedContainers() throws Exception {
- NodeManager nm = new NodeManager();
- YarnConfiguration conf = new YarnConfiguration();
- conf.set(
+ @Test(timeout = 90000)
+ public void testRecentlyFinishedContainers() throws Exception {
+ NodeManager nm = new NodeManager();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(
NodeStatusUpdaterImpl.YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
"10000");
nm.init(conf);
@@ -780,27 +797,112 @@ public class TestNodeStatusUpdater {
ApplicationId appId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 0);
- ContainerId cId = ContainerId.newInstance(appAttemptId, 0);
-
-
+ ContainerId cId = ContainerId.newInstance(appAttemptId, 0);
+ nm.getNMContext().getApplications().putIfAbsent(appId,
+ mock(Application.class));
+ nm.getNMContext().getContainers().putIfAbsent(cId, mock(Container.class));
+
nodeStatusUpdater.addCompletedContainer(cId);
Assert.assertTrue(nodeStatusUpdater.isContainerRecentlyStopped(cId));
-
+
+ nm.getNMContext().getContainers().remove(cId);
long time1 = System.currentTimeMillis();
int waitInterval = 15;
while (waitInterval-- > 0
&& nodeStatusUpdater.isContainerRecentlyStopped(cId)) {
- nodeStatusUpdater.removeVeryOldStoppedContainersFromCache();
+ nodeStatusUpdater.removeVeryOldStoppedContainersFromCache();
Thread.sleep(1000);
}
- long time2 = System.currentTimeMillis();
+ long time2 = System.currentTimeMillis();
// By this time the container will be removed from cache. need to verify.
- Assert.assertFalse(nodeStatusUpdater.isContainerRecentlyStopped(cId));
- Assert.assertTrue((time2 - time1) >= 10000 && (time2 -time1) <= 250000);
- }
-
+ Assert.assertFalse(nodeStatusUpdater.isContainerRecentlyStopped(cId));
+ Assert.assertTrue((time2 - time1) >= 10000 && (time2 - time1) <= 250000);
+ }
+ @Test(timeout = 90000)
+ public void testRemovePreviousCompletedContainersFromContext() throws Exception {
+ NodeManager nm = new NodeManager();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(
+ NodeStatusUpdaterImpl
+ .YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
+ "10000");
+ nm.init(conf);
+ NodeStatusUpdaterImpl nodeStatusUpdater =
+ (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 0);
+ ContainerId cId = ContainerId.newInstance(appAttemptId, 1);
+ Token containerToken =
+ BuilderUtils.newContainerToken(cId, "anyHost", 1234, "anyUser",
+ BuilderUtils.newResource(1024, 1), 0, 123,
+ "password".getBytes(), 0);
+ Container anyCompletedContainer = new ContainerImpl(conf, null,
+ null, null, null, null,
+ BuilderUtils.newContainerTokenIdentifier(containerToken)) {
+
+ @Override
+ public ContainerState getCurrentState() {
+ return ContainerState.COMPLETE;
+ }
+ };
+
+ nm.getNMContext().getApplications().putIfAbsent(appId,
+ mock(Application.class));
+ nm.getNMContext().getContainers().put(cId, anyCompletedContainer);
+ Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
+
+ List<ContainerId> ackedContainers = new ArrayList<ContainerId>();
+ ackedContainers.add(cId);
+
+ nodeStatusUpdater.removeCompletedContainersFromContext(ackedContainers);
+ Assert.assertTrue(nodeStatusUpdater.getContainerStatuses().isEmpty());
+ }
+
+ @Test
+ public void testCleanedupApplicationContainerCleanup() throws IOException {
+ NodeManager nm = new NodeManager();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(NodeStatusUpdaterImpl
+ .YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
+ "1000000");
+ nm.init(conf);
+
+ NodeStatusUpdaterImpl nodeStatusUpdater =
+ (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 0);
+
+ ContainerId cId = ContainerId.newInstance(appAttemptId, 1);
+ Token containerToken =
+ BuilderUtils.newContainerToken(cId, "anyHost", 1234, "anyUser",
+ BuilderUtils.newResource(1024, 1), 0, 123,
+ "password".getBytes(), 0);
+ Container anyCompletedContainer = new ContainerImpl(conf, null,
+ null, null, null, null,
+ BuilderUtils.newContainerTokenIdentifier(containerToken)) {
+
+ @Override
+ public ContainerState getCurrentState() {
+ return ContainerState.COMPLETE;
+ }
+ };
+
+ nm.getNMContext().getApplications().putIfAbsent(appId,
+ mock(Application.class));
+ nm.getNMContext().getContainers().put(cId, anyCompletedContainer);
+
+ Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
+
+ nm.getNMContext().getApplications().remove(appId);
+ nodeStatusUpdater.removeCompletedContainersFromContext(new ArrayList
+ <ContainerId>());
+ Assert.assertEquals(0, nodeStatusUpdater.getContainerStatuses().size());
+ }
+
@Test
public void testNMRegistration() throws InterruptedException {
nm = new NodeManager() {
@@ -860,7 +962,7 @@ public class TestNodeStatusUpdater {
nm.stop();
}
-
+
@Test
public void testStopReentrant() throws Exception {
final AtomicInteger numCleanups = new AtomicInteger(0);
@@ -875,7 +977,7 @@ public class TestNodeStatusUpdater {
myNodeStatusUpdater.resourceTracker = myResourceTracker2;
return myNodeStatusUpdater;
}
-
+
@Override
protected ContainerManagerImpl createContainerManager(Context context,
ContainerExecutor exec, DeletionService del,
@@ -897,7 +999,7 @@ public class TestNodeStatusUpdater {
YarnConfiguration conf = createNMConfig();
nm.init(conf);
nm.start();
-
+
int waitCount = 0;
while (heartBeatID < 1 && waitCount++ != 200) {
Thread.sleep(500);
@@ -906,7 +1008,7 @@ public class TestNodeStatusUpdater {
// Meanwhile call stop directly as the shutdown hook would
nm.stop();
-
+
// NM takes a while to reach the STOPPED state.
waitCount = 0;
while (nm.getServiceState() != STATE.STOPPED && waitCount++ != 20) {
@@ -1172,9 +1274,13 @@ public class TestNodeStatusUpdater {
nm.start();
int waitCount = 0;
- while (heartBeatID <= 3 && waitCount++ != 20) {
+ while (heartBeatID <= 4 && waitCount++ != 20) {
Thread.sleep(500);
}
+ if (heartBeatID <= 4) {
+ Assert.fail("Failed to get all heartbeats in time, " +
+ "heartbeatID:" + heartBeatID);
+ }
if(assertionFailedInThread.get()) {
Assert.fail("ContainerStatus Backup failed");
}
@@ -1182,7 +1288,7 @@ public class TestNodeStatusUpdater {
}
@Test(timeout = 200000)
- public void testNodeStatusUpdaterRetryAndNMShutdown()
+ public void testNodeStatusUpdaterRetryAndNMShutdown()
throws Exception {
final long connectionWaitSecs = 1000;
final long connectionRetryIntervalMs = 1000;
@@ -1190,7 +1296,7 @@ public class TestNodeStatusUpdater {
conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
connectionWaitSecs);
conf.setLong(YarnConfiguration
- .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
connectionRetryIntervalMs);
conf.setLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS, 5000);
conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
@@ -1281,30 +1387,36 @@ public class TestNodeStatusUpdater {
} else if (heartBeatID == 1) {
ContainerStatus containerStatus2 =
createContainerStatus(2, ContainerState.RUNNING);
- Container container2 = getMockContainer(containerStatus2);
- containers.put(containerStatus2.getContainerId(), container2);
+ putMockContainer(containerStatus2);
ContainerStatus containerStatus3 =
createContainerStatus(3, ContainerState.COMPLETE);
- Container container3 = getMockContainer(containerStatus3);
- containers.put(containerStatus3.getContainerId(), container3);
+ putMockContainer(containerStatus3);
return containers;
} else if (heartBeatID == 2) {
ContainerStatus containerStatus4 =
createContainerStatus(4, ContainerState.RUNNING);
- Container container4 = getMockContainer(containerStatus4);
- containers.put(containerStatus4.getContainerId(), container4);
+ putMockContainer(containerStatus4);
ContainerStatus containerStatus5 =
createContainerStatus(5, ContainerState.COMPLETE);
- Container container5 = getMockContainer(containerStatus5);
- containers.put(containerStatus5.getContainerId(), container5);
+ putMockContainer(containerStatus5);
+ return containers;
+ } else if (heartBeatID == 3 || heartBeatID == 4) {
return containers;
} else {
containers.clear();
return containers;
}
}
+
+ private void putMockContainer(ContainerStatus containerStatus) {
+ Container container = getMockContainer(containerStatus);
+ containers.put(containerStatus.getContainerId(), container);
+ applications.putIfAbsent(containerStatus.getContainerId()
+ .getApplicationAttemptId().getApplicationId(),
+ mock(Application.class));
+ }
}
public static ContainerStatus createContainerStatus(int id,
@@ -1345,7 +1457,7 @@ public class TestNodeStatusUpdater {
throw e;
}
}
-
+
// the service should be stopped
Assert.assertEquals("NM state is wrong!", STATE.STOPPED, nm
.getServiceState());
@@ -1364,7 +1476,7 @@ public class TestNodeStatusUpdater {
}
conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":12345");
- conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":12346");
+ conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":12346");
conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
remoteLogsDir.getAbsolutePath());
@@ -1372,7 +1484,7 @@ public class TestNodeStatusUpdater {
conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
return conf;
}
-
+
private NodeManager getNodeManager(final NodeAction nodeHeartBeatAction) {
return new NodeManager() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 4798120..4222888 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -198,7 +198,7 @@ public class ResourceTrackerService extends AbstractService implements
*/
@SuppressWarnings("unchecked")
@VisibleForTesting
- void handleNMContainerStatus(NMContainerStatus containerStatus) {
+ void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
ApplicationAttemptId appAttemptId =
containerStatus.getContainerId().getApplicationAttemptId();
RMApp rmApp =
@@ -229,7 +229,8 @@ public class ResourceTrackerService extends AbstractService implements
containerStatus.getContainerExitStatus());
// sending master container finished event.
RMAppAttemptContainerFinishedEvent evt =
- new RMAppAttemptContainerFinishedEvent(appAttemptId, status);
+ new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
+ nodeId);
rmContext.getDispatcher().getEventHandler().handle(evt);
}
}
@@ -324,7 +325,7 @@ public class ResourceTrackerService extends AbstractService implements
LOG.info("received container statuses on node manager register :"
+ request.getNMContainerStatuses());
for (NMContainerStatus status : request.getNMContainerStatuses()) {
- handleNMContainerStatus(status);
+ handleNMContainerStatus(status, nodeId);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index ff520be..0b8f321 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1181,7 +1181,7 @@ public class RMAppImpl implements RMApp, Recoverable {
int numberOfFailure = app.getNumFailedAppAttempts();
if (!app.submissionContext.getUnmanagedAM()
&& numberOfFailure < app.maxAppAttempts) {
- boolean transferStateFromPreviousAttempt = false;
+ boolean transferStateFromPreviousAttempt;
RMAppFailedAttemptEvent failedEvent = (RMAppFailedAttemptEvent) event;
transferStateFromPreviousAttempt =
failedEvent.getTransferStateFromPreviousAttempt();
@@ -1191,11 +1191,11 @@ public class RMAppImpl implements RMApp, Recoverable {
// Transfer the state from the previous attempt to the current attempt.
// Note that the previous failed attempt may still be collecting the
// container events from the scheduler and update its data structures
- // before the new attempt is created.
- if (transferStateFromPreviousAttempt) {
- ((RMAppAttemptImpl) app.currentAttempt)
- .transferStateFromPreviousAttempt(oldAttempt);
- }
+ // before the new attempt is created. We always transferState for
+ // finished containers so that they can be acked to NM,
+ // but when pulling finished container we will check this flag again.
+ ((RMAppAttemptImpl) app.currentAttempt)
+ .transferStateFromPreviousAttempt(oldAttempt);
return initialState;
} else {
if (numberOfFailure >= app.maxAppAttempts) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
index 943a5e5..cf8c2bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
import java.util.List;
+import java.util.concurrent.ConcurrentMap;
import javax.crypto.SecretKey;
@@ -31,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -120,14 +122,29 @@ public interface RMAppAttempt extends EventHandler<RMAppAttemptEvent> {
List<ContainerStatus> pullJustFinishedContainers();
/**
- * Return the list of last set of finished containers. This does not reset the
- * finished containers.
- * @return the list of just finished contianers, this does not reset the
+ * Returns a reference to the map of last set of finished containers to the
+ * corresponding node. This does not reset the finished containers.
+ * @return the list of just finished containers, this does not reset the
* finished containers.
*/
+ ConcurrentMap<NodeId, List<ContainerStatus>>
+ getJustFinishedContainersReference();
+
+ /**
+ * Return the list of last set of finished containers. This does not reset
+ * the finished containers.
+ * @return the list of just finished containers
+ */
List<ContainerStatus> getJustFinishedContainers();
/**
+ * The map of conatiners per Node that are already sent to the AM.
+ * @return map of per node list of finished container status sent to AM
+ */
+ ConcurrentMap<NodeId, List<ContainerStatus>>
+ getFinishedContainersSentToAMReference();
+
+ /**
* The container on which the Application Master is running.
* @return the {@link Container} on which the application master is running.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 7ca57ee..d75a871 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -24,9 +24,12 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -52,6 +55,7 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
@@ -83,6 +87,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStatusupdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
@@ -129,9 +134,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
private final ApplicationSubmissionContext submissionContext;
private Token<AMRMTokenIdentifier> amrmToken = null;
private SecretKey clientTokenMasterKey = null;
-
- private List<ContainerStatus> justFinishedContainers =
- new ArrayList<ContainerStatus>();
+
+ private ConcurrentMap<NodeId, List<ContainerStatus>>
+ justFinishedContainers =
+ new ConcurrentHashMap<NodeId, List<ContainerStatus>>();
+ // Tracks the previous finished containers that are waiting to be
+ // verified as received by the AM. If the AM sends the next allocate
+ // request it implicitly acks this list.
+ private ConcurrentMap<NodeId, List<ContainerStatus>>
+ finishedContainersSentToAM =
+ new ConcurrentHashMap<NodeId, List<ContainerStatus>>();
private Container masterContainer;
private float progress = 0;
@@ -627,10 +639,28 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
}
+ @VisibleForTesting
@Override
public List<ContainerStatus> getJustFinishedContainers() {
this.readLock.lock();
try {
+ List<ContainerStatus> returnList = new ArrayList<ContainerStatus>();
+ for (Collection<ContainerStatus> containerStatusList :
+ justFinishedContainers.values()) {
+ returnList.addAll(containerStatusList);
+ }
+ return returnList;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
+ public ConcurrentMap<NodeId, List<ContainerStatus>>
+ getJustFinishedContainersReference
+ () {
+ this.readLock.lock();
+ try {
return this.justFinishedContainers;
} finally {
this.readLock.unlock();
@@ -638,14 +668,67 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
@Override
+ public ConcurrentMap<NodeId, List<ContainerStatus>>
+ getFinishedContainersSentToAMReference() {
+ this.readLock.lock();
+ try {
+ return this.finishedContainersSentToAM;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
public List<ContainerStatus> pullJustFinishedContainers() {
this.writeLock.lock();
try {
- List<ContainerStatus> returnList = new ArrayList<ContainerStatus>(
- this.justFinishedContainers.size());
- returnList.addAll(this.justFinishedContainers);
- this.justFinishedContainers.clear();
+ List<ContainerStatus> returnList = new ArrayList<ContainerStatus>();
+
+ // A new allocate means the AM received the previously sent
+ // finishedContainers. We can ack this to NM now
+ for (NodeId nodeId:finishedContainersSentToAM.keySet()) {
+
+ // Clear and get current values
+ List<ContainerStatus> currentSentContainers =
+ finishedContainersSentToAM
+ .put(nodeId, new ArrayList<ContainerStatus>());
+ List<ContainerId> containerIdList = new ArrayList<ContainerId>
+ (currentSentContainers.size());
+ for (ContainerStatus containerStatus:currentSentContainers) {
+ containerIdList.add(containerStatus.getContainerId());
+ }
+ eventHandler.handle(new RMNodeFinishedContainersPulledByAMEvent(
+ nodeId, containerIdList));
+ }
+
+ // Mark every containerStatus as being sent to AM though we may return
+ // only the ones that belong to the current attempt
+ boolean keepContainersAcressAttempts = this.submissionContext
+ .getKeepContainersAcrossApplicationAttempts();
+ for (NodeId nodeId:justFinishedContainers.keySet()) {
+
+ // Clear and get current values
+ List<ContainerStatus> finishedContainers = justFinishedContainers.put
+ (nodeId, new ArrayList<ContainerStatus>());
+
+ if (keepContainersAcressAttempts) {
+ returnList.addAll(finishedContainers);
+ } else {
+ // Filter out containers from previous attempt
+ for (ContainerStatus containerStatus: finishedContainers) {
+ if (containerStatus.getContainerId().getApplicationAttemptId()
+ .equals(this.getAppAttemptId())) {
+ returnList.add(containerStatus);
+ }
+ }
+ }
+
+ finishedContainersSentToAM.putIfAbsent(nodeId, new ArrayList
+ <ContainerStatus>());
+ finishedContainersSentToAM.get(nodeId).addAll(finishedContainers);
+ }
+
return returnList;
} finally {
this.writeLock.unlock();
@@ -732,7 +815,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
setMasterContainer(attemptState.getMasterContainer());
recoverAppAttemptCredentials(attemptState.getAppAttemptCredentials(),
- attemptState.getState());
+ attemptState.getState());
this.recoveredFinalState = attemptState.getState();
this.originalTrackingUrl = attemptState.getFinalTrackingUrl();
this.proxiedTrackingUrl = generateProxyUriWithScheme(originalTrackingUrl);
@@ -744,7 +827,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
public void transferStateFromPreviousAttempt(RMAppAttempt attempt) {
- this.justFinishedContainers = attempt.getJustFinishedContainers();
+ this.justFinishedContainers = attempt.getJustFinishedContainersReference();
+ this.finishedContainersSentToAM =
+ attempt.getFinishedContainersSentToAMReference();
}
private void recoverAppAttemptCredentials(Credentials appAttemptTokens,
@@ -1507,6 +1592,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
ContainerStatus containerStatus =
containerFinishedEvent.getContainerStatus();
+ // Add all finished containers so that they can be acked to NM
+ addJustFinishedContainer(appAttempt, containerFinishedEvent);
+
// Is this container the AmContainer? If the finished container is same as
// the AMContainer, AppAttempt fails
if (appAttempt.masterContainer != null
@@ -1519,12 +1607,18 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
return RMAppAttemptState.FINAL_SAVING;
}
- // Normal container.Put it in completed containers list
- appAttempt.justFinishedContainers.add(containerStatus);
return this.currentState;
}
}
+ private static void addJustFinishedContainer(RMAppAttemptImpl appAttempt,
+ RMAppAttemptContainerFinishedEvent containerFinishedEvent) {
+ appAttempt.justFinishedContainers.putIfAbsent(containerFinishedEvent
+ .getNodeId(), new ArrayList<ContainerStatus>());
+ appAttempt.justFinishedContainers.get(containerFinishedEvent
+ .getNodeId()).add(containerFinishedEvent.getContainerStatus());
+ }
+
private static final class ContainerFinishedAtFinalStateTransition
extends BaseTransition {
@Override
@@ -1533,10 +1627,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
RMAppAttemptContainerFinishedEvent containerFinishedEvent =
(RMAppAttemptContainerFinishedEvent) event;
- ContainerStatus containerStatus =
- containerFinishedEvent.getContainerStatus();
// Normal container. Add it in completed containers list
- appAttempt.justFinishedContainers.add(containerStatus);
+ addJustFinishedContainer(appAttempt, containerFinishedEvent);
}
}
@@ -1569,6 +1661,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
ContainerStatus containerStatus =
containerFinishedEvent.getContainerStatus();
+ // Add all finished containers so that they can be acked to NM.
+ addJustFinishedContainer(appAttempt, containerFinishedEvent);
+
// Is this container the ApplicationMaster container?
if (appAttempt.masterContainer.getId().equals(
containerStatus.getContainerId())) {
@@ -1576,8 +1671,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt, containerFinishedEvent);
return RMAppAttemptState.FINISHED;
}
- // Normal container.
- appAttempt.justFinishedContainers.add(containerStatus);
+
return RMAppAttemptState.FINISHING;
}
}
@@ -1592,6 +1686,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
ContainerStatus containerStatus =
containerFinishedEvent.getContainerStatus();
+ // Add all finished containers so that they can be acked to NM.
+ addJustFinishedContainer(appAttempt, containerFinishedEvent);
+
// If this is the AM container, it means the AM container is finished,
// but we are not yet acknowledged that the final state has been saved.
// Thus, we still return FINAL_SAVING state here.
@@ -1611,8 +1708,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt.eventCausingFinalSaving), RMAppAttemptState.FINISHED);
return;
}
- // Normal container.
- appAttempt.justFinishedContainers.add(containerStatus);
}
}
@@ -1629,7 +1724,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) {
appAttempt.updateInfoOnAMUnregister(amUnregisteredEvent);
new FinalTransition(RMAppAttemptState.FINISHED).transition(appAttempt,
- event);
+ event);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java
index 3660597..39c6f29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java
@@ -20,21 +20,27 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
public class RMAppAttemptContainerFinishedEvent extends RMAppAttemptEvent {
private final ContainerStatus containerStatus;
+ private final NodeId nodeId;
public RMAppAttemptContainerFinishedEvent(ApplicationAttemptId appAttemptId,
- ContainerStatus containerStatus) {
+ ContainerStatus containerStatus, NodeId nodeId) {
super(appAttemptId, RMAppAttemptEventType.CONTAINER_FINISHED);
this.containerStatus = containerStatus;
+ this.nodeId = nodeId;
}
public ContainerStatus getContainerStatus() {
return this.containerStatus;
}
+ public NodeId getNodeId() {
+ return this.nodeId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 885e864..479734a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -78,13 +78,13 @@ public class RMContainerImpl implements RMContainer {
RMContainerEventType.RECOVER, new ContainerRecoveredTransition())
// Transitions from RESERVED state
- .addTransition(RMContainerState.RESERVED, RMContainerState.RESERVED,
+ .addTransition(RMContainerState.RESERVED, RMContainerState.RESERVED,
RMContainerEventType.RESERVED, new ContainerReservedTransition())
- .addTransition(RMContainerState.RESERVED, RMContainerState.ALLOCATED,
+ .addTransition(RMContainerState.RESERVED, RMContainerState.ALLOCATED,
RMContainerEventType.START, new ContainerStartedTransition())
- .addTransition(RMContainerState.RESERVED, RMContainerState.KILLED,
+ .addTransition(RMContainerState.RESERVED, RMContainerState.KILLED,
RMContainerEventType.KILL) // nothing to do
- .addTransition(RMContainerState.RESERVED, RMContainerState.RELEASED,
+ .addTransition(RMContainerState.RESERVED, RMContainerState.RELEASED,
RMContainerEventType.RELEASED) // nothing to do
@@ -100,7 +100,7 @@ public class RMContainerImpl implements RMContainer {
.addTransition(RMContainerState.ACQUIRED, RMContainerState.RUNNING,
RMContainerEventType.LAUNCHED, new LaunchedTransition())
.addTransition(RMContainerState.ACQUIRED, RMContainerState.COMPLETED,
- RMContainerEventType.FINISHED, new ContainerFinishedAtAcquiredState())
+ RMContainerEventType.FINISHED, new ContainerFinishedAtAcquiredState())
.addTransition(RMContainerState.ACQUIRED, RMContainerState.RELEASED,
RMContainerEventType.RELEASED, new KillTransition())
.addTransition(RMContainerState.ACQUIRED, RMContainerState.EXPIRED,
@@ -495,7 +495,8 @@ public class RMContainerImpl implements RMContainer {
updateAttemptMetrics(container);
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
- container.appAttemptId, finishedEvent.getRemoteContainerStatus()));
+ container.appAttemptId, finishedEvent.getRemoteContainerStatus(),
+ container.getAllocatedNode()));
container.rmContext.getRMApplicationHistoryWriter().containerFinished(
container);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
index c0096b9..b4d0b8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
@@ -40,6 +40,9 @@ public enum RMNodeEventType {
CONTAINER_ALLOCATED,
CLEANUP_CONTAINER,
+ // Source: RMAppAttempt
+ FINISHED_CONTAINERS_PULLED_BY_AM,
+
// Source: NMLivelinessMonitor
EXPIRE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeFinishedContainersPulledByAMEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeFinishedContainersPulledByAMEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeFinishedContainersPulledByAMEvent.java
new file mode 100644
index 0000000..a4fb707
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeFinishedContainersPulledByAMEvent.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+
+import java.util.List;
+
+// Happens after an implicit ack from AM that the container completion has
+// been notified successfully to the AM
+public class RMNodeFinishedContainersPulledByAMEvent extends RMNodeEvent {
+
+ private List<ContainerId> containers;
+
+ public RMNodeFinishedContainersPulledByAMEvent(NodeId nodeId,
+ List<ContainerId> containers) {
+ super(nodeId, RMNodeEventType.FINISHED_CONTAINERS_PULLED_BY_AM);
+ this.containers = containers;
+ }
+
+ public List<ContainerId> getContainers() {
+ return this.containers;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 1265958..f0ae826 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -112,6 +112,10 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private final Set<ContainerId> containersToClean = new TreeSet<ContainerId>(
new ContainerIdComparator());
+ /* set of containers that were notified to AM about their completion */
+ private final Set<ContainerId> finishedContainersPulledByAM =
+ new HashSet<ContainerId>();
+
/* the list of applications that have finished and need to be purged */
private final List<ApplicationId> finishedApplications = new ArrayList<ApplicationId>();
@@ -135,7 +139,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
new UpdateNodeResourceWhenUnusableTransition())
//Transitions from RUNNING state
- .addTransition(NodeState.RUNNING,
+ .addTransition(NodeState.RUNNING,
EnumSet.of(NodeState.RUNNING, NodeState.UNHEALTHY),
RMNodeEventType.STATUS_UPDATE, new StatusUpdateWhenHealthyTransition())
.addTransition(NodeState.RUNNING, NodeState.DECOMMISSIONED,
@@ -152,29 +156,39 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
.addTransition(NodeState.RUNNING, NodeState.RUNNING,
RMNodeEventType.CLEANUP_CONTAINER, new CleanUpContainerTransition())
.addTransition(NodeState.RUNNING, NodeState.RUNNING,
+ RMNodeEventType.FINISHED_CONTAINERS_PULLED_BY_AM,
+ new FinishedContainersPulledByAMTransition())
+ .addTransition(NodeState.RUNNING, NodeState.RUNNING,
RMNodeEventType.RECONNECTED, new ReconnectNodeTransition())
.addTransition(NodeState.RUNNING, NodeState.RUNNING,
RMNodeEventType.RESOURCE_UPDATE, new UpdateNodeResourceWhenRunningTransition())
//Transitions from REBOOTED state
.addTransition(NodeState.REBOOTED, NodeState.REBOOTED,
- RMNodeEventType.RESOURCE_UPDATE,
+ RMNodeEventType.RESOURCE_UPDATE,
new UpdateNodeResourceWhenUnusableTransition())
//Transitions from DECOMMISSIONED state
.addTransition(NodeState.DECOMMISSIONED, NodeState.DECOMMISSIONED,
- RMNodeEventType.RESOURCE_UPDATE,
+ RMNodeEventType.RESOURCE_UPDATE,
new UpdateNodeResourceWhenUnusableTransition())
-
+ .addTransition(NodeState.DECOMMISSIONED, NodeState.DECOMMISSIONED,
+ RMNodeEventType.FINISHED_CONTAINERS_PULLED_BY_AM,
+ new FinishedContainersPulledByAMTransition())
+
//Transitions from LOST state
.addTransition(NodeState.LOST, NodeState.LOST,
- RMNodeEventType.RESOURCE_UPDATE,
+ RMNodeEventType.RESOURCE_UPDATE,
new UpdateNodeResourceWhenUnusableTransition())
+ .addTransition(NodeState.LOST, NodeState.LOST,
+ RMNodeEventType.FINISHED_CONTAINERS_PULLED_BY_AM,
+ new FinishedContainersPulledByAMTransition())
//Transitions from UNHEALTHY state
- .addTransition(NodeState.UNHEALTHY,
+ .addTransition(NodeState.UNHEALTHY,
EnumSet.of(NodeState.UNHEALTHY, NodeState.RUNNING),
- RMNodeEventType.STATUS_UPDATE, new StatusUpdateWhenUnHealthyTransition())
+ RMNodeEventType.STATUS_UPDATE,
+ new StatusUpdateWhenUnHealthyTransition())
.addTransition(NodeState.UNHEALTHY, NodeState.DECOMMISSIONED,
RMNodeEventType.DECOMMISSION,
new DeactivateNodeTransition(NodeState.DECOMMISSIONED))
@@ -192,7 +206,10 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
RMNodeEventType.CLEANUP_CONTAINER, new CleanUpContainerTransition())
.addTransition(NodeState.UNHEALTHY, NodeState.UNHEALTHY,
RMNodeEventType.RESOURCE_UPDATE, new UpdateNodeResourceWhenUnusableTransition())
-
+ .addTransition(NodeState.UNHEALTHY, NodeState.UNHEALTHY,
+ RMNodeEventType.FINISHED_CONTAINERS_PULLED_BY_AM,
+ new FinishedContainersPulledByAMTransition())
+
// create the topology tables
.installTopology();
@@ -365,8 +382,11 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
response.addAllContainersToCleanup(
new ArrayList<ContainerId>(this.containersToClean));
response.addAllApplicationsToCleanup(this.finishedApplications);
+ response.addFinishedContainersPulledByAM(
+ new ArrayList<ContainerId>(this.finishedContainersPulledByAM));
this.containersToClean.clear();
this.finishedApplications.clear();
+ this.finishedContainersPulledByAM.clear();
} finally {
this.writeLock.unlock();
}
@@ -652,6 +672,16 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
}
+ public static class FinishedContainersPulledByAMTransition implements
+ SingleArcTransition<RMNodeImpl, RMNodeEvent> {
+
+ @Override
+ public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
+ rmNode.finishedContainersPulledByAM.addAll(((
+ RMNodeFinishedContainersPulledByAMEvent) event).getContainers());
+ }
+ }
+
public static class DeactivateNodeTransition
implements SingleArcTransition<RMNodeImpl, RMNodeEvent> {
@@ -726,7 +756,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
new ArrayList<ContainerStatus>();
for (ContainerStatus remoteContainer : statusEvent.getContainers()) {
ContainerId containerId = remoteContainer.getContainerId();
-
+
// Don't bother with containers already scheduled for cleanup, or for
// applications already killed. The scheduler doens't need to know any
// more about this container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 115f0b4..877a122 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -491,7 +491,7 @@ public class TestResourceTrackerService {
ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1),
ContainerState.COMPLETE, Resource.newInstance(1024, 1),
"Dummy Completed", 0, Priority.newInstance(10), 1234);
- rm.getResourceTrackerService().handleNMContainerStatus(report);
+ rm.getResourceTrackerService().handleNMContainerStatus(report, null);
verify(handler, never()).handle((Event) any());
// Case 1.2: Master container is null
@@ -502,7 +502,7 @@ public class TestResourceTrackerService {
ContainerId.newInstance(currentAttempt.getAppAttemptId(), 0),
ContainerState.COMPLETE, Resource.newInstance(1024, 1),
"Dummy Completed", 0, Priority.newInstance(10), 1234);
- rm.getResourceTrackerService().handleNMContainerStatus(report);
+ rm.getResourceTrackerService().handleNMContainerStatus(report, null);
verify(handler, never()).handle((Event)any());
// Case 2: Managed AM
@@ -515,7 +515,7 @@ public class TestResourceTrackerService {
ContainerState.COMPLETE, Resource.newInstance(1024, 1),
"Dummy Completed", 0, Priority.newInstance(10), 1234);
try {
- rm.getResourceTrackerService().handleNMContainerStatus(report);
+ rm.getResourceTrackerService().handleNMContainerStatus(report, null);
} catch (Exception e) {
// expected - ignore
}
@@ -530,7 +530,7 @@ public class TestResourceTrackerService {
ContainerState.COMPLETE, Resource.newInstance(1024, 1),
"Dummy Completed", 0, Priority.newInstance(10), 1234);
try {
- rm.getResourceTrackerService().handleNMContainerStatus(report);
+ rm.getResourceTrackerService().handleNMContainerStatus(report, null);
} catch (Exception e) {
// expected - ignore
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
index fcb4e45..ba592fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
@@ -98,6 +98,9 @@ public class TestAMRestart {
Thread.sleep(200);
}
+ ContainerId amContainerId = ContainerId.newInstance(am1
+ .getApplicationAttemptId(), 1);
+
// launch the 2nd container, for testing running container transferred.
nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
ContainerId containerId2 =
@@ -196,11 +199,15 @@ public class TestAMRestart {
// completed containerId4 is also transferred to the new attempt.
RMAppAttempt newAttempt =
app1.getRMAppAttempt(am2.getApplicationAttemptId());
- // 4 containers finished, acquired/allocated/reserved/completed.
- waitForContainersToFinish(4, newAttempt);
+ // 4 containers finished, acquired/allocated/reserved/completed + AM
+ // container.
+ waitForContainersToFinish(5, newAttempt);
boolean container3Exists = false, container4Exists = false, container5Exists =
- false, container6Exists = false;
+ false, container6Exists = false, amContainerExists = false;
for(ContainerStatus status : newAttempt.getJustFinishedContainers()) {
+ if(status.getContainerId().equals(amContainerId)) {
+ amContainerExists = true;
+ }
if(status.getContainerId().equals(containerId3)) {
// containerId3 is the container ran by previous attempt but finished by the
// new attempt.
@@ -220,8 +227,11 @@ public class TestAMRestart {
container6Exists = true;
}
}
- Assert.assertTrue(container3Exists && container4Exists && container5Exists
- && container6Exists);
+ Assert.assertTrue(amContainerExists);
+ Assert.assertTrue(container3Exists);
+ Assert.assertTrue(container4Exists);
+ Assert.assertTrue(container5Exists);
+ Assert.assertTrue(container6Exists);
// New SchedulerApplicationAttempt also has the containers info.
rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
@@ -240,14 +250,14 @@ public class TestAMRestart {
// all 4 normal containers finished.
System.out.println("New attempt's just finished containers: "
+ newAttempt.getJustFinishedContainers());
- waitForContainersToFinish(5, newAttempt);
+ waitForContainersToFinish(6, newAttempt);
rm1.stop();
}
private void waitForContainersToFinish(int expectedNum, RMAppAttempt attempt)
throws InterruptedException {
int count = 0;
- while (attempt.getJustFinishedContainers().size() != expectedNum
+ while (attempt.getJustFinishedContainers().size() < expectedNum
&& count < 500) {
Thread.sleep(100);
count++;
[10/50] [abbrv] git commit: HDFS-6948. DN rejects blocks if it has
older UC block. Contributed by Eric Payne.
Posted by cu...@apache.org.
HDFS-6948. DN rejects blocks if it has older UC block. Contributed by
Eric Payne.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f02d934f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f02d934f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f02d934f
Branch: refs/heads/YARN-1051
Commit: f02d934fedf00f0ce43d6f3f9b06d89ccc6851a5
Parents: 6fe5c6b
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Sep 19 08:50:43 2014 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Sep 19 08:50:43 2014 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 14 ++++++++---
.../fsdataset/impl/TestWriteToReplica.java | 25 +++++++++++++++++++-
3 files changed, 38 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f02d934f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 436d2f0..9d76c37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -564,6 +564,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6970. Move startFile EDEK retries to the DFSClient. (wang)
+ HDFS-6948. DN rejects blocks if it has older UC block
+ (Eric Payne via kihwal)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f02d934f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index d28d616..a2179dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1090,9 +1090,17 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
ExtendedBlock b) throws IOException {
ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getBlockId());
if (replicaInfo != null) {
- throw new ReplicaAlreadyExistsException("Block " + b +
- " already exists in state " + replicaInfo.getState() +
- " and thus cannot be created.");
+ if (replicaInfo.getGenerationStamp() < b.getGenerationStamp()
+ && replicaInfo instanceof ReplicaInPipeline) {
+ // Stop the previous writer
+ ((ReplicaInPipeline)replicaInfo)
+ .stopWriter(datanode.getDnConf().getXceiverStopTimeout());
+ invalidate(b.getBlockPoolId(), new Block[]{replicaInfo});
+ } else {
+ throw new ReplicaAlreadyExistsException("Block " + b +
+ " already exists in state " + replicaInfo.getState() +
+ " and thus cannot be created.");
+ }
}
FsVolumeImpl v = volumes.getNextVolume(storageType, b.getNumBytes());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f02d934f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
index e6a03d2..a870aa9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
@@ -111,7 +111,7 @@ public class TestWriteToReplica {
// test writeToTemporary
@Test
- public void testWriteToTempoary() throws Exception {
+ public void testWriteToTemporary() throws Exception {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).build();
try {
cluster.waitActive();
@@ -475,5 +475,28 @@ public class TestWriteToReplica {
}
dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+
+ try {
+ dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+ Assert.fail("Should not have created a replica that had already been "
+ + "created " + blocks[NON_EXISTENT]);
+ } catch (Exception e) {
+ Assert.assertTrue(
+ e.getMessage().contains(blocks[NON_EXISTENT].getBlockName()));
+ Assert.assertTrue(e instanceof ReplicaAlreadyExistsException);
+ }
+
+ long newGenStamp = blocks[NON_EXISTENT].getGenerationStamp() * 10;
+ blocks[NON_EXISTENT].setGenerationStamp(newGenStamp);
+ try {
+ ReplicaInPipeline replicaInfo =
+ dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+ Assert.assertTrue(replicaInfo.getGenerationStamp() == newGenStamp);
+ Assert.assertTrue(
+ replicaInfo.getBlockId() == blocks[NON_EXISTENT].getBlockId());
+ } catch (ReplicaAlreadyExistsException e) {
+ Assert.fail("createRbw() Should have removed the block with the older "
+ + "genstamp and replaced it with the newer one: " + blocks[NON_EXISTENT]);
+ }
}
}
[44/50] [abbrv] git commit: YARN-1709. In-memory data structures used
to track resources over time to enable reservations.
Posted by cu...@apache.org.
YARN-1709. In-memory data structures used to track resources over time to enable reservations.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/36a83267
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/36a83267
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/36a83267
Branch: refs/heads/YARN-1051
Commit: 36a832674fc8a9027d9f50ded17b95240d65a1d1
Parents: 3f1d205
Author: subru <su...@outlook.com>
Authored: Fri Sep 12 17:22:08 2014 -0700
Committer: carlo curino <Carlo Curino>
Committed: Mon Sep 22 16:32:11 2014 -0700
----------------------------------------------------------------------
YARN-1051-CHANGES.txt | 3 +
.../reservation/InMemoryPlan.java | 507 +++++++++++++++++++
.../InMemoryReservationAllocation.java | 151 ++++++
.../resourcemanager/reservation/Plan.java | 32 ++
.../reservation/PlanContext.java | 101 ++++
.../resourcemanager/reservation/PlanEdit.java | 61 +++
.../resourcemanager/reservation/PlanView.java | 89 ++++
.../RLESparseResourceAllocation.java | 332 ++++++++++++
.../reservation/ReservationAllocation.java | 104 ++++
.../reservation/ReservationInterval.java | 67 +++
.../exceptions/PlanningException.java | 25 +
.../reservation/ReservationSystemTestUtil.java | 210 ++++++++
.../reservation/TestInMemoryPlan.java | 477 +++++++++++++++++
.../TestInMemoryReservationAllocation.java | 206 ++++++++
.../TestRLESparseResourceAllocation.java | 169 +++++++
15 files changed, 2534 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/YARN-1051-CHANGES.txt
----------------------------------------------------------------------
diff --git a/YARN-1051-CHANGES.txt b/YARN-1051-CHANGES.txt
index a7c08a0..410d974 100644
--- a/YARN-1051-CHANGES.txt
+++ b/YARN-1051-CHANGES.txt
@@ -5,3 +5,6 @@ YARN-2475. Logic for responding to capacity drops for the
ReservationSystem. (Carlo Curino and Subru Krishnan via curino)
YARN-1708. Public YARN APIs for creating/updating/deleting reservations. (subru)
+
+YARN-1709. In-memory data structures used to track resources over time to
+enable reservations. (subru)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
new file mode 100644
index 0000000..99231c4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
@@ -0,0 +1,507 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.UTCClock;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class InMemoryPlan implements Plan {
+
+ private static final Logger LOG = LoggerFactory.getLogger(InMemoryPlan.class);
+
+ private static final Resource ZERO_RESOURCE = Resource.newInstance(0, 0);
+
+ private TreeMap<ReservationInterval, Set<InMemoryReservationAllocation>> currentReservations =
+ new TreeMap<ReservationInterval, Set<InMemoryReservationAllocation>>();
+
+ private RLESparseResourceAllocation rleSparseVector;
+
+ private Map<String, RLESparseResourceAllocation> userResourceAlloc =
+ new HashMap<String, RLESparseResourceAllocation>();
+
+ private Map<ReservationId, InMemoryReservationAllocation> reservationTable =
+ new HashMap<ReservationId, InMemoryReservationAllocation>();
+
+ private final ReentrantReadWriteLock readWriteLock =
+ new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+ private final SharingPolicy policy;
+ private final ReservationAgent agent;
+ private final long step;
+ private final ResourceCalculator resCalc;
+ private final Resource minAlloc, maxAlloc;
+ private final String queueName;
+ private final QueueMetrics queueMetrics;
+ private final Planner replanner;
+ private final boolean getMoveOnExpiry;
+ private final Clock clock;
+
+ private Resource totalCapacity;
+
+ InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
+ ReservationAgent agent, Resource totalCapacity, long step,
+ ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
+ String queueName, Planner replanner, boolean getMoveOnExpiry) {
+ this(queueMetrics, policy, agent, totalCapacity, step, resCalc, minAlloc,
+ maxAlloc, queueName, replanner, getMoveOnExpiry, new UTCClock());
+ }
+
+ InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
+ ReservationAgent agent, Resource totalCapacity, long step,
+ ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
+ String queueName, Planner replanner, boolean getMoveOnExpiry, Clock clock) {
+ this.queueMetrics = queueMetrics;
+ this.policy = policy;
+ this.agent = agent;
+ this.step = step;
+ this.totalCapacity = totalCapacity;
+ this.resCalc = resCalc;
+ this.minAlloc = minAlloc;
+ this.maxAlloc = maxAlloc;
+ this.rleSparseVector = new RLESparseResourceAllocation(resCalc, minAlloc);
+ this.queueName = queueName;
+ this.replanner = replanner;
+ this.getMoveOnExpiry = getMoveOnExpiry;
+ this.clock = clock;
+ }
+
+ @Override
+ public QueueMetrics getQueueMetrics() {
+ return queueMetrics;
+ }
+
+ private void incrementAllocation(ReservationAllocation reservation) {
+ assert (readWriteLock.isWriteLockedByCurrentThread());
+ Map<ReservationInterval, ReservationRequest> allocationRequests =
+ reservation.getAllocationRequests();
+ // check if we have encountered the user earlier and if not add an entry
+ String user = reservation.getUser();
+ RLESparseResourceAllocation resAlloc = userResourceAlloc.get(user);
+ if (resAlloc == null) {
+ resAlloc = new RLESparseResourceAllocation(resCalc, minAlloc);
+ userResourceAlloc.put(user, resAlloc);
+ }
+ for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests
+ .entrySet()) {
+ resAlloc.addInterval(r.getKey(), r.getValue());
+ rleSparseVector.addInterval(r.getKey(), r.getValue());
+ }
+ }
+
+ private void decrementAllocation(ReservationAllocation reservation) {
+ assert (readWriteLock.isWriteLockedByCurrentThread());
+ Map<ReservationInterval, ReservationRequest> allocationRequests =
+ reservation.getAllocationRequests();
+ String user = reservation.getUser();
+ RLESparseResourceAllocation resAlloc = userResourceAlloc.get(user);
+ for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests
+ .entrySet()) {
+ resAlloc.removeInterval(r.getKey(), r.getValue());
+ rleSparseVector.removeInterval(r.getKey(), r.getValue());
+ }
+ if (resAlloc.isEmpty()) {
+ userResourceAlloc.remove(resAlloc);
+ }
+ }
+
+ public Set<ReservationAllocation> getAllReservations() {
+ readLock.lock();
+ try {
+ if (currentReservations != null) {
+ Set<ReservationAllocation> flattenedReservations =
+ new HashSet<ReservationAllocation>();
+ for (Set<InMemoryReservationAllocation> reservationEntries : currentReservations
+ .values()) {
+ flattenedReservations.addAll(reservationEntries);
+ }
+ return flattenedReservations;
+ } else {
+ return null;
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public boolean addReservation(ReservationAllocation reservation)
+ throws PlanningException {
+ // Verify the allocation is memory based otherwise it is not supported
+ InMemoryReservationAllocation inMemReservation =
+ (InMemoryReservationAllocation) reservation;
+ if (inMemReservation.getUser() == null) {
+ String errMsg =
+ "The specified Reservation with ID "
+ + inMemReservation.getReservationId()
+ + " is not mapped to any user";
+ LOG.error(errMsg);
+ throw new IllegalArgumentException(errMsg);
+ }
+ writeLock.lock();
+ try {
+ if (reservationTable.containsKey(inMemReservation.getReservationId())) {
+ String errMsg =
+ "The specified Reservation with ID "
+ + inMemReservation.getReservationId() + " already exists";
+ LOG.error(errMsg);
+ throw new IllegalArgumentException(errMsg);
+ }
+ // Validate if we can accept this reservation, throws exception if
+ // validation fails
+ policy.validate(this, inMemReservation);
+ // we record here the time in which the allocation has been accepted
+ reservation.setAcceptanceTimestamp(clock.getTime());
+ ReservationInterval searchInterval =
+ new ReservationInterval(inMemReservation.getStartTime(),
+ inMemReservation.getEndTime());
+ Set<InMemoryReservationAllocation> reservations =
+ currentReservations.get(searchInterval);
+ if (reservations == null) {
+ reservations = new HashSet<InMemoryReservationAllocation>();
+ }
+ if (!reservations.add(inMemReservation)) {
+ LOG.error("Unable to add reservation: {} to plan.",
+ inMemReservation.getReservationId());
+ return false;
+ }
+ currentReservations.put(searchInterval, reservations);
+ reservationTable.put(inMemReservation.getReservationId(),
+ inMemReservation);
+ incrementAllocation(inMemReservation);
+ LOG.info("Sucessfully added reservation: {} to plan.",
+ inMemReservation.getReservationId());
+ return true;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public boolean updateReservation(ReservationAllocation reservation)
+ throws PlanningException {
+ writeLock.lock();
+ boolean result = false;
+ try {
+ ReservationId resId = reservation.getReservationId();
+ ReservationAllocation currReservation = getReservationById(resId);
+ if (currReservation == null) {
+ String errMsg =
+ "The specified Reservation with ID " + resId
+ + " does not exist in the plan";
+ LOG.error(errMsg);
+ throw new IllegalArgumentException(errMsg);
+ }
+ if (!removeReservation(currReservation)) {
+ LOG.error("Unable to replace reservation: {} from plan.",
+ reservation.getReservationId());
+ return result;
+ }
+ try {
+ result = addReservation(reservation);
+ } catch (PlanningException e) {
+ LOG.error("Unable to update reservation: {} from plan due to {}.",
+ reservation.getReservationId(), e.getMessage());
+ }
+ if (result) {
+ LOG.info("Sucessfully updated reservation: {} in plan.",
+ reservation.getReservationId());
+ return result;
+ } else {
+ // rollback delete
+ addReservation(currReservation);
+ LOG.info("Rollbacked update reservation: {} from plan.",
+ reservation.getReservationId());
+ return result;
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private boolean removeReservation(ReservationAllocation reservation) {
+ assert (readWriteLock.isWriteLockedByCurrentThread());
+ ReservationInterval searchInterval =
+ new ReservationInterval(reservation.getStartTime(),
+ reservation.getEndTime());
+ Set<InMemoryReservationAllocation> reservations =
+ currentReservations.get(searchInterval);
+ if (reservations != null) {
+ if (!reservations.remove(reservation)) {
+ LOG.error("Unable to remove reservation: {} from plan.",
+ reservation.getReservationId());
+ return false;
+ }
+ if (reservations.isEmpty()) {
+ currentReservations.remove(searchInterval);
+ }
+ } else {
+ String errMsg =
+ "The specified Reservation with ID " + reservation.getReservationId()
+ + " does not exist in the plan";
+ LOG.error(errMsg);
+ throw new IllegalArgumentException(errMsg);
+ }
+ reservationTable.remove(reservation.getReservationId());
+ decrementAllocation(reservation);
+ LOG.info("Sucessfully deleted reservation: {} in plan.",
+ reservation.getReservationId());
+ return true;
+ }
+
+ @Override
+ public boolean deleteReservation(ReservationId reservationID) {
+ writeLock.lock();
+ try {
+ ReservationAllocation reservation = getReservationById(reservationID);
+ if (reservation == null) {
+ String errMsg =
+ "The specified Reservation with ID " + reservationID
+ + " does not exist in the plan";
+ LOG.error(errMsg);
+ throw new IllegalArgumentException(errMsg);
+ }
+ return removeReservation(reservation);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void archiveCompletedReservations(long tick) {
+ // Since we are looking for old reservations, read lock is optimal
+ LOG.debug("Running archival at time: {}", tick);
+ readLock.lock();
+ List<InMemoryReservationAllocation> expiredReservations =
+ new ArrayList<InMemoryReservationAllocation>();
+ // archive reservations and delete the ones which are beyond
+ // the reservation policy "window"
+ try {
+ long archivalTime = tick - policy.getValidWindow();
+ ReservationInterval searchInterval =
+ new ReservationInterval(archivalTime, archivalTime);
+ SortedMap<ReservationInterval, Set<InMemoryReservationAllocation>> reservations =
+ currentReservations.headMap(searchInterval, true);
+ if (!reservations.isEmpty()) {
+ for (Set<InMemoryReservationAllocation> reservationEntries : reservations
+ .values()) {
+ for (InMemoryReservationAllocation reservation : reservationEntries) {
+ if (reservation.getEndTime() <= archivalTime) {
+ expiredReservations.add(reservation);
+ }
+ }
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+ if (expiredReservations.isEmpty()) {
+ return;
+ }
+ // Need write lock only if there are any reservations to be deleted
+ writeLock.lock();
+ try {
+ for (InMemoryReservationAllocation expiredReservation : expiredReservations) {
+ removeReservation(expiredReservation);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public Set<ReservationAllocation> getReservationsAtTime(long tick) {
+ readLock.lock();
+ ReservationInterval searchInterval =
+ new ReservationInterval(tick, Long.MAX_VALUE);
+ try {
+ SortedMap<ReservationInterval, Set<InMemoryReservationAllocation>> reservations =
+ currentReservations.headMap(searchInterval, true);
+ if (!reservations.isEmpty()) {
+ Set<ReservationAllocation> flattenedReservations =
+ new HashSet<ReservationAllocation>();
+ for (Set<InMemoryReservationAllocation> reservationEntries : reservations
+ .values()) {
+ for (InMemoryReservationAllocation reservation : reservationEntries) {
+ if (reservation.getEndTime() > tick) {
+ flattenedReservations.add(reservation);
+ }
+ }
+ }
+ return Collections.unmodifiableSet(flattenedReservations);
+ } else {
+ return Collections.emptySet();
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public long getStep() {
+ return step;
+ }
+
+ @Override
+ public SharingPolicy getSharingPolicy() {
+ return policy;
+ }
+
+ @Override
+ public ReservationAgent getReservationAgent() {
+ return agent;
+ }
+
+ @Override
+ public Resource getConsumptionForUser(String user, long t) {
+ readLock.lock();
+ try {
+ RLESparseResourceAllocation userResAlloc = userResourceAlloc.get(user);
+ if (userResAlloc != null) {
+ return userResAlloc.getCapacityAtTime(t);
+ } else {
+ return Resources.clone(ZERO_RESOURCE);
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Resource getTotalCommittedResources(long t) {
+ readLock.lock();
+ try {
+ return rleSparseVector.getCapacityAtTime(t);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public ReservationAllocation getReservationById(ReservationId reservationID) {
+ if (reservationID == null) {
+ return null;
+ }
+ readLock.lock();
+ try {
+ return reservationTable.get(reservationID);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Resource getTotalCapacity() {
+ readLock.lock();
+ try {
+ return Resources.clone(totalCapacity);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Resource getMinimumAllocation() {
+ return Resources.clone(minAlloc);
+ }
+
+ @Override
+ public void setTotalCapacity(Resource cap) {
+ writeLock.lock();
+ try {
+ totalCapacity = Resources.clone(cap);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public long getEarliestStartTime() {
+ readLock.lock();
+ try {
+ return rleSparseVector.getEarliestStartTime();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public long getLastEndTime() {
+ readLock.lock();
+ try {
+ return rleSparseVector.getLatestEndTime();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public ResourceCalculator getResourceCalculator() {
+ return resCalc;
+ }
+
+ @Override
+ public String getQueueName() {
+ return queueName;
+ }
+
+ @Override
+ public Resource getMaximumAllocation() {
+ return Resources.clone(maxAlloc);
+ }
+
+ public String toCumulativeString() {
+ readLock.lock();
+ try {
+ return rleSparseVector.toString();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Planner getReplanner() {
+ return replanner;
+ }
+
+ @Override
+ public boolean getMoveOnExpiry() {
+ return getMoveOnExpiry;
+ }
+
+ @Override
+ public String toString() {
+ readLock.lock();
+ try {
+ StringBuffer planStr = new StringBuffer("In-memory Plan: ");
+ planStr.append("Parent Queue: ").append(queueName)
+ .append("Total Capacity: ").append(totalCapacity).append("Step: ")
+ .append(step);
+ for (ReservationAllocation reservation : getAllReservations()) {
+ planStr.append(reservation);
+ }
+ return planStr.toString();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
new file mode 100644
index 0000000..10cc55f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
@@ -0,0 +1,151 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * An in memory implementation of a reservation allocation using the
+ * {@link RLESparseResourceAllocation}
+ *
+ */
+class InMemoryReservationAllocation implements ReservationAllocation {
+
+ private final String planName;
+ private final ReservationId reservationID;
+ private final String user;
+ private final ReservationDefinition contract;
+ private final long startTime;
+ private final long endTime;
+ private final Map<ReservationInterval, ReservationRequest> allocationRequests;
+ private boolean hasGang = false;
+ private long acceptedAt = -1;
+
+ private RLESparseResourceAllocation resourcesOverTime;
+
+ InMemoryReservationAllocation(ReservationId reservationID,
+ ReservationDefinition contract, String user, String planName,
+ long startTime, long endTime,
+ Map<ReservationInterval, ReservationRequest> allocationRequests,
+ ResourceCalculator calculator, Resource minAlloc) {
+ this.contract = contract;
+ this.startTime = startTime;
+ this.endTime = endTime;
+ this.reservationID = reservationID;
+ this.user = user;
+ this.allocationRequests = allocationRequests;
+ this.planName = planName;
+ resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc);
+ for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests
+ .entrySet()) {
+ resourcesOverTime.addInterval(r.getKey(), r.getValue());
+ if (r.getValue().getConcurrency() > 1) {
+ hasGang = true;
+ }
+ }
+ }
+
+ @Override
+ public ReservationId getReservationId() {
+ return reservationID;
+ }
+
+ @Override
+ public ReservationDefinition getReservationDefinition() {
+ return contract;
+ }
+
+ @Override
+ public long getStartTime() {
+ return startTime;
+ }
+
+ @Override
+ public long getEndTime() {
+ return endTime;
+ }
+
+ @Override
+ public Map<ReservationInterval, ReservationRequest> getAllocationRequests() {
+ return Collections.unmodifiableMap(allocationRequests);
+ }
+
+ @Override
+ public String getPlanName() {
+ return planName;
+ }
+
+ @Override
+ public String getUser() {
+ return user;
+ }
+
+ @Override
+ public boolean containsGangs() {
+ return hasGang;
+ }
+
+ @Override
+ public void setAcceptanceTimestamp(long acceptedAt) {
+ this.acceptedAt = acceptedAt;
+ }
+
+ @Override
+ public long getAcceptanceTime() {
+ return acceptedAt;
+ }
+
+ @Override
+ public Resource getResourcesAtTime(long tick) {
+ if (tick < startTime || tick >= endTime) {
+ return Resource.newInstance(0, 0);
+ }
+ return Resources.clone(resourcesOverTime.getCapacityAtTime(tick));
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sBuf = new StringBuilder();
+ sBuf.append(getReservationId()).append(" user:").append(getUser())
+ .append(" startTime: ").append(getStartTime()).append(" endTime: ")
+ .append(getEndTime()).append(" alloc:[")
+ .append(resourcesOverTime.toString()).append("] ");
+ return sBuf.toString();
+ }
+
+ @Override
+ public int compareTo(ReservationAllocation other) {
+ // reverse order of acceptance
+ if (this.getAcceptanceTime() > other.getAcceptanceTime()) {
+ return -1;
+ }
+ if (this.getAcceptanceTime() < other.getAcceptanceTime()) {
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return reservationID.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ InMemoryReservationAllocation other = (InMemoryReservationAllocation) obj;
+ return this.reservationID.equals(other.getReservationId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/Plan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/Plan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/Plan.java
new file mode 100644
index 0000000..cf2aed7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/Plan.java
@@ -0,0 +1,32 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+/**
+ * A Plan represents the central data structure of a reservation system that
+ * maintains the "agenda" for the cluster. In particular, it maintains
+ * information on how a set of {@link ReservationDefinition} that have been
+ * previously accepted will be honored.
+ *
+ * {@link ReservationDefinition} submitted by the users through the RM public
+ * APIs are passed to appropriate {@link ReservationAgent}s, which in turn will
+ * consult the Plan (via the {@link PlanView} interface) and try to determine
+ * whether there are sufficient resources available in this Plan to satisfy the
+ * temporal and resource constraints of a {@link ReservationDefinition}. If a
+ * valid allocation is found the agent will try to store it in the plan (via the
+ * {@link PlanEdit} interface). Upon success the system return to the user a
+ * positive acknowledgment, and a reservation identifier to be later used to
+ * access the reserved resources.
+ *
+ * A {@link PlanFollower} will continuously read from the Plan and will
+ * affect the instantaneous allocation of resources among jobs running by
+ * publishing the "current" slice of the Plan to the underlying scheduler. I.e.,
+ * the configuration of queues/weights of the scheduler are modified to reflect
+ * the allocations in the Plan.
+ *
+ * As this interface have several methods we decompose them into three groups:
+ * {@link PlanContext}: containing configuration type information,
+ * {@link PlanView} read-only access to the plan state, and {@link PlanEdit}
+ * write access to the plan state.
+ */
+public interface Plan extends PlanContext, PlanView, PlanEdit {
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanContext.java
new file mode 100644
index 0000000..40a25a6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanContext.java
@@ -0,0 +1,101 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+
+/**
+ * This interface provides read-only access to configuration-type parameter for
+ * a plan.
+ *
+ */
+public interface PlanContext {
+
+ /**
+ * Returns the configured "step" or granularity of time of the plan in millis.
+ *
+ * @return plan step in millis
+ */
+ public long getStep();
+
+ /**
+ * Return the {@link ReservationAgent} configured for this plan that is
+ * responsible for optimally placing various reservation requests
+ *
+ * @return the {@link ReservationAgent} configured for this plan
+ */
+ public ReservationAgent getReservationAgent();
+
+ /**
+ * Return an instance of a {@link Planner}, which will be invoked in response
+ * to unexpected reduction in the resources of this plan
+ *
+ * @return an instance of a {@link Planner}, which will be invoked in response
+ * to unexpected reduction in the resources of this plan
+ */
+ public Planner getReplanner();
+
+ /**
+ * Return the configured {@link SharingPolicy} that governs the sharing of the
+ * resources of the plan between its various users
+ *
+ * @return the configured {@link SharingPolicy} that governs the sharing of
+ * the resources of the plan between its various users
+ */
+ public SharingPolicy getSharingPolicy();
+
+ /**
+ * Returns the system {@link ResourceCalculator}
+ *
+ * @return the system {@link ResourceCalculator}
+ */
+ public ResourceCalculator getResourceCalculator();
+
+ /**
+ * Returns the single smallest {@link Resource} allocation that can be
+ * reserved in this plan
+ *
+ * @return the single smallest {@link Resource} allocation that can be
+ * reserved in this plan
+ */
+ public Resource getMinimumAllocation();
+
+ /**
+ * Returns the single largest {@link Resource} allocation that can be reserved
+ * in this plan
+ *
+ * @return the single largest {@link Resource} allocation that can be reserved
+ * in this plan
+ */
+ public Resource getMaximumAllocation();
+
+ /**
+ * Return the name of the queue in the {@link ResourceScheduler} corresponding
+ * to this plan
+ *
+ * @return the name of the queue in the {@link ResourceScheduler}
+ * corresponding to this plan
+ */
+ public String getQueueName();
+
+ /**
+ * Return the {@link QueueMetrics} for the queue in the
+ * {@link ResourceScheduler} corresponding to this plan
+ *
+ * @return the {@link QueueMetrics} for the queue in the
+ * {@link ResourceScheduler} corresponding to this plan
+ */
+ public QueueMetrics getQueueMetrics();
+
+ /**
+ * Instructs the {@link PlanFollower} on what to do for applications
+ * which are still running when the reservation is expiring (move-to-default
+ * vs kill)
+ *
+ * @return true if remaining applications have to be killed, false if they
+ * have to migrated
+ */
+ public boolean getMoveOnExpiry();
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
new file mode 100644
index 0000000..648edba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
@@ -0,0 +1,61 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+
+/**
+ * This interface groups the methods used to modify the state of a Plan.
+ */
+public interface PlanEdit extends PlanContext, PlanView {
+
+ /**
+ * Add a new {@link ReservationAllocation} to the plan
+ *
+ * @param reservation the {@link ReservationAllocation} to be added to the
+ * plan
+ * @return true if addition is successful, false otherwise
+ */
+ public boolean addReservation(ReservationAllocation reservation)
+ throws PlanningException;
+
+ /**
+ * Updates an existing {@link ReservationAllocation} in the plan. This is
+ * required for re-negotiation
+ *
+ * @param reservation the {@link ReservationAllocation} to be updated the plan
+ * @return true if update is successful, false otherwise
+ */
+ public boolean updateReservation(ReservationAllocation reservation)
+ throws PlanningException;
+
+ /**
+ * Delete an existing {@link ReservationAllocation} from the plan identified
+ * uniquely by its {@link ReservationId}. This will generally be used for
+ * garbage collection
+ *
+ * @param reservation the {@link ReservationAllocation} to be deleted from the
+ * plan identified uniquely by its {@link ReservationId}
+ * @return true if delete is successful, false otherwise
+ */
+ public boolean deleteReservation(ReservationId reservationID)
+ throws PlanningException;
+
+ /**
+ * Method invoked to garbage collect old reservations. It cleans up expired
+ * reservations that have fallen out of the sliding archival window
+ *
+ * @param tick the current time from which the archival window is computed
+ */
+ public void archiveCompletedReservations(long tick) throws PlanningException;
+
+ /**
+ * Sets the overall capacity in terms of {@link Resource} assigned to this
+ * plan
+ *
+ * @param capacity the overall capacity in terms of {@link Resource} assigned
+ * to this plan
+ */
+ public void setTotalCapacity(Resource capacity);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
new file mode 100644
index 0000000..6e58dde
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
@@ -0,0 +1,89 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+/**
+ * This interface provides a read-only view on the allocations made in this
+ * plan. This methods are used for example by {@link ReservationAgent}s to
+ * determine the free resources in a certain point in time, and by
+ * PlanFollowerPolicy to publish this plan to the scheduler.
+ */
+public interface PlanView extends PlanContext {
+
+ /**
+ * Return a {@link ReservationAllocation} identified by its
+ * {@link ReservationId}
+ *
+ * @param reservationID the unique id to identify the
+ * {@link ReservationAllocation}
+ * @return {@link ReservationAllocation} identified by the specified id
+ */
+ public ReservationAllocation getReservationById(ReservationId reservationID);
+
+ /**
+ * Gets all the active reservations at the specified point of time
+ *
+ * @param tick the time (UTC in ms) for which the active reservations are
+ * requested
+ * @return set of active reservations at the specified time
+ */
+ public Set<ReservationAllocation> getReservationsAtTime(long tick);
+
+ /**
+ * Gets all the reservations in the plan
+ *
+ * @return set of all reservations handled by this Plan
+ */
+ public Set<ReservationAllocation> getAllReservations();
+
+ /**
+ * Returns the total {@link Resource} reserved for all users at the specified
+ * time
+ *
+ * @param tick the time (UTC in ms) for which the reserved resources are
+ * requested
+ * @return the total {@link Resource} reserved for all users at the specified
+ * time
+ */
+ public Resource getTotalCommittedResources(long tick);
+
+ /**
+ * Returns the total {@link Resource} reserved for a given user at the
+ * specified time
+ *
+ * @param user the user who made the reservation(s)
+ * @param tick the time (UTC in ms) for which the reserved resources are
+ * requested
+ * @return the total {@link Resource} reserved for a given user at the
+ * specified time
+ */
+ public Resource getConsumptionForUser(String user, long tick);
+
+ /**
+ * Returns the overall capacity in terms of {@link Resource} assigned to this
+ * plan (typically will correspond to the absolute capacity of the
+ * corresponding queue).
+ *
+ * @return the overall capacity in terms of {@link Resource} assigned to this
+ * plan
+ */
+ public Resource getTotalCapacity();
+
+ /**
+ * Gets the time (UTC in ms) at which the first reservation starts
+ *
+ * @return the time (UTC in ms) at which the first reservation starts
+ */
+ public long getEarliestStartTime();
+
+ /**
+ * Returns the time (UTC in ms) at which the last reservation terminates
+ *
+ * @return the time (UTC in ms) at which the last reservation terminates
+ */
+ public long getLastEndTime();
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
new file mode 100644
index 0000000..fa8db30
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
@@ -0,0 +1,332 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import com.google.gson.stream.JsonWriter;
+
+/**
+ * This is a run length encoded sparse data structure that maintains resource
+ * allocations over time
+ */
+public class RLESparseResourceAllocation {
+
+ private static final int THRESHOLD = 100;
+ private static final Resource ZERO_RESOURCE = Resource.newInstance(0, 0);
+
+ private TreeMap<Long, Resource> cumulativeCapacity =
+ new TreeMap<Long, Resource>();
+
+ private final ReentrantReadWriteLock readWriteLock =
+ new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+ private final ResourceCalculator resourceCalculator;
+ private final Resource minAlloc;
+
+ public RLESparseResourceAllocation(ResourceCalculator resourceCalculator,
+ Resource minAlloc) {
+ this.resourceCalculator = resourceCalculator;
+ this.minAlloc = minAlloc;
+ }
+
+ private boolean isSameAsPrevious(Long key, Resource capacity) {
+ Entry<Long, Resource> previous = cumulativeCapacity.lowerEntry(key);
+ return (previous != null && previous.getValue().equals(capacity));
+ }
+
+ private boolean isSameAsNext(Long key, Resource capacity) {
+ Entry<Long, Resource> next = cumulativeCapacity.higherEntry(key);
+ return (next != null && next.getValue().equals(capacity));
+ }
+
+ /**
+ * Add a resource for the specified interval
+ *
+ * @param reservationInterval the interval for which the resource is to be
+ * added
+ * @param capacity the resource to be added
+ * @return true if addition is successful, false otherwise
+ */
+ public boolean addInterval(ReservationInterval reservationInterval,
+ ReservationRequest capacity) {
+ Resource totCap =
+ Resources.multiply(capacity.getCapability(),
+ (float) capacity.getNumContainers());
+ if (totCap.equals(ZERO_RESOURCE)) {
+ return true;
+ }
+ writeLock.lock();
+ try {
+ long startKey = reservationInterval.getStartTime();
+ long endKey = reservationInterval.getEndTime();
+ NavigableMap<Long, Resource> ticks =
+ cumulativeCapacity.headMap(endKey, false);
+ if (ticks != null && !ticks.isEmpty()) {
+ Resource updatedCapacity = Resource.newInstance(0, 0);
+ Entry<Long, Resource> lowEntry = ticks.floorEntry(startKey);
+ if (lowEntry == null) {
+ // This is the earliest starting interval
+ cumulativeCapacity.put(startKey, totCap);
+ } else {
+ updatedCapacity = Resources.add(lowEntry.getValue(), totCap);
+ // Add a new tick only if the updated value is different
+ // from the previous tick
+ if ((startKey == lowEntry.getKey())
+ && (isSameAsPrevious(lowEntry.getKey(), updatedCapacity))) {
+ cumulativeCapacity.remove(lowEntry.getKey());
+ } else {
+ cumulativeCapacity.put(startKey, updatedCapacity);
+ }
+ }
+ // Increase all the capacities of overlapping intervals
+ Set<Entry<Long, Resource>> overlapSet =
+ ticks.tailMap(startKey, false).entrySet();
+ for (Entry<Long, Resource> entry : overlapSet) {
+ updatedCapacity = Resources.add(entry.getValue(), totCap);
+ entry.setValue(updatedCapacity);
+ }
+ } else {
+ // This is the first interval to be added
+ cumulativeCapacity.put(startKey, totCap);
+ }
+ Resource nextTick = cumulativeCapacity.get(endKey);
+ if (nextTick != null) {
+ // If there is overlap, remove the duplicate entry
+ if (isSameAsPrevious(endKey, nextTick)) {
+ cumulativeCapacity.remove(endKey);
+ }
+ } else {
+ // Decrease capacity as this is end of the interval
+ cumulativeCapacity.put(endKey, Resources.subtract(cumulativeCapacity
+ .floorEntry(endKey).getValue(), totCap));
+ }
+ return true;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Add multiple resources for the specified interval
+ *
+ * @param reservationInterval the interval for which the resource is to be
+ * added
+ * @param ReservationRequests the resources to be added
+ * @param clusterResource the total resources in the cluster
+ * @return true if addition is successful, false otherwise
+ */
+ public boolean addCompositeInterval(ReservationInterval reservationInterval,
+ List<ReservationRequest> ReservationRequests, Resource clusterResource) {
+ ReservationRequest aggregateReservationRequest =
+ Records.newRecord(ReservationRequest.class);
+ Resource capacity = Resource.newInstance(0, 0);
+ for (ReservationRequest ReservationRequest : ReservationRequests) {
+ Resources.addTo(capacity, Resources.multiply(
+ ReservationRequest.getCapability(),
+ ReservationRequest.getNumContainers()));
+ }
+ aggregateReservationRequest.setNumContainers((int) Math.ceil(Resources
+ .divide(resourceCalculator, clusterResource, capacity, minAlloc)));
+ aggregateReservationRequest.setCapability(minAlloc);
+
+ return addInterval(reservationInterval, aggregateReservationRequest);
+ }
+
+ /**
+ * Removes a resource for the specified interval
+ *
+ * @param reservationInterval the interval for which the resource is to be
+ * removed
+ * @param capacity the resource to be removed
+ * @return true if removal is successful, false otherwise
+ */
+ public boolean removeInterval(ReservationInterval reservationInterval,
+ ReservationRequest capacity) {
+ Resource totCap =
+ Resources.multiply(capacity.getCapability(),
+ (float) capacity.getNumContainers());
+ if (totCap.equals(ZERO_RESOURCE)) {
+ return true;
+ }
+ writeLock.lock();
+ try {
+ long startKey = reservationInterval.getStartTime();
+ long endKey = reservationInterval.getEndTime();
+ // update the start key
+ NavigableMap<Long, Resource> ticks =
+ cumulativeCapacity.headMap(endKey, false);
+ // Decrease all the capacities of overlapping intervals
+ SortedMap<Long, Resource> overlapSet = ticks.tailMap(startKey);
+ if (overlapSet != null && !overlapSet.isEmpty()) {
+ Resource updatedCapacity = Resource.newInstance(0, 0);
+ long currentKey = -1;
+ for (Iterator<Entry<Long, Resource>> overlapEntries =
+ overlapSet.entrySet().iterator(); overlapEntries.hasNext();) {
+ Entry<Long, Resource> entry = overlapEntries.next();
+ currentKey = entry.getKey();
+ updatedCapacity = Resources.subtract(entry.getValue(), totCap);
+ // update each entry between start and end key
+ cumulativeCapacity.put(currentKey, updatedCapacity);
+ }
+ // Remove the first overlap entry if it is same as previous after
+ // updation
+ Long firstKey = overlapSet.firstKey();
+ if (isSameAsPrevious(firstKey, overlapSet.get(firstKey))) {
+ cumulativeCapacity.remove(firstKey);
+ }
+ // Remove the next entry if it is same as end entry after updation
+ if ((currentKey != -1) && (isSameAsNext(currentKey, updatedCapacity))) {
+ cumulativeCapacity.remove(cumulativeCapacity.higherKey(currentKey));
+ }
+ }
+ return true;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Returns the capacity, i.e. total resources allocated at the specified point
+ * of time
+ *
+ * @param tick the time (UTC in ms) at which the capacity is requested
+ * @return the resources allocated at the specified time
+ */
+ public Resource getCapacityAtTime(long tick) {
+ readLock.lock();
+ try {
+ Entry<Long, Resource> closestStep = cumulativeCapacity.floorEntry(tick);
+ if (closestStep != null) {
+ return Resources.clone(closestStep.getValue());
+ }
+ return Resources.clone(ZERO_RESOURCE);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Get the timestamp of the earliest resource allocation
+ *
+ * @return the timestamp of the first resource allocation
+ */
+ public long getEarliestStartTime() {
+ readLock.lock();
+ try {
+ if (cumulativeCapacity.isEmpty()) {
+ return -1;
+ } else {
+ return cumulativeCapacity.firstKey();
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Get the timestamp of the latest resource allocation
+ *
+ * @return the timestamp of the last resource allocation
+ */
+ public long getLatestEndTime() {
+ readLock.lock();
+ try {
+ if (cumulativeCapacity.isEmpty()) {
+ return -1;
+ } else {
+ return cumulativeCapacity.lastKey();
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Returns true if there are no non-zero entries
+ *
+ * @return true if there are no allocations or false otherwise
+ */
+ public boolean isEmpty() {
+ readLock.lock();
+ try {
+ if (cumulativeCapacity.isEmpty()) {
+ return true;
+ }
+ // Deletion leaves a single zero entry so check for that
+ if (cumulativeCapacity.size() == 1) {
+ return cumulativeCapacity.firstEntry().getValue().equals(ZERO_RESOURCE);
+ }
+ return false;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder ret = new StringBuilder();
+ readLock.lock();
+ try {
+ if (cumulativeCapacity.size() > THRESHOLD) {
+ ret.append("Number of steps: ").append(cumulativeCapacity.size())
+ .append(" earliest entry: ").append(cumulativeCapacity.firstKey())
+ .append(" latest entry: ").append(cumulativeCapacity.lastKey());
+ } else {
+ for (Map.Entry<Long, Resource> r : cumulativeCapacity.entrySet()) {
+ ret.append(r.getKey()).append(": ").append(r.getValue())
+ .append("\n ");
+ }
+ }
+ return ret.toString();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Returns the JSON string representation of the current resources allocated
+ * over time
+ *
+ * @return the JSON string representation of the current resources allocated
+ * over time
+ */
+ public String toMemJSONString() {
+ StringWriter json = new StringWriter();
+ JsonWriter jsonWriter = new JsonWriter(json);
+ readLock.lock();
+ try {
+ jsonWriter.beginObject();
+ // jsonWriter.name("timestamp").value("resource");
+ for (Map.Entry<Long, Resource> r : cumulativeCapacity.entrySet()) {
+ jsonWriter.name(r.getKey().toString()).value(r.getValue().toString());
+ }
+ jsonWriter.endObject();
+ jsonWriter.close();
+ return json.toString();
+ } catch (IOException e) {
+ // This should not happen
+ return "";
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
new file mode 100644
index 0000000..bca3aa8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
@@ -0,0 +1,104 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+/**
+ * A ReservationAllocation represents a concrete allocation of resources over
+ * time that satisfy a certain {@link ReservationDefinition}. This is used
+ * internally by a {@link Plan} to store information about how each of the
+ * accepted {@link ReservationDefinition} have been allocated.
+ */
+public interface ReservationAllocation extends
+ Comparable<ReservationAllocation> {
+
+ /**
+ * Returns the unique identifier {@link ReservationId} that represents the
+ * reservation
+ *
+ * @return reservationId the unique identifier {@link ReservationId} that
+ * represents the reservation
+ */
+ public ReservationId getReservationId();
+
+ /**
+ * Returns the original {@link ReservationDefinition} submitted by the client
+ *
+ * @return
+ */
+ public ReservationDefinition getReservationDefinition();
+
+ /**
+ * Returns the time at which the reservation is activated
+ *
+ * @return the time at which the reservation is activated
+ */
+ public long getStartTime();
+
+ /**
+ * Returns the time at which the reservation terminates
+ *
+ * @return the time at which the reservation terminates
+ */
+ public long getEndTime();
+
+ /**
+ * Returns the map of resources requested against the time interval for which
+ * they were
+ *
+ * @return the allocationRequests the map of resources requested against the
+ * time interval for which they were
+ */
+ public Map<ReservationInterval, ReservationRequest> getAllocationRequests();
+
+ /**
+ * Return a string identifying the plan to which the reservation belongs
+ *
+ * @return the plan to which the reservation belongs
+ */
+ public String getPlanName();
+
+ /**
+ * Returns the user who requested the reservation
+ *
+ * @return the user who requested the reservation
+ */
+ public String getUser();
+
+ /**
+ * Returns whether the reservation has gang semantics or not
+ *
+ * @return true if there is a gang request, false otherwise
+ */
+ public boolean containsGangs();
+
+ /**
+ * Sets the time at which the reservation was accepted by the system
+ *
+ * @param acceptedAt the time at which the reservation was accepted by the
+ * system
+ */
+ public void setAcceptanceTimestamp(long acceptedAt);
+
+ /**
+ * Returns the time at which the reservation was accepted by the system
+ *
+ * @return the time at which the reservation was accepted by the system
+ */
+ public long getAcceptanceTime();
+
+ /**
+ * Returns the capacity represented by cumulative resources reserved by the
+ * reservation at the specified point of time
+ *
+ * @param tick the time (UTC in ms) for which the reserved resources are
+ * requested
+ * @return the resources reserved at the specified time
+ */
+ public Resource getResourcesAtTime(long tick);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInterval.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInterval.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInterval.java
new file mode 100644
index 0000000..d3a6d51
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInterval.java
@@ -0,0 +1,67 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+/**
+ * This represents the time duration of the reservation
+ *
+ */
+public class ReservationInterval implements Comparable<ReservationInterval> {
+
+ private final long startTime;
+
+ private final long endTime;
+
+ public ReservationInterval(long startTime, long endTime) {
+ this.startTime = startTime;
+ this.endTime = endTime;
+ }
+
+ /**
+ * Get the start time of the reservation interval
+ *
+ * @return the startTime
+ */
+ public long getStartTime() {
+ return startTime;
+ }
+
+ /**
+ * Get the end time of the reservation interval
+ *
+ * @return the endTime
+ */
+ public long getEndTime() {
+ return endTime;
+ }
+
+ /**
+ * Returns whether the interval is active at the specified instant of time
+ *
+ * @param tick the instance of the time to check
+ * @return true if active, false otherwise
+ */
+ public boolean isOverlap(long tick) {
+ return (startTime <= tick && tick <= endTime);
+ }
+
+ @Override
+ public int compareTo(ReservationInterval anotherInterval) {
+ long diff = 0;
+ if (startTime == anotherInterval.getStartTime()) {
+ diff = endTime - anotherInterval.getEndTime();
+ } else {
+ diff = startTime - anotherInterval.getStartTime();
+ }
+ if (diff < 0) {
+ return -1;
+ } else if (diff > 0) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ public String toString() {
+ return "[" + startTime + ", " + endTime + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningException.java
new file mode 100644
index 0000000..aa9e9fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningException.java
@@ -0,0 +1,25 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+
+/**
+ * Exception thrown by the admission control subsystem when there is a problem
+ * in trying to find an allocation for a user {@link ReservationSubmissionRequest}.
+ */
+public class PlanningException extends Exception {
+
+ private static final long serialVersionUID = -684069387367879218L;
+
+ public PlanningException(String message) {
+ super(message);
+ }
+
+ public PlanningException(Throwable cause) {
+ super(cause);
+ }
+
+ public PlanningException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
new file mode 100644
index 0000000..cbca6dc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
@@ -0,0 +1,210 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+public class ReservationSystemTestUtil {
+
+ private static Random rand = new Random();
+
+ public final static String reservationQ = "dedicated";
+
+ public static ReservationId getNewReservationId() {
+ return ReservationId.newInstance(rand.nextLong(), rand.nextLong());
+ }
+
+ public CapacityScheduler mockCapacityScheduler(int numContainers)
+ throws IOException {
+ // stolen from TestCapacityScheduler
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(conf);
+
+ CapacityScheduler cs = Mockito.spy(new CapacityScheduler());
+ cs.setConf(new YarnConfiguration());
+ RMContext mockRmContext =
+ Mockito.spy(new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(conf),
+ new NMTokenSecretManagerInRM(conf),
+ new ClientToAMTokenSecretManagerInRM(), null));
+ cs.setRMContext(mockRmContext);
+ try {
+ cs.serviceInit(conf);
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ when(mockRmContext.getScheduler()).thenReturn(cs);
+ Resource r = Resource.newInstance(numContainers * 1024, numContainers);
+ doReturn(r).when(cs).getClusterResource();
+ return cs;
+ }
+
+ public static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
+ // Define default queue
+ final String defQ = CapacitySchedulerConfiguration.ROOT + ".default";
+ conf.setCapacity(defQ, 10);
+
+ // Define top-level queues
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {
+ "default", "a", reservationQ });
+
+ final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+ conf.setCapacity(A, 10);
+
+ final String dedicated =
+ CapacitySchedulerConfiguration.ROOT
+ + CapacitySchedulerConfiguration.DOT + reservationQ;
+ conf.setCapacity(dedicated, 80);
+ // Set as reservation queue
+ conf.setReservableQueue(dedicated, true);
+
+ // Define 2nd-level queues
+ final String A1 = A + ".a1";
+ final String A2 = A + ".a2";
+ conf.setQueues(A, new String[] { "a1", "a2" });
+ conf.setCapacity(A1, 30);
+ conf.setCapacity(A2, 70);
+ }
+
+ public String getFullReservationQueueName() {
+ return CapacitySchedulerConfiguration.ROOT
+ + CapacitySchedulerConfiguration.DOT + reservationQ;
+ }
+
+ public String getreservationQueueName() {
+ return reservationQ;
+ }
+
+ public void updateQueueConfiguration(CapacitySchedulerConfiguration conf,
+ String newQ) {
+ // Define default queue
+ final String prefix =
+ CapacitySchedulerConfiguration.ROOT
+ + CapacitySchedulerConfiguration.DOT;
+ final String defQ = prefix + "default";
+ conf.setCapacity(defQ, 5);
+
+ // Define top-level queues
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {
+ "default", "a", reservationQ, newQ });
+
+ final String A = prefix + "a";
+ conf.setCapacity(A, 5);
+
+ final String dedicated = prefix + reservationQ;
+ conf.setCapacity(dedicated, 80);
+ // Set as reservation queue
+ conf.setReservableQueue(dedicated, true);
+
+ conf.setCapacity(prefix + newQ, 10);
+ // Set as reservation queue
+ conf.setReservableQueue(prefix + newQ, true);
+
+ // Define 2nd-level queues
+ final String A1 = A + ".a1";
+ final String A2 = A + ".a2";
+ conf.setQueues(A, new String[] { "a1", "a2" });
+ conf.setCapacity(A1, 30);
+ conf.setCapacity(A2, 70);
+ }
+
+ public static ReservationDefinition generateRandomRR(Random rand, long i) {
+ rand.setSeed(i);
+ long now = System.currentTimeMillis();
+
+ // start time at random in the next 12 hours
+ long arrival = rand.nextInt(12 * 3600 * 1000);
+ // deadline at random in the next day
+ long deadline = arrival + rand.nextInt(24 * 3600 * 1000);
+
+ // create a request with a single atomic ask
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(now + arrival);
+ rr.setDeadline(now + deadline);
+
+ int gang = 1 + rand.nextInt(9);
+ int par = (rand.nextInt(1000) + 1) * gang;
+ long dur = rand.nextInt(2 * 3600 * 1000); // random duration within 2h
+ ReservationRequest r =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1), par,
+ gang, dur);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setReservationResources(Collections.singletonList(r));
+ rand.nextInt(3);
+ ReservationRequestInterpreter[] type =
+ ReservationRequestInterpreter.values();
+ reqs.setInterpreter(type[rand.nextInt(type.length)]);
+ rr.setReservationRequests(reqs);
+
+ return rr;
+
+ }
+
+ public static ReservationDefinition generateBigRR(Random rand, long i) {
+ rand.setSeed(i);
+ long now = System.currentTimeMillis();
+
+ // start time at random in the next 2 hours
+ long arrival = rand.nextInt(2 * 3600 * 1000);
+ // deadline at random in the next day
+ long deadline = rand.nextInt(24 * 3600 * 1000);
+
+ // create a request with a single atomic ask
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(now + arrival);
+ rr.setDeadline(now + deadline);
+
+ int gang = 1;
+ int par = 100000; // 100k tasks
+ long dur = rand.nextInt(60 * 1000); // 1min tasks
+ ReservationRequest r =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1), par,
+ gang, dur);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setReservationResources(Collections.singletonList(r));
+ rand.nextInt(3);
+ ReservationRequestInterpreter[] type =
+ ReservationRequestInterpreter.values();
+ reqs.setInterpreter(type[rand.nextInt(type.length)]);
+ rr.setReservationRequests(reqs);
+
+ return rr;
+ }
+
+ public static Map<ReservationInterval, ReservationRequest> generateAllocation(
+ long startTime, long step, int[] alloc) {
+ Map<ReservationInterval, ReservationRequest> req =
+ new TreeMap<ReservationInterval, ReservationRequest>();
+ for (int i = 0; i < alloc.length; i++) {
+ req.put(new ReservationInterval(startTime + i * step, startTime + (i + 1)
+ * step), ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), alloc[i]));
+ }
+ return req;
+ }
+
+}
[48/50] [abbrv] YARN-2080. Integrating reservation system with
ResourceManager and client-RM protocol. Contributed by Subru Krishnan and
Carlo Curino.
Posted by cu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
index 7e0b89e..a54e4bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
@@ -19,25 +19,33 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
public class AppAddedSchedulerEvent extends SchedulerEvent {
private final ApplicationId applicationId;
private final String queue;
private final String user;
+ private final ReservationId reservationID;
private final boolean isAppRecovering;
public AppAddedSchedulerEvent(
ApplicationId applicationId, String queue, String user) {
- this(applicationId, queue, user, false);
+ this(applicationId, queue, user, false, null);
}
public AppAddedSchedulerEvent(ApplicationId applicationId, String queue,
- String user, boolean isAppRecovering) {
+ String user, ReservationId reservationID) {
+ this(applicationId, queue, user, false, reservationID);
+ }
+
+ public AppAddedSchedulerEvent(ApplicationId applicationId, String queue,
+ String user, boolean isAppRecovering, ReservationId reservationID) {
super(SchedulerEventType.APP_ADDED);
this.applicationId = applicationId;
this.queue = queue;
this.user = user;
+ this.reservationID = reservationID;
this.isAppRecovering = isAppRecovering;
}
@@ -56,4 +64,8 @@ public class AppAddedSchedulerEvent extends SchedulerEvent {
public boolean getIsAppRecovering() {
return isAppRecovering;
}
+
+ public ReservationId getReservationID() {
+ return reservationID;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index db867a9..954e21d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -33,6 +33,7 @@ import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -74,6 +75,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -91,7 +98,12 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -107,6 +119,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWri
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
@@ -117,11 +130,16 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.UTCClock;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -1199,4 +1217,102 @@ public class TestClientRMService {
when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
return yarnScheduler;
}
+
+ @Test
+ public void testReservationAPIs() {
+ // initialize
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ ReservationSystemTestUtil.setupQueueConfiguration(conf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
+ MockRM rm = new MockRM(conf);
+ rm.start();
+ MockNM nm;
+ try {
+ nm = rm.registerNode("127.0.0.1:0", 102400, 100);
+ // allow plan follower to synchronize
+ Thread.sleep(1050);
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Create a client.
+ ClientRMService clientService = rm.getClientRMService();
+
+ // create a reservation
+ Clock clock = new UTCClock();
+ long arrival = clock.getTime();
+ long duration = 60000;
+ long deadline = (long) (arrival + 1.05 * duration);
+ ReservationSubmissionRequest sRequest =
+ createSimpleReservationRequest(4, arrival, deadline, duration);
+ ReservationSubmissionResponse sResponse = null;
+ try {
+ sResponse = clientService.submitReservation(sRequest);
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(sResponse);
+ ReservationId reservationID = sResponse.getReservationId();
+ Assert.assertNotNull(reservationID);
+ LOG.info("Submit reservation response: " + reservationID);
+
+ // Update the reservation
+ ReservationDefinition rDef = sRequest.getReservationDefinition();
+ ReservationRequest rr =
+ rDef.getReservationRequests().getReservationResources().get(0);
+ rr.setNumContainers(5);
+ arrival = clock.getTime();
+ duration = 30000;
+ deadline = (long) (arrival + 1.05 * duration);
+ rr.setDuration(duration);
+ rDef.setArrival(arrival);
+ rDef.setDeadline(deadline);
+ ReservationUpdateRequest uRequest =
+ ReservationUpdateRequest.newInstance(rDef, reservationID);
+ ReservationUpdateResponse uResponse = null;
+ try {
+ uResponse = clientService.updateReservation(uRequest);
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(sResponse);
+ LOG.info("Update reservation response: " + uResponse);
+
+ // Delete the reservation
+ ReservationDeleteRequest dRequest =
+ ReservationDeleteRequest.newInstance(reservationID);
+ ReservationDeleteResponse dResponse = null;
+ try {
+ dResponse = clientService.deleteReservation(dRequest);
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(sResponse);
+ LOG.info("Delete reservation response: " + dResponse);
+
+ // clean-up
+ rm.stop();
+ nm = null;
+ rm = null;
+ }
+
+ private ReservationSubmissionRequest createSimpleReservationRequest(
+ int numContainers, long arrival, long deadline, long duration) {
+ // create a request with a single atomic ask
+ ReservationRequest r =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ numContainers, 1, duration);
+ ReservationRequests reqs =
+ ReservationRequests.newInstance(Collections.singletonList(r),
+ ReservationRequestInterpreter.R_ALL);
+ ReservationDefinition rDef =
+ ReservationDefinition.newInstance(arrival, deadline, reqs,
+ "testClientRMService#reservation");
+ ReservationSubmissionRequest request =
+ ReservationSubmissionRequest.newInstance(rDef,
+ ReservationSystemTestUtil.reservationQ);
+ return request;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java
new file mode 100644
index 0000000..2a77791
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java
@@ -0,0 +1,102 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestCapacityReservationSystem {
+
+ @Test
+ public void testInitialize() {
+ ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+ CapacityScheduler capScheduler = null;
+ try {
+ capScheduler = testUtil.mockCapacityScheduler(10);
+ } catch (IOException e) {
+ Assert.fail(e.getMessage());
+ }
+ CapacityReservationSystem reservationSystem =
+ new CapacityReservationSystem();
+ reservationSystem.setRMContext(capScheduler.getRMContext());
+ try {
+ reservationSystem.reinitialize(capScheduler.getConf(),
+ capScheduler.getRMContext());
+ } catch (YarnException e) {
+ Assert.fail(e.getMessage());
+ }
+ String planQName = testUtil.getreservationQueueName();
+ Plan plan = reservationSystem.getPlan(planQName);
+ Assert.assertNotNull(plan);
+ Assert.assertTrue(plan instanceof InMemoryPlan);
+ Assert.assertEquals(planQName, plan.getQueueName());
+ Assert.assertEquals(8192, plan.getTotalCapacity().getMemory());
+ Assert
+ .assertTrue(plan.getReservationAgent() instanceof GreedyReservationAgent);
+ Assert
+ .assertTrue(plan.getSharingPolicy() instanceof CapacityOverTimePolicy);
+ }
+
+ @Test
+ public void testReinitialize() {
+ ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+ CapacityScheduler capScheduler = null;
+ try {
+ capScheduler = testUtil.mockCapacityScheduler(10);
+ } catch (IOException e) {
+ Assert.fail(e.getMessage());
+ }
+ CapacityReservationSystem reservationSystem =
+ new CapacityReservationSystem();
+ CapacitySchedulerConfiguration conf = capScheduler.getConfiguration();
+ RMContext mockContext = capScheduler.getRMContext();
+ reservationSystem.setRMContext(mockContext);
+ try {
+ reservationSystem.reinitialize(capScheduler.getConfiguration(),
+ mockContext);
+ } catch (YarnException e) {
+ Assert.fail(e.getMessage());
+ }
+ // Assert queue in original config
+ String planQName = testUtil.getreservationQueueName();
+ Plan plan = reservationSystem.getPlan(planQName);
+ Assert.assertNotNull(plan);
+ Assert.assertTrue(plan instanceof InMemoryPlan);
+ Assert.assertEquals(planQName, plan.getQueueName());
+ Assert.assertEquals(8192, plan.getTotalCapacity().getMemory());
+ Assert
+ .assertTrue(plan.getReservationAgent() instanceof GreedyReservationAgent);
+ Assert
+ .assertTrue(plan.getSharingPolicy() instanceof CapacityOverTimePolicy);
+
+ // Dynamically add a plan
+ String newQ = "reservation";
+ Assert.assertNull(reservationSystem.getPlan(newQ));
+ testUtil.updateQueueConfiguration(conf, newQ);
+ try {
+ capScheduler.reinitialize(conf, mockContext);
+ } catch (IOException e) {
+ Assert.fail(e.getMessage());
+ }
+ try {
+ reservationSystem.reinitialize(conf, mockContext);
+ } catch (YarnException e) {
+ Assert.fail(e.getMessage());
+ }
+ Plan newPlan = reservationSystem.getPlan(newQ);
+ Assert.assertNotNull(newPlan);
+ Assert.assertTrue(newPlan instanceof InMemoryPlan);
+ Assert.assertEquals(newQ, newPlan.getQueueName());
+ Assert.assertEquals(1024, newPlan.getTotalCapacity().getMemory());
+ Assert
+ .assertTrue(newPlan.getReservationAgent() instanceof GreedyReservationAgent);
+ Assert
+ .assertTrue(newPlan.getSharingPolicy() instanceof CapacityOverTimePolicy);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
new file mode 100644
index 0000000..f5917bb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
@@ -0,0 +1,560 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.text.MessageFormat;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationDeleteRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationSubmissionRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationUpdateRequestPBImpl;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestReservationInputValidator {
+
+ private static final Log LOG = LogFactory
+ .getLog(TestReservationInputValidator.class);
+
+ private static final String PLAN_NAME = "test-reservation";
+
+ private Clock clock;
+ private Map<String, Plan> plans = new HashMap<String, Plan>(1);
+ private ReservationSystem rSystem;
+ private Plan plan;
+
+ private ReservationInputValidator rrValidator;
+
+ @Before
+ public void setUp() {
+ clock = mock(Clock.class);
+ plan = mock(Plan.class);
+ rSystem = mock(ReservationSystem.class);
+ plans.put(PLAN_NAME, plan);
+ rrValidator = new ReservationInputValidator(clock);
+ when(clock.getTime()).thenReturn(1L);
+ ResourceCalculator rCalc = new DefaultResourceCalculator();
+ Resource resource = Resource.newInstance(10240, 10);
+ when(plan.getResourceCalculator()).thenReturn(rCalc);
+ when(plan.getTotalCapacity()).thenReturn(resource);
+ when(rSystem.getQueueForReservation(any(ReservationId.class))).thenReturn(
+ PLAN_NAME);
+ when(rSystem.getPlan(PLAN_NAME)).thenReturn(plan);
+ }
+
+ @After
+ public void tearDown() {
+ rrValidator = null;
+ clock = null;
+ plan = null;
+ }
+
+ @Test
+ public void testSubmitReservationNormal() {
+ ReservationSubmissionRequest request =
+ createSimpleReservationSubmissionRequest(1, 1, 1, 5, 3);
+ Plan plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ } catch (YarnException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(plan);
+ }
+
+ @Test
+ public void testSubmitReservationDoesnotExist() {
+ ReservationSubmissionRequest request =
+ new ReservationSubmissionRequestPBImpl();
+ Plan plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .equals("The queue to submit is not specified. Please try again with a valid reservable queue."));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testSubmitReservationInvalidPlan() {
+ ReservationSubmissionRequest request =
+ createSimpleReservationSubmissionRequest(1, 1, 1, 5, 3);
+ when(rSystem.getPlan(PLAN_NAME)).thenReturn(null);
+ Plan plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .endsWith(" is not managed by reservation system. Please try again with a valid reservable queue."));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testSubmitReservationNoDefinition() {
+ ReservationSubmissionRequest request =
+ new ReservationSubmissionRequestPBImpl();
+ request.setQueue(PLAN_NAME);
+ Plan plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .equals("Missing reservation definition. Please try again by specifying a reservation definition."));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testSubmitReservationInvalidDeadline() {
+ ReservationSubmissionRequest request =
+ createSimpleReservationSubmissionRequest(1, 1, 1, 0, 3);
+ Plan plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert.assertTrue(message
+ .startsWith("The specified deadline: 0 is the past"));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testSubmitReservationInvalidRR() {
+ ReservationSubmissionRequest request =
+ createSimpleReservationSubmissionRequest(0, 0, 1, 5, 3);
+ Plan plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert.assertTrue(message
+ .startsWith("No resources have been specified to reserve"));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testSubmitReservationEmptyRR() {
+ ReservationSubmissionRequest request =
+ createSimpleReservationSubmissionRequest(1, 0, 1, 5, 3);
+ Plan plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert.assertTrue(message
+ .startsWith("No resources have been specified to reserve"));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testSubmitReservationInvalidDuration() {
+ ReservationSubmissionRequest request =
+ createSimpleReservationSubmissionRequest(1, 1, 1, 3, 4);
+ Plan plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert.assertTrue(message.startsWith("The time difference"));
+ Assert
+ .assertTrue(message
+ .contains("must be greater or equal to the minimum resource duration"));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testSubmitReservationExceedsGangSize() {
+ ReservationSubmissionRequest request =
+ createSimpleReservationSubmissionRequest(1, 1, 1, 5, 4);
+ Resource resource = Resource.newInstance(512, 1);
+ when(plan.getTotalCapacity()).thenReturn(resource);
+ Plan plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .startsWith("The size of the largest gang in the reservation refinition"));
+ Assert.assertTrue(message.contains("exceed the capacity available "));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testUpdateReservationNormal() {
+ ReservationUpdateRequest request =
+ createSimpleReservationUpdateRequest(1, 1, 1, 5, 3);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationUpdateRequest(rSystem, request);
+ } catch (YarnException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(plan);
+ }
+
+ @Test
+ public void testUpdateReservationNoID() {
+ ReservationUpdateRequest request = new ReservationUpdateRequestPBImpl();
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationUpdateRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .startsWith("Missing reservation id. Please try again by specifying a reservation id."));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testUpdateReservationDoesnotExist() {
+ ReservationUpdateRequest request =
+ createSimpleReservationUpdateRequest(1, 1, 1, 5, 4);
+ ReservationId rId = request.getReservationId();
+ when(rSystem.getQueueForReservation(rId)).thenReturn(null);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationUpdateRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message.equals(MessageFormat
+ .format(
+ "The specified reservation with ID: {0} is unknown. Please try again with a valid reservation.",
+ rId)));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testUpdateReservationInvalidPlan() {
+ ReservationUpdateRequest request =
+ createSimpleReservationUpdateRequest(1, 1, 1, 5, 4);
+ when(rSystem.getPlan(PLAN_NAME)).thenReturn(null);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationUpdateRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .endsWith(" is not associated with any valid plan. Please try again with a valid reservation."));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testUpdateReservationNoDefinition() {
+ ReservationUpdateRequest request = new ReservationUpdateRequestPBImpl();
+ request.setReservationId(ReservationSystemTestUtil.getNewReservationId());
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationUpdateRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .startsWith("Missing reservation definition. Please try again by specifying a reservation definition."));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testUpdateReservationInvalidDeadline() {
+ ReservationUpdateRequest request =
+ createSimpleReservationUpdateRequest(1, 1, 1, 0, 3);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationUpdateRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert.assertTrue(message
+ .startsWith("The specified deadline: 0 is the past"));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testUpdateReservationInvalidRR() {
+ ReservationUpdateRequest request =
+ createSimpleReservationUpdateRequest(0, 0, 1, 5, 3);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationUpdateRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert.assertTrue(message
+ .startsWith("No resources have been specified to reserve"));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testUpdateReservationEmptyRR() {
+ ReservationUpdateRequest request =
+ createSimpleReservationUpdateRequest(1, 0, 1, 5, 3);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationUpdateRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert.assertTrue(message
+ .startsWith("No resources have been specified to reserve"));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testUpdateReservationInvalidDuration() {
+ ReservationUpdateRequest request =
+ createSimpleReservationUpdateRequest(1, 1, 1, 3, 4);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationUpdateRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .contains("must be greater or equal to the minimum resource duration"));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testUpdateReservationExceedsGangSize() {
+ ReservationUpdateRequest request =
+ createSimpleReservationUpdateRequest(1, 1, 1, 5, 4);
+ Resource resource = Resource.newInstance(512, 1);
+ when(plan.getTotalCapacity()).thenReturn(resource);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationUpdateRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .startsWith("The size of the largest gang in the reservation refinition"));
+ Assert.assertTrue(message.contains("exceed the capacity available "));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testDeleteReservationNormal() {
+ ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ request.setReservationId(reservationID);
+ ReservationAllocation reservation = mock(ReservationAllocation.class);
+ when(plan.getReservationById(reservationID)).thenReturn(reservation);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
+ } catch (YarnException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(plan);
+ }
+
+ @Test
+ public void testDeleteReservationNoID() {
+ ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .startsWith("Missing reservation id. Please try again by specifying a reservation id."));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testDeleteReservationDoesnotExist() {
+ ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
+ ReservationId rId = ReservationSystemTestUtil.getNewReservationId();
+ request.setReservationId(rId);
+ when(rSystem.getQueueForReservation(rId)).thenReturn(null);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message.equals(MessageFormat
+ .format(
+ "The specified reservation with ID: {0} is unknown. Please try again with a valid reservation.",
+ rId)));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testDeleteReservationInvalidPlan() {
+ ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ request.setReservationId(reservationID);
+ when(rSystem.getPlan(PLAN_NAME)).thenReturn(null);
+ Plan plan = null;
+ try {
+ plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert
+ .assertTrue(message
+ .endsWith(" is not associated with any valid plan. Please try again with a valid reservation."));
+ LOG.info(message);
+ }
+ }
+
+ private ReservationSubmissionRequest createSimpleReservationSubmissionRequest(
+ int numRequests, int numContainers, long arrival, long deadline,
+ long duration) {
+ // create a request with a single atomic ask
+ ReservationSubmissionRequest request =
+ new ReservationSubmissionRequestPBImpl();
+ ReservationDefinition rDef = new ReservationDefinitionPBImpl();
+ rDef.setArrival(arrival);
+ rDef.setDeadline(deadline);
+ if (numRequests > 0) {
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ rDef.setReservationRequests(reqs);
+ if (numContainers > 0) {
+ ReservationRequest r =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ numContainers, 1, duration);
+
+ reqs.setReservationResources(Collections.singletonList(r));
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
+ }
+ }
+ request.setQueue(PLAN_NAME);
+ request.setReservationDefinition(rDef);
+ return request;
+ }
+
+ private ReservationUpdateRequest createSimpleReservationUpdateRequest(
+ int numRequests, int numContainers, long arrival, long deadline,
+ long duration) {
+ // create a request with a single atomic ask
+ ReservationUpdateRequest request = new ReservationUpdateRequestPBImpl();
+ ReservationDefinition rDef = new ReservationDefinitionPBImpl();
+ rDef.setArrival(arrival);
+ rDef.setDeadline(deadline);
+ if (numRequests > 0) {
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ rDef.setReservationRequests(reqs);
+ if (numContainers > 0) {
+ ReservationRequest r =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ numContainers, 1, duration);
+
+ reqs.setReservationResources(Collections.singletonList(r));
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
+ }
+ }
+ request.setReservationDefinition(rDef);
+ request.setReservationId(ReservationSystemTestUtil.getNewReservationId());
+ return request;
+ }
+
+}
[27/50] [abbrv] git commit: YARN-2452. TestRMApplicationHistoryWriter
fails with FairScheduler. (Zhihai Xu via kasha)
Posted by cu...@apache.org.
YARN-2452. TestRMApplicationHistoryWriter fails with FairScheduler. (Zhihai Xu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c50fc925
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c50fc925
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c50fc925
Branch: refs/heads/YARN-1051
Commit: c50fc92502934aa2a8f84ea2466d4da1e3eace9d
Parents: b6fd5cc
Author: Karthik Kambatla <ka...@apache.org>
Authored: Sun Sep 21 13:14:01 2014 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Sun Sep 21 13:15:04 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../fair/FairSchedulerConfiguration.java | 2 +-
.../ahs/TestRMApplicationHistoryWriter.java | 24 +++++++++++++++++++-
3 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c50fc925/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9c484c7..3052ed3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -406,6 +406,9 @@ Release 2.6.0 - UNRELEASED
YARN-2460. Remove obsolete entries from yarn-default.xml (Ray Chiang via
aw)
+ YARN-2452. TestRMApplicationHistoryWriter fails with FairScheduler.
+ (Zhihai Xu via kasha)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c50fc925/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 473c369..a96952d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -112,7 +112,7 @@ public class FairSchedulerConfiguration extends Configuration {
protected static final int DEFAULT_WAIT_TIME_BEFORE_KILL = 15000;
/** Whether to assign multiple containers in one check-in. */
- protected static final String ASSIGN_MULTIPLE = CONF_PREFIX + "assignmultiple";
+ public static final String ASSIGN_MULTIPLE = CONF_PREFIX + "assignmultiple";
protected static final boolean DEFAULT_ASSIGN_MULTIPLE = false;
/** Whether to give more weight to apps requiring many resources. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c50fc925/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
index 1c90ad2..78077d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
@@ -60,6 +60,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -369,10 +372,29 @@ public class TestRMApplicationHistoryWriter {
}
@Test
- public void testRMWritingMassiveHistory() throws Exception {
+ public void testRMWritingMassiveHistoryForFairSche() throws Exception {
+ //test WritingMassiveHistory for Fair Scheduler.
+ testRMWritingMassiveHistory(true);
+ }
+
+ @Test
+ public void testRMWritingMassiveHistoryForCapacitySche() throws Exception {
+ //test WritingMassiveHistory for Capacity Scheduler.
+ testRMWritingMassiveHistory(false);
+ }
+
+ private void testRMWritingMassiveHistory(boolean isFS) throws Exception {
// 1. Show RM can run with writing history data
// 2. Test additional workload of processing history events
YarnConfiguration conf = new YarnConfiguration();
+ if (isFS) {
+ conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
+ conf.set("yarn.resourcemanager.scheduler.class",
+ FairScheduler.class.getName());
+ } else {
+ conf.set("yarn.resourcemanager.scheduler.class",
+ CapacityScheduler.class.getName());
+ }
// don't process history events
MockRM rm = new MockRM(conf) {
@Override
[40/50] [abbrv] git commit: YARN-2475. Logic for responding to
capacity drops for the ReservationSystem. Contributed by Carlo Curino and
Subru Krishnan.
Posted by cu...@apache.org.
YARN-2475. Logic for responding to capacity drops for the ReservationSystem. Contributed by Carlo Curino and Subru Krishnan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15478ca5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15478ca5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15478ca5
Branch: refs/heads/YARN-1051
Commit: 15478ca516fa7571cc5444ef00a595c5cf5ba3a6
Parents: 60d4f29
Author: carlo curino <Carlo Curino>
Authored: Fri Sep 12 16:52:54 2014 -0700
Committer: carlo curino <Carlo Curino>
Committed: Mon Sep 22 16:28:48 2014 -0700
----------------------------------------------------------------------
YARN-1051-CHANGES.txt | 5 +
.../reservation/SimpleCapacityReplanner.java | 95 ++++++++++++
.../TestSimpleCapacityReplanner.java | 149 +++++++++++++++++++
3 files changed, 249 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15478ca5/YARN-1051-CHANGES.txt
----------------------------------------------------------------------
diff --git a/YARN-1051-CHANGES.txt b/YARN-1051-CHANGES.txt
new file mode 100644
index 0000000..9fd4b3b
--- /dev/null
+++ b/YARN-1051-CHANGES.txt
@@ -0,0 +1,5 @@
+YARN-1707. Introduce APIs to add/remove/resize queues in the
+CapacityScheduler. (Carlo Curino and Subru Krishnan via curino)
+
+YARN-2475. Logic for responding to capacity drops for the
+ReservationSystem. (Carlo Curino and Subru Krishnan via curino)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15478ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SimpleCapacityReplanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SimpleCapacityReplanner.java
new file mode 100644
index 0000000..8384538
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SimpleCapacityReplanner.java
@@ -0,0 +1,95 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.UTCClock;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * This (re)planner scan a period of time from now to a maximum time window (or
+ * the end of the last session, whichever comes first) checking the overall
+ * capacity is not violated.
+ *
+ * It greedily removes sessions in reversed order of acceptance (latest accepted
+ * is the first removed).
+ */
+public class SimpleCapacityReplanner implements Planner {
+
+ private static final Log LOG = LogFactory
+ .getLog(SimpleCapacityReplanner.class);
+
+ private static final Resource ZERO_RESOURCE = Resource.newInstance(0, 0);
+
+ private final Clock clock;
+
+ // this allows to control to time-span of this replanning
+ // far into the future time instants might be worth replanning for
+ // later on
+ private long lengthOfCheckZone;
+
+ public SimpleCapacityReplanner() {
+ this(new UTCClock());
+ }
+
+ @VisibleForTesting
+ SimpleCapacityReplanner(Clock clock) {
+ this.clock = clock;
+ }
+
+ @Override
+ public void init(String planQueueName, CapacitySchedulerConfiguration conf) {
+ this.lengthOfCheckZone = conf.getEnforcementWindow(planQueueName);
+ }
+
+ @Override
+ public void plan(Plan plan, List<ReservationDefinition> contracts)
+ throws PlanningException {
+
+ if (contracts != null) {
+ throw new RuntimeException(
+ "SimpleCapacityReplanner cannot handle new reservation contracts");
+ }
+
+ ResourceCalculator resCalc = plan.getResourceCalculator();
+ Resource totCap = plan.getTotalCapacity();
+ long now = clock.getTime();
+
+ // loop on all moment in time from now to the end of the check Zone
+ // or the end of the planned sessions whichever comes first
+ for (long t = now; (t < plan.getLastEndTime() && t < (now + lengthOfCheckZone)); t +=
+ plan.getStep()) {
+ Resource excessCap =
+ Resources.subtract(plan.getTotalCommittedResources(t), totCap);
+ // if we are violating
+ if (Resources.greaterThan(resCalc, totCap, excessCap, ZERO_RESOURCE)) {
+ // sorted on reverse order of acceptance, so newest reservations first
+ Set<ReservationAllocation> curReservations =
+ new TreeSet<ReservationAllocation>(plan.getReservationsAtTime(t));
+ for (Iterator<ReservationAllocation> resIter =
+ curReservations.iterator(); resIter.hasNext()
+ && Resources.greaterThan(resCalc, totCap, excessCap, ZERO_RESOURCE);) {
+ ReservationAllocation reservation = resIter.next();
+ plan.deleteReservation(reservation.getReservationId());
+ excessCap =
+ Resources.subtract(excessCap, reservation.getResourcesAtTime(t));
+ LOG.info("Removing reservation " + reservation.getReservationId()
+ + " to repair physical-resource constraints in the plan: "
+ + plan.getQueueName());
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15478ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java
new file mode 100644
index 0000000..f2313e6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java
@@ -0,0 +1,149 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.junit.Test;
+
+public class TestSimpleCapacityReplanner {
+
+ @Test
+ public void testReplanningPlanCapacityLoss() throws PlanningException {
+
+ Resource clusterCapacity = Resource.newInstance(100 * 1024, 10);
+ Resource minAlloc = Resource.newInstance(1024, 1);
+ Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
+
+ ResourceCalculator res = new DefaultResourceCalculator();
+ long step = 1L;
+ Clock clock = mock(Clock.class);
+ ReservationAgent agent = mock(ReservationAgent.class);
+
+ SharingPolicy policy = new NoOverCommitPolicy();
+ policy.init("root.dedicated", null, new HashSet<String>());
+
+ QueueMetrics queueMetrics = mock(QueueMetrics.class);
+
+ when(clock.getTime()).thenReturn(0L);
+ SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
+
+ CapacitySchedulerConfiguration conf =
+ mock(CapacitySchedulerConfiguration.class);
+ when(conf.getEnforcementWindow(any(String.class))).thenReturn(6L);
+
+ conf.setLong(CapacitySchedulerConfiguration.PREFIX + "blah"
+ + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.RESERVATION_ENFORCEMENT_WINDOW, 6);
+ enf.init("blah", conf);
+
+ // Initialize the plan with more resources
+ InMemoryPlan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step,
+ res, minAlloc, maxAlloc, "dedicated", enf, true, clock);
+
+ // add reservation filling the plan (separating them 1ms, so we are sure
+ // s2 follows s1 on acceptance
+ long ts = System.currentTimeMillis();
+ ReservationId r1 = ReservationId.newInstance(ts, 1);
+ int[] f5 = { 20, 20, 20, 20, 20 };
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(r1, null, "u3",
+ "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res,
+ minAlloc)));
+ when(clock.getTime()).thenReturn(1L);
+ ReservationId r2 = ReservationId.newInstance(ts, 2);
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(r2, null, "u4",
+ "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res,
+ minAlloc)));
+ when(clock.getTime()).thenReturn(2L);
+ ReservationId r3 = ReservationId.newInstance(ts, 3);
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(r3, null, "u5",
+ "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res,
+ minAlloc)));
+ when(clock.getTime()).thenReturn(3L);
+ ReservationId r4 = ReservationId.newInstance(ts, 4);
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(r4, null, "u6",
+ "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res,
+ minAlloc)));
+ when(clock.getTime()).thenReturn(4L);
+ ReservationId r5 = ReservationId.newInstance(ts, 5);
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(r5, null, "u7",
+ "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res,
+ minAlloc)));
+
+ int[] f6 = { 50, 50, 50, 50, 50 };
+ ReservationId r6 = ReservationId.newInstance(ts, 6);
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(r6, null, "u3",
+ "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res,
+ minAlloc)));
+ when(clock.getTime()).thenReturn(6L);
+ ReservationId r7 = ReservationId.newInstance(ts, 7);
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(r7, null, "u4",
+ "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res,
+ minAlloc)));
+
+ // remove some of the resources (requires replanning)
+ plan.setTotalCapacity(Resource.newInstance(70 * 1024, 70));
+
+ when(clock.getTime()).thenReturn(0L);
+
+ // run the replanner
+ enf.plan(plan, null);
+
+ // check which reservation are still present
+ assertNotNull(plan.getReservationById(r1));
+ assertNotNull(plan.getReservationById(r2));
+ assertNotNull(plan.getReservationById(r3));
+ assertNotNull(plan.getReservationById(r6));
+ assertNotNull(plan.getReservationById(r7));
+
+ // and which ones are removed
+ assertNull(plan.getReservationById(r4));
+ assertNull(plan.getReservationById(r5));
+
+ // check resources at each moment in time no more exceed capacity
+ for (int i = 0; i < 20; i++) {
+ int tot = 0;
+ for (ReservationAllocation r : plan.getReservationsAtTime(i)) {
+ tot = r.getResourcesAtTime(i).getMemory();
+ }
+ assertTrue(tot <= 70 * 1024);
+ }
+ }
+
+ private Map<ReservationInterval, ReservationRequest> generateAllocation(
+ int startTime, int[] alloc) {
+ Map<ReservationInterval, ReservationRequest> req =
+ new TreeMap<ReservationInterval, ReservationRequest>();
+ for (int i = 0; i < alloc.length; i++) {
+ req.put(new ReservationInterval(startTime + i, startTime + i + 1),
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ alloc[i]));
+ }
+ return req;
+ }
+
+}
[18/50] [abbrv] git commit: HDFS-7046. HA NN can NPE upon transition
to active. Contributed by Kihwal Lee.
Posted by cu...@apache.org.
HDFS-7046. HA NN can NPE upon transition to active. Contributed by
Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e355719
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e355719
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e355719
Branch: refs/heads/YARN-1051
Commit: 9e355719653c5e7b48b601090634882e4f29a743
Parents: adf0b67
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Sep 19 17:07:06 2014 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Sep 19 17:07:06 2014 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 6 +++++-
2 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e355719/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9d4d8ed..8c45d69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -782,6 +782,8 @@ Release 2.6.0 - UNRELEASED
HDFS-7096. Fix TestRpcProgramNfs3 to use DFS_ENCRYPTION_KEY_PROVIDER_URI
(clamb via cmccabe)
+ HDFS-7046. HA NN can NPE upon transition to active. (kihwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e355719/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 2b03362..4dc2786 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1156,8 +1156,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
cacheManager.startMonitorThread();
blockManager.getDatanodeManager().setShouldSendCachingCommands(true);
} finally {
- writeUnlock();
startingActiveService = false;
+ checkSafeMode();
+ writeUnlock();
}
}
@@ -5570,6 +5571,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
// Have to have write-lock since leaving safemode initializes
// repl queues, which requires write lock
assert hasWriteLock();
+ if (inTransitionToActive()) {
+ return;
+ }
// if smmthread is already running, the block threshold must have been
// reached before, there is no need to enter the safe mode again
if (smmthread == null && needEnter()) {
[42/50] [abbrv] git commit: YARN-1708. Public YARN APIs for
creating/updating/deleting reservations.
Posted by cu...@apache.org.
YARN-1708. Public YARN APIs for creating/updating/deleting reservations.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f1d205a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f1d205a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f1d205a
Branch: refs/heads/YARN-1051
Commit: 3f1d205a73bffac56e295cad989a38aa260af80b
Parents: 15478ca
Author: subru <su...@outlook.com>
Authored: Fri Sep 12 17:16:07 2014 -0700
Committer: carlo curino <Carlo Curino>
Committed: Mon Sep 22 16:32:11 2014 -0700
----------------------------------------------------------------------
YARN-1051-CHANGES.txt | 2 +
.../yarn/api/ApplicationClientProtocol.java | 113 ++++++++++
.../ReservationDeleteRequest.java | 72 ++++++
.../ReservationDeleteResponse.java | 48 ++++
.../ReservationSubmissionRequest.java | 97 ++++++++
.../ReservationSubmissionResponse.java | 78 +++++++
.../ReservationUpdateRequest.java | 96 ++++++++
.../ReservationUpdateResponse.java | 48 ++++
.../records/ApplicationSubmissionContext.java | 25 ++-
.../yarn/api/records/ReservationDefinition.java | 137 ++++++++++++
.../hadoop/yarn/api/records/ReservationId.java | 147 ++++++++++++
.../yarn/api/records/ReservationRequest.java | 222 +++++++++++++++++++
.../records/ReservationRequestInterpreter.java | 101 +++++++++
.../yarn/api/records/ReservationRequests.java | 94 ++++++++
.../main/proto/applicationclient_protocol.proto | 3 +
.../src/main/proto/yarn_protos.proto | 36 +++
.../src/main/proto/yarn_service_protos.proto | 28 +++
.../ApplicationClientProtocolPBClientImpl.java | 58 ++++-
.../ApplicationClientProtocolPBServiceImpl.java | 61 +++++
.../impl/pb/ReservationDeleteRequestPBImpl.java | 127 +++++++++++
.../pb/ReservationDeleteResponsePBImpl.java | 68 ++++++
.../pb/ReservationSubmissionRequestPBImpl.java | 152 +++++++++++++
.../pb/ReservationSubmissionResponsePBImpl.java | 129 +++++++++++
.../impl/pb/ReservationUpdateRequestPBImpl.java | 169 ++++++++++++++
.../pb/ReservationUpdateResponsePBImpl.java | 68 ++++++
.../pb/ApplicationSubmissionContextPBImpl.java | 38 ++++
.../yarn/api/records/impl/pb/ProtoUtils.java | 17 ++
.../impl/pb/ReservationDefinitionPBImpl.java | 169 ++++++++++++++
.../records/impl/pb/ReservationIdPBImpl.java | 75 +++++++
.../impl/pb/ReservationRequestPBImpl.java | 152 +++++++++++++
.../impl/pb/ReservationRequestsPBImpl.java | 189 ++++++++++++++++
.../org/apache/hadoop/yarn/util/UTCClock.java | 39 ++++
32 files changed, 2856 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/YARN-1051-CHANGES.txt
----------------------------------------------------------------------
diff --git a/YARN-1051-CHANGES.txt b/YARN-1051-CHANGES.txt
index 9fd4b3b..a7c08a0 100644
--- a/YARN-1051-CHANGES.txt
+++ b/YARN-1051-CHANGES.txt
@@ -3,3 +3,5 @@ CapacityScheduler. (Carlo Curino and Subru Krishnan via curino)
YARN-2475. Logic for responding to capacity drops for the
ReservationSystem. (Carlo Curino and Subru Krishnan via curino)
+
+YARN-1708. Public YARN APIs for creating/updating/deleting reservations. (subru)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
index e449c1e..863a068 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
@@ -57,6 +57,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesReq
import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -67,6 +73,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.Token;
@@ -543,4 +550,110 @@ public interface ApplicationClientProtocol {
public GetContainersResponse getContainers(GetContainersRequest request)
throws YarnException, IOException;
+ /**
+ * <p>
+ * The interface used by clients to submit a new reservation to the
+ * {@link ResourceManager}.
+ * </p>
+ *
+ * <p>
+ * The client packages all details of its request in a
+ * {@link ReservationSubmissionRequest} object. This contains information
+ * about the amount of capacity, temporal constraints, and concurrency needs.
+ * Furthermore, the reservation might be composed of multiple stages, with
+ * ordering dependencies among them.
+ * </p>
+ *
+ * <p>
+ * In order to respond, a new admission control component in the
+ * {@link ResourceManager} performs an analysis of the resources that have
+ * been committed over the period of time the user is requesting, verify that
+ * the user requests can be fulfilled, and that it respect a sharing policy
+ * (e.g., {@link CapacityOverTimePolicy}). Once it has positively determined
+ * that the ReservationSubmissionRequest is satisfiable the
+ * {@link ResourceManager} answers with a
+ * {@link ReservationSubmissionResponse} that include a non-null
+ * {@link ReservationId}. Upon failure to find a valid allocation the response
+ * is an exception with the reason.
+ *
+ * On application submission the client can use this {@link ReservationId} to
+ * obtain access to the reserved resources.
+ * </p>
+ *
+ * <p>
+ * The system guarantees that during the time-range specified by the user, the
+ * reservationID will be corresponding to a valid reservation. The amount of
+ * capacity dedicated to such queue can vary overtime, depending of the
+ * allocation that has been determined. But it is guaranteed to satisfy all
+ * the constraint expressed by the user in the
+ * {@link ReservationSubmissionRequest}.
+ * </p>
+ *
+ * @param request the request to submit a new Reservation
+ * @return response the {@link ReservationId} on accepting the submission
+ * @throws YarnException if the request is invalid or reservation cannot be
+ * created successfully
+ * @throws IOException
+ *
+ */
+ @Public
+ @Unstable
+ public ReservationSubmissionResponse submitReservation(
+ ReservationSubmissionRequest request) throws YarnException, IOException;
+
+ /**
+ * <p>
+ * The interface used by clients to update an existing Reservation. This is
+ * referred to as a re-negotiation process, in which a user that has
+ * previously submitted a Reservation.
+ * </p>
+ *
+ * <p>
+ * The allocation is attempted by virtually substituting all previous
+ * allocations related to this Reservation with new ones, that satisfy the new
+ * {@link ReservationUpdateRequest}. Upon success the previous allocation is
+ * substituted by the new one, and on failure (i.e., if the system cannot find
+ * a valid allocation for the updated request), the previous allocation
+ * remains valid.
+ *
+ * The {@link ReservationId} is not changed, and applications currently
+ * running within this reservation will automatically receive the resources
+ * based on the new allocation.
+ * </p>
+ *
+ * @param request to update an existing Reservation (the ReservationRequest
+ * should refer to an existing valid {@link ReservationId})
+ * @return response empty on successfully updating the existing reservation
+ * @throws YarnException if the request is invalid or reservation cannot be
+ * updated successfully
+ * @throws IOException
+ *
+ */
+ @Public
+ @Unstable
+ public ReservationUpdateResponse updateReservation(
+ ReservationUpdateRequest request) throws YarnException, IOException;
+
+ /**
+ * <p>
+ * The interface used by clients to remove an existing Reservation.
+ *
+ * Upon deletion of a reservation applications running with this reservation,
+ * are automatically downgraded to normal jobs running without any dedicated
+ * reservation.
+ * </p>
+ *
+ * @param request to remove an existing Reservation (the ReservationRequest
+ * should refer to an existing valid {@link ReservationId})
+ * @return response empty on successfully deleting the existing reservation
+ * @throws YarnException if the request is invalid or reservation cannot be
+ * deleted successfully
+ * @throws IOException
+ *
+ */
+ @Public
+ @Unstable
+ public ReservationDeleteResponse deleteReservation(
+ ReservationDeleteRequest request) throws YarnException, IOException;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationDeleteRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationDeleteRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationDeleteRequest.java
new file mode 100644
index 0000000..1dd876b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationDeleteRequest.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@link ReservationDeleteRequest} captures the set of requirements the user
+ * has to delete an existing reservation.
+ *
+ * @see ReservationDefinition
+ *
+ */
+@Public
+@Unstable
+public abstract class ReservationDeleteRequest {
+
+ @Public
+ @Unstable
+ public static ReservationDeleteRequest newInstance(ReservationId reservationId) {
+ ReservationDeleteRequest request =
+ Records.newRecord(ReservationDeleteRequest.class);
+ request.setReservationId(reservationId);
+ return request;
+ }
+
+ /**
+ * Get the {@link ReservationId}, that corresponds to a valid resource
+ * allocation in the scheduler (between start and end time of this
+ * reservation)
+ *
+ * @return the {@link ReservationId} representing the unique id of the
+ * corresponding reserved resource allocation in the scheduler
+ */
+ @Public
+ @Unstable
+ public abstract ReservationId getReservationId();
+
+ /**
+ * Set the {@link ReservationId}, that correspond to a valid resource
+ * allocation in the scheduler (between start and end time of this
+ * reservation)
+ *
+ * @param reservationId the {@link ReservationId} representing the the unique
+ * id of the corresponding reserved resource allocation in the
+ * scheduler
+ */
+ @Public
+ @Unstable
+ public abstract void setReservationId(ReservationId reservationId);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationDeleteResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationDeleteResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationDeleteResponse.java
new file mode 100644
index 0000000..eedd3a4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationDeleteResponse.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@link ReservationDeleteResponse} contains the answer of the admission
+ * control system in the {@link ResourceManager} to a reservation delete
+ * operation. Currently response is empty if the operation was successful, if
+ * not an exception reporting reason for a failure.
+ *
+ * @see ReservationDefinition
+ *
+ */
+@Public
+@Unstable
+public abstract class ReservationDeleteResponse {
+
+ @Private
+ @Unstable
+ public static ReservationDeleteResponse newInstance() {
+ ReservationDeleteResponse response =
+ Records.newRecord(ReservationDeleteResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionRequest.java
new file mode 100644
index 0000000..e550fe2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionRequest.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@link ReservationSubmissionRequest} captures the set of requirements the
+ * user has to create a reservation.
+ *
+ * @see ReservationDefinition
+ *
+ */
+@Public
+@Unstable
+public abstract class ReservationSubmissionRequest {
+
+ @Public
+ @Unstable
+ public static ReservationSubmissionRequest newInstance(
+ ReservationDefinition reservationDefinition, String queueName) {
+ ReservationSubmissionRequest request =
+ Records.newRecord(ReservationSubmissionRequest.class);
+ request.setReservationDefinition(reservationDefinition);
+ request.setQueue(queueName);
+ return request;
+ }
+
+ /**
+ * Get the {@link ReservationDefinition} representing the user constraints for
+ * this reservation
+ *
+ * @return the reservation definition representing user constraints
+ */
+ @Public
+ @Unstable
+ public abstract ReservationDefinition getReservationDefinition();
+
+ /**
+ * Set the {@link ReservationDefinition} representing the user constraints for
+ * this reservation
+ *
+ * @param reservationDefinition the reservation request representing the
+ * reservation
+ */
+ @Public
+ @Unstable
+ public abstract void setReservationDefinition(
+ ReservationDefinition reservationDefinition);
+
+ /**
+ * Get the name of the {@link Plan} that corresponds to the name of the
+ * {@link QueueInfo} in the scheduler to which the reservation will be
+ * submitted to.
+ *
+ * @return the name of the {@link Plan} that corresponds to the name of the
+ * {@link QueueInfo} in the scheduler to which the reservation will be
+ * submitted to
+ */
+ @Public
+ @Unstable
+ public abstract String getQueue();
+
+ /**
+ * Set the name of the {@link Plan} that corresponds to the name of the
+ * {@link QueueInfo} in the scheduler to which the reservation will be
+ * submitted to
+ *
+ * @param the name of the parent {@link Plan} that corresponds to the name of
+ * the {@link QueueInfo} in the scheduler to which the reservation
+ * will be submitted to
+ */
+ @Public
+ @Unstable
+ public abstract void setQueue(String queueName);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionResponse.java
new file mode 100644
index 0000000..b57ef52
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionResponse.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.util.Records;
+
+import com.sun.naming.internal.ResourceManager;
+
+/**
+ * {@link ReservationSubmissionResponse} contains the answer of the admission
+ * control system in the {@link ResourceManager} to a reservation create
+ * operation. Response contains a {@link ReservationId} if the operation was
+ * successful, if not an exception reporting reason for a failure.
+ *
+ * @see ReservationDefinition
+ *
+ */
+@Public
+@Unstable
+public abstract class ReservationSubmissionResponse {
+
+ @Private
+ @Unstable
+ public static ReservationSubmissionResponse newInstance(
+ ReservationId reservationId) {
+ ReservationSubmissionResponse response =
+ Records.newRecord(ReservationSubmissionResponse.class);
+ response.setReservationId(reservationId);
+ return response;
+ }
+
+ /**
+ * Get the {@link ReservationId}, that corresponds to a valid resource
+ * allocation in the scheduler (between start and end time of this
+ * reservation)
+ *
+ * @return the {@link ReservationId} representing the unique id of the
+ * corresponding reserved resource allocation in the scheduler
+ */
+ @Public
+ @Unstable
+ public abstract ReservationId getReservationId();
+
+ /**
+ * Set the {@link ReservationId}, that correspond to a valid resource
+ * allocation in the scheduler (between start and end time of this
+ * reservation)
+ *
+ * @param reservationId the {@link ReservationId} representing the the unique
+ * id of the corresponding reserved resource allocation in the
+ * scheduler
+ */
+ @Private
+ @Unstable
+ public abstract void setReservationId(ReservationId reservationId);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationUpdateRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationUpdateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationUpdateRequest.java
new file mode 100644
index 0000000..b5a4b7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationUpdateRequest.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@link ReservationUpdateRequest} captures the set of requirements the user
+ * has to update an existing reservation.
+ *
+ * @see ReservationDefinition
+ *
+ */
+@Public
+@Unstable
+public abstract class ReservationUpdateRequest {
+
+ @Public
+ @Unstable
+ public static ReservationUpdateRequest newInstance(
+ ReservationDefinition reservationDefinition, ReservationId reservationId) {
+ ReservationUpdateRequest request =
+ Records.newRecord(ReservationUpdateRequest.class);
+ request.setReservationDefinition(reservationDefinition);
+ request.setReservationId(reservationId);
+ return request;
+ }
+
+ /**
+ * Get the {@link ReservationDefinition} representing the updated user
+ * constraints for this reservation
+ *
+ * @return the reservation definition representing user constraints
+ */
+ @Public
+ @Unstable
+ public abstract ReservationDefinition getReservationDefinition();
+
+ /**
+ * Set the {@link ReservationDefinition} representing the updated user
+ * constraints for this reservation
+ *
+ * @param reservationDefinition the reservation request representing the
+ * reservation
+ */
+ @Public
+ @Unstable
+ public abstract void setReservationDefinition(
+ ReservationDefinition reservationDefinition);
+
+ /**
+ * Get the {@link ReservationId}, that corresponds to a valid resource
+ * allocation in the scheduler (between start and end time of this
+ * reservation)
+ *
+ * @return the {@link ReservationId} representing the unique id of the
+ * corresponding reserved resource allocation in the scheduler
+ */
+ @Public
+ @Unstable
+ public abstract ReservationId getReservationId();
+
+ /**
+ * Set the {@link ReservationId}, that correspond to a valid resource
+ * allocation in the scheduler (between start and end time of this
+ * reservation)
+ *
+ * @param reservationId the {@link ReservationId} representing the the unique
+ * id of the corresponding reserved resource allocation in the
+ * scheduler
+ */
+ @Public
+ @Unstable
+ public abstract void setReservationId(ReservationId reservationId);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationUpdateResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationUpdateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationUpdateResponse.java
new file mode 100644
index 0000000..bfd2d51
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationUpdateResponse.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@link ReservationUpdateResponse} contains the answer of the admission
+ * control system in the {@link ResourceManager} to a reservation update
+ * operation. Currently response is empty if the operation was successful, if
+ * not an exception reporting reason for a failure.
+ *
+ * @see ReservationDefinition
+ *
+ */
+@Public
+@Unstable
+public abstract class ReservationUpdateResponse {
+
+ @Private
+ @Unstable
+ public static ReservationUpdateResponse newInstance() {
+ ReservationUpdateResponse response =
+ Records.newRecord(ReservationUpdateResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index 723a2e0..bb2fbfc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -381,4 +381,27 @@ public abstract class ApplicationSubmissionContext {
@Stable
public abstract void setAttemptFailuresValidityInterval(
long attemptFailuresValidityInterval);
-}
\ No newline at end of file
+
+ /**
+ * Get the reservation id, that corresponds to a valid resource allocation in
+ * the scheduler (between start and end time of the corresponding reservation)
+ *
+ * @return the reservation id representing the unique id of the corresponding
+ * reserved resource allocation in the scheduler
+ */
+ @Public
+ @Unstable
+ public abstract ReservationId getReservationID();
+
+ /**
+ * Set the reservation id, that correspond to a valid resource allocation in
+ * the scheduler (between start and end time of the corresponding reservation)
+ *
+ * @param reservationId representing the unique id of the
+ * corresponding reserved resource allocation in the scheduler
+ */
+ @Public
+ @Unstable
+ public abstract void setReservationID(ReservationId reservationID);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
new file mode 100644
index 0000000..c94463d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@link ReservationDefinition} captures the set of resource and time
+ * constraints the user cares about regarding a reservation.
+ *
+ * @see ResourceRequest
+ *
+ */
+@Public
+@Unstable
+public abstract class ReservationDefinition {
+
+ @Public
+ @Unstable
+ public static ReservationDefinition newInstance(long arrival, long deadline,
+ ReservationRequests reservationRequests, String name) {
+ ReservationDefinition rDefinition =
+ Records.newRecord(ReservationDefinition.class);
+ rDefinition.setArrival(arrival);
+ rDefinition.setDeadline(deadline);
+ rDefinition.setReservationRequests(reservationRequests);
+ rDefinition.setReservationName(name);
+ return rDefinition;
+ }
+
+ /**
+ * Get the arrival time or the earliest time from which the resource(s) can be
+ * allocated. Time expressed as UTC.
+ *
+ * @return the earliest valid time for this reservation
+ */
+ @Public
+ @Unstable
+ public abstract long getArrival();
+
+ /**
+ * Set the arrival time or the earliest time from which the resource(s) can be
+ * allocated. Time expressed as UTC.
+ *
+ * @param earliestStartTime the earliest valid time for this reservation
+ */
+ @Public
+ @Unstable
+ public abstract void setArrival(long earliestStartTime);
+
+ /**
+ * Get the deadline or the latest time by when the resource(s) must be
+ * allocated. Time expressed as UTC.
+ *
+ * @return the deadline or the latest time by when the resource(s) must be
+ * allocated
+ */
+ @Public
+ @Unstable
+ public abstract long getDeadline();
+
+ /**
+ * Set the deadline or the latest time by when the resource(s) must be
+ * allocated. Time expressed as UTC.
+ *
+ * @param latestEndTime the deadline or the latest time by when the
+ * resource(s) should be allocated
+ */
+ @Public
+ @Unstable
+ public abstract void setDeadline(long latestEndTime);
+
+ /**
+ * Get the list of {@link ReservationRequests} representing the resources
+ * required by the application
+ *
+ * @return the list of {@link ReservationRequests}
+ */
+ @Public
+ @Unstable
+ public abstract ReservationRequests getReservationRequests();
+
+ /**
+ * Set the list of {@link ReservationRequests} representing the resources
+ * required by the application
+ *
+ * @param resources the list of {@link ReservationRequests}
+ */
+ @Public
+ @Unstable
+ public abstract void setReservationRequests(
+ ReservationRequests reservationRequests);
+
+ /**
+ * Get the name for this reservation. The name need-not be unique, and it is
+ * just a mnemonic for the user (akin to job names). Accepted reservations are
+ * uniquely identified by a system-generated ReservationId.
+ *
+ * @return string representing the name of the corresponding reserved resource
+ * allocation in the scheduler
+ */
+ @Public
+ @Evolving
+ public abstract String getReservationName();
+
+ /**
+ * Set the name for this reservation. The name need-not be unique, and it is
+ * just a mnemonic for the user (akin to job names). Accepted reservations are
+ * uniquely identified by a system-generated ReservationId.
+ *
+ * @param name representing the name of the corresponding reserved resource
+ * allocation in the scheduler
+ */
+ @Public
+ @Evolving
+ public abstract void setReservationName(String name);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java
new file mode 100644
index 0000000..47a8c40
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * {@link ReservationId} represents the <em>globally unique</em> identifier for
+ * a reservation.
+ * </p>
+ *
+ * <p>
+ * The globally unique nature of the identifier is achieved by using the
+ * <em>cluster timestamp</em> i.e. start-time of the {@link ResourceManager}
+ * along with a monotonically increasing counter for the reservation.
+ * </p>
+ */
+@Public
+@Unstable
+public abstract class ReservationId implements Comparable<ReservationId> {
+
+ @Private
+ @Unstable
+ public static final String reserveIdStrPrefix = "reservation_";
+ protected long clusterTimestamp;
+ protected long id;
+
+ @Private
+ @Unstable
+ public static ReservationId newInstance(long clusterTimestamp, long id) {
+ ReservationId reservationId = Records.newRecord(ReservationId.class);
+ reservationId.setClusterTimestamp(clusterTimestamp);
+ reservationId.setId(id);
+ reservationId.build();
+ return reservationId;
+ }
+
+ /**
+ * Get the long identifier of the {@link ReservationId} which is unique for
+ * all Reservations started by a particular instance of the
+ * {@link ResourceManager}.
+ *
+ * @return long identifier of the {@link ReservationId}
+ */
+ @Public
+ @Unstable
+ public abstract long getId();
+
+ @Private
+ @Unstable
+ protected abstract void setId(long id);
+
+ /**
+ * Get the <em>start time</em> of the {@link ResourceManager} which is used to
+ * generate globally unique {@link ReservationId}.
+ *
+ * @return <em>start time</em> of the {@link ResourceManager}
+ */
+ @Public
+ @Unstable
+ public abstract long getClusterTimestamp();
+
+ @Private
+ @Unstable
+ protected abstract void setClusterTimestamp(long clusterTimestamp);
+
+ protected abstract void build();
+
+ static final ThreadLocal<NumberFormat> reservIdFormat =
+ new ThreadLocal<NumberFormat>() {
+ @Override
+ public NumberFormat initialValue() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(4);
+ return fmt;
+ }
+ };
+
+ @Override
+ public int compareTo(ReservationId other) {
+ if (this.getClusterTimestamp() - other.getClusterTimestamp() == 0) {
+ return getId() > getId() ? 1 : getId() < getId() ? -1 : 0;
+ } else {
+ return this.getClusterTimestamp() > other.getClusterTimestamp() ? 1
+ : this.getClusterTimestamp() < other.getClusterTimestamp() ? -1 : 0;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return reserveIdStrPrefix + this.getClusterTimestamp() + "_"
+ + reservIdFormat.get().format(getId());
+ }
+
+ @Override
+ public int hashCode() {
+ // generated by eclipse
+ final int prime = 31;
+ int result = 1;
+ result =
+ prime * result
+ + (int) (getClusterTimestamp() ^ (getClusterTimestamp() >>> 32));
+ result = prime * result + (int) (getId() ^ (getId() >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // generated by eclipse
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ ReservationId other = (ReservationId) obj;
+ if (getClusterTimestamp() != other.getClusterTimestamp())
+ return false;
+ if (getId() != other.getId())
+ return false;
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequest.java
new file mode 100644
index 0000000..ded6d06
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequest.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import java.io.Serializable;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * {@link ReservationRequest} represents the request made by an application to
+ * the {@link ResourceManager} to reserve {@link Resource}s.
+ * </p>
+ *
+ * <p>
+ * It includes:
+ * <ul>
+ * <li>{@link Resource} required for each request.</li>
+ * <li>
+ * Number of containers, of above specifications, which are required by the
+ * application.</li>
+ * <li>
+ * Concurrency that indicates the gang size of the request.</li>
+ * </ul>
+ * </p>
+ *
+ */
+@Public
+@Unstable
+public abstract class ReservationRequest implements
+ Comparable<ReservationRequest> {
+
+ @Public
+ @Unstable
+ public static ReservationRequest newInstance(Resource capability,
+ int numContainers) {
+ return newInstance(capability, numContainers, 1, -1);
+ }
+
+ @Public
+ @Unstable
+ public static ReservationRequest newInstance(Resource capability,
+ int numContainers, int concurrency, long duration) {
+ ReservationRequest request = Records.newRecord(ReservationRequest.class);
+ request.setCapability(capability);
+ request.setNumContainers(numContainers);
+ request.setConcurrency(concurrency);
+ request.setDuration(duration);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public static class ReservationRequestComparator implements
+ java.util.Comparator<ReservationRequest>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public int compare(ReservationRequest r1, ReservationRequest r2) {
+ // Compare numContainers, concurrency and capability
+ int ret = r1.getNumContainers() - r2.getNumContainers();
+ if (ret == 0) {
+ ret = r1.getConcurrency() - r2.getConcurrency();
+ }
+ if (ret == 0) {
+ ret = r1.getCapability().compareTo(r2.getCapability());
+ }
+ return ret;
+ }
+ }
+
+ /**
+ * Get the {@link Resource} capability of the request.
+ *
+ * @return {@link Resource} capability of the request
+ */
+ @Public
+ @Unstable
+ public abstract Resource getCapability();
+
+ /**
+ * Set the {@link Resource} capability of the request
+ *
+ * @param capability {@link Resource} capability of the request
+ */
+ @Public
+ @Unstable
+ public abstract void setCapability(Resource capability);
+
+ /**
+ * Get the number of containers required with the given specifications.
+ *
+ * @return number of containers required with the given specifications
+ */
+ @Public
+ @Unstable
+ public abstract int getNumContainers();
+
+ /**
+ * Set the number of containers required with the given specifications
+ *
+ * @param numContainers number of containers required with the given
+ * specifications
+ */
+ @Public
+ @Unstable
+ public abstract void setNumContainers(int numContainers);
+
+ /**
+ * Get the number of containers that need to be scheduled concurrently. The
+ * default value of 1 would fall back to the current non concurrency
+ * constraints on the scheduling behavior.
+ *
+ * @return the number of containers to be concurrently scheduled
+ */
+ @Public
+ @Unstable
+ public abstract int getConcurrency();
+
+ /**
+ * Set the number of containers that need to be scheduled concurrently. The
+ * default value of 1 would fall back to the current non concurrency
+ * constraints on the scheduling behavior.
+ *
+ * @param numContainers the number of containers to be concurrently scheduled
+ */
+ @Public
+ @Unstable
+ public abstract void setConcurrency(int numContainers);
+
+ /**
+ * Get the duration in milliseconds for which the resource is required. A
+ * default value of -1, indicates an unspecified lease duration, and fallback
+ * to current behavior.
+ *
+ * @return the duration in milliseconds for which the resource is required
+ */
+ @Public
+ @Unstable
+ public abstract long getDuration();
+
+ /**
+ * Set the duration in milliseconds for which the resource is required.
+ *
+ * @param duration the duration in milliseconds for which the resource is
+ * required
+ */
+ @Public
+ @Unstable
+ public abstract void setDuration(long duration);
+
+ @Override
+ public int hashCode() {
+ final int prime = 2153;
+ int result = 2459;
+ Resource capability = getCapability();
+ result =
+ prime * result + ((capability == null) ? 0 : capability.hashCode());
+ result = prime * result + getNumContainers();
+ result = prime * result + getConcurrency();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ ReservationRequest other = (ReservationRequest) obj;
+ Resource capability = getCapability();
+ if (capability == null) {
+ if (other.getCapability() != null)
+ return false;
+ } else if (!capability.equals(other.getCapability()))
+ return false;
+ if (getNumContainers() != other.getNumContainers())
+ return false;
+ if (getConcurrency() != other.getConcurrency())
+ return false;
+ return true;
+ }
+
+ @Override
+ public int compareTo(ReservationRequest other) {
+ int numContainersComparison =
+ this.getNumContainers() - other.getNumContainers();
+ if (numContainersComparison == 0) {
+ int concurrencyComparison =
+ this.getConcurrency() - other.getConcurrency();
+ if (concurrencyComparison == 0) {
+ return this.getCapability().compareTo(other.getCapability());
+ } else {
+ return concurrencyComparison;
+ }
+ } else {
+ return numContainersComparison;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequestInterpreter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequestInterpreter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequestInterpreter.java
new file mode 100644
index 0000000..4621e0d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequestInterpreter.java
@@ -0,0 +1,101 @@
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+/**
+ * Enumeration of various types of dependencies among multiple
+ * {@link ReservationRequests} within one {@link ReservationDefinition} (from
+ * least constraining to most constraining).
+ */
+@Public
+@Evolving
+public enum ReservationRequestInterpreter {
+ /**
+ * Requires that exactly ONE among the {@link ReservationRequest} submitted as
+ * of a {@link ReservationDefinition} is satisfied to satisfy the overall
+ * {@link ReservationDefinition}.
+ *
+ * WHEN TO USE THIS: This is useful when the user have multiple equivalent
+ * ways to run an application, and wants to expose to the ReservationAgent
+ * such flexibility. For example an application could use one <32GB,16core>
+ * container for 10min, or 16 <2GB,1core> containers for 15min, the
+ * ReservationAgent will decide which one of the two it is best for the system
+ * to place.
+ *
+ */
+ R_ANY,
+
+ /**
+ * Requires that ALL of the {@link ReservationRequest} submitted as part of a
+ * {@link ReservationDefinition} are satisfied for the overall
+ * {@link ReservationDefinition} to be satisfied. No constraints are imposed
+ * on the temporal ordering of the allocation used to satisfy the
+ * ResourceRequeusts.
+ *
+ * WHEN TO USE THIS: This is useful to capture a scenario in which the user
+ * cares for multiple ReservationDefinition to be all accepted, or none. For
+ * example, a user might want a reservation R1: with 10 x <8GB,4core> for
+ * 10min, and a reservation R2: with 2 <1GB,1core> for 1h, and only if both
+ * are satisfied the workflow run in this reservation succeeds. The key
+ * differentiator from ALL and ORDER, ORDER_NO_GAP, is that ALL imposes no
+ * restrictions on the relative allocations used to place R1 and R2 above.
+ *
+ */
+ R_ALL,
+
+ /**
+ * Requires that ALL of the {@link ReservationRequest} submitted as part of a
+ * {@link ReservationDefinition} are satisfied for the overall
+ * {@link ReservationDefinition} to be satisfied. Moreover, it imposes a
+ * strict temporal ordering on the allocation used to satisfy the
+ * {@link ReservationRequest}s. The allocations satisfying the
+ * {@link ReservationRequest} in position k must strictly precede the
+ * allocations for the {@link ReservationRequest} at position k+1. No
+ * constraints are imposed on temporal gaps between subsequent allocations
+ * (the last instant of the previous allocation can be an arbitrary long
+ * period of time before the first instant of the subsequent allocation).
+ *
+ * WHEN TO USE THIS: Like ALL this requires all ReservationDefinitions to be
+ * placed, but it also imposes a time ordering on the allocations used. This
+ * is important if the ReservationDefinition(s) are used to describe a
+ * workflow with inherent inter-stage dependencies. For example, a first job
+ * runs in a ReservaitonDefinition R1 (10 x <1GB,1core> for 20min), and its
+ * output is consumed by a second job described by a ReservationDefinition R2
+ * (5 x <1GB,1core>) for 50min). R2 allocation cannot overlap R1, as R2 models
+ * a job depending on the output of the job modeled by R1.
+ */
+ R_ORDER,
+
+ /**
+ * Requires that ALL of the {@link ReservationRequest} submitted as part of a
+ * {@link ReservationDefinition} are satisfied for the overall
+ * {@link ReservationDefinition} to be satisfied. Moreover, it imposes a
+ * strict temporal ordering on the allocation used to satisfy the
+ * {@link ResourceRequeust}s. It imposes a strict temporal ordering on the
+ * allocation used to satisfy the {@link ReservationRequest}s. The allocations
+ * satisfying the {@link ReservationRequest} in position k must strictly
+ * precede the allocations for the {@link ReservationRequest} at position k+1.
+ * Moreover it imposes a "zero-size gap" between subsequent allocations, i.e.,
+ * the last instant in time of the allocations associated with the
+ * {@link ReservationRequest} at position k must be exactly preceding the
+ * first instant in time of the {@link ReservationRequest} at position k+1.
+ * Time ranges are interpreted as [a,b) inclusive left, exclusive right.
+ *
+ * WHEN TO USE THIS: This is a stricter version of R_ORDER, which allows no
+ * gaps between the allocations that satisfy R1 and R2. The use of this is
+ * twofold: 1) prevent long gaps between subsequent stages that produce very
+ * large intermediate output (e.g., the output of R1 is too large to be kept
+ * around for long before the job running in R2 consumes it, and disposes of
+ * it), 2) if the job being modeled has a time-varying resource need, one can
+ * combine multiple ResourceDefinition each approximating a portion of the job
+ * execution (think of using multiple rectangular bounding boxes to described
+ * an arbitrarily shaped area). By asking for no-gaps we guarantee
+ * "continuity" of resources given to this job. This still allow for some
+ * flexibility, as the entire "train" of allocations can be moved rigidly back
+ * or forth within the start-deadline time range (if there is slack).
+ *
+ */
+ R_ORDER_NO_GAP
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequests.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequests.java
new file mode 100644
index 0000000..8cc3fbc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationRequests.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@link ReservationRequests} captures the set of resource and constraints the
+ * user cares about regarding a reservation.
+ *
+ * @see ReservationRequest
+ *
+ */
+@Public
+@Unstable
+public abstract class ReservationRequests {
+
+ @Public
+ @Unstable
+ public static ReservationRequests newInstance(
+ List<ReservationRequest> reservationResources,
+ ReservationRequestInterpreter type) {
+ ReservationRequests reservationRequests =
+ Records.newRecord(ReservationRequests.class);
+ reservationRequests.setReservationResources(reservationResources);
+ reservationRequests.setInterpreter(type);
+ return reservationRequests;
+ }
+
+ /**
+ * Get the list of {@link ReservationRequest} representing the resources
+ * required by the application
+ *
+ * @return the list of {@link ReservationRequest}
+ */
+ @Public
+ @Unstable
+ public abstract List<ReservationRequest> getReservationResources();
+
+ /**
+ * Set the list of {@link ReservationRequest} representing the resources
+ * required by the application
+ *
+ * @param resources the list of {@link ReservationRequest}
+ */
+ @Public
+ @Unstable
+ public abstract void setReservationResources(
+ List<ReservationRequest> reservationResources);
+
+ /**
+ * Get the {@link ReservationRequestInterpreter}, representing how the list of
+ * resources should be allocated, this captures temporal ordering and other
+ * constraints.
+ *
+ * @return the list of {@link ReservationRequestInterpreter}
+ */
+ @Public
+ @Unstable
+ public abstract ReservationRequestInterpreter getInterpreter();
+
+ /**
+ * Set the {@link ReservationRequestInterpreter}, representing how the list of
+ * resources should be allocated, this captures temporal ordering and other
+ * constraints.
+ *
+ * @param interpreter the {@link ReservationRequestInterpreter} for this
+ * reservation
+ */
+ @Public
+ @Unstable
+ public abstract void setInterpreter(ReservationRequestInterpreter interpreter);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto
index 2d8cae6..3346aef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto
@@ -49,5 +49,8 @@ service ApplicationClientProtocolService {
rpc getApplicationAttempts (GetApplicationAttemptsRequestProto) returns (GetApplicationAttemptsResponseProto);
rpc getContainerReport (GetContainerReportRequestProto) returns (GetContainerReportResponseProto);
rpc getContainers (GetContainersRequestProto) returns (GetContainersResponseProto);
+ rpc submitReservation (ReservationSubmissionRequestProto) returns (ReservationSubmissionResponseProto);
+ rpc updateReservation (ReservationUpdateRequestProto) returns (ReservationUpdateResponseProto);
+ rpc deleteReservation (ReservationDeleteRequestProto) returns (ReservationDeleteResponseProto);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index d8c42cc..dc9c661 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -292,6 +292,7 @@ message ApplicationSubmissionContextProto {
optional bool keep_containers_across_application_attempts = 11 [default = false];
repeated string applicationTags = 12;
optional int64 attempt_failures_validity_interval = 13 [default = -1];
+ optional ReservationIdProto reservation_id = 14;
}
enum ApplicationAccessTypeProto {
@@ -334,6 +335,41 @@ message QueueUserACLInfoProto {
}
////////////////////////////////////////////////////////////////////////
+////// From reservation_protocol /////////////////////////////////////
+////////////////////////////////////////////////////////////////////////
+
+message ReservationIdProto {
+ optional int64 id = 1;
+ optional int64 cluster_timestamp = 2;
+}
+
+message ReservationRequestProto {
+ optional ResourceProto capability = 1;
+ optional int32 num_containers = 2 [default = 1];
+ optional int32 concurrency = 3 [default = 1];
+ optional int64 duration = 4 [default = -1];
+}
+
+message ReservationRequestsProto {
+ repeated ReservationRequestProto reservation_resources = 1;
+ optional ReservationRequestInterpreterProto interpreter = 2 [default = R_ALL];
+}
+
+message ReservationDefinitionProto {
+ optional ReservationRequestsProto reservation_requests = 1;
+ optional int64 arrival = 2;
+ optional int64 deadline = 3;
+ optional string reservation_name = 4;
+}
+
+enum ReservationRequestInterpreterProto {
+ R_ANY = 0;
+ R_ALL = 1;
+ R_ORDER = 2;
+ R_ORDER_NO_GAP = 3;
+ }
+
+////////////////////////////////////////////////////////////////////////
////// From container_manager //////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 4203744..2585f78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -292,3 +292,31 @@ message GetContainersRequestProto {
message GetContainersResponseProto {
repeated ContainerReportProto containers = 1;
}
+
+//////////////////////////////////////////////////////
+// reservation_protocol
+//////////////////////////////////////////////////////
+
+message ReservationSubmissionRequestProto {
+ optional string queue = 1;
+ optional ReservationDefinitionProto reservation_definition = 2;
+}
+
+message ReservationSubmissionResponseProto {
+ optional ReservationIdProto reservation_id = 1;
+}
+
+message ReservationUpdateRequestProto {
+ optional ReservationDefinitionProto reservation_definition = 1;
+ optional ReservationIdProto reservation_id = 2;
+}
+
+message ReservationUpdateResponseProto {
+}
+
+message ReservationDeleteRequestProto {
+ optional ReservationIdProto reservation_id = 1;
+}
+
+message ReservationDeleteResponseProto {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java
index 5794da0..b4b9f43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java
@@ -63,6 +63,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesReq
import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl;
@@ -97,6 +103,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.MoveApplicationAcrossQ
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.MoveApplicationAcrossQueuesResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RenewDelegationTokenResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationDeleteRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationDeleteResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationSubmissionRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationSubmissionResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationUpdateRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationUpdateResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -110,13 +122,15 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.MoveApplicationAcrossQueuesRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerReportRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainersRequestProto;
-
import com.google.protobuf.ServiceException;
@Private
@@ -391,4 +405,46 @@ public class ApplicationClientProtocolPBClientImpl implements ApplicationClientP
}
}
+ @Override
+ public ReservationSubmissionResponse submitReservation(ReservationSubmissionRequest request)
+ throws YarnException, IOException {
+ ReservationSubmissionRequestProto requestProto =
+ ((ReservationSubmissionRequestPBImpl) request).getProto();
+ try {
+ return new ReservationSubmissionResponsePBImpl(proxy.submitReservation(null,
+ requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
+
+ @Override
+ public ReservationUpdateResponse updateReservation(ReservationUpdateRequest request)
+ throws YarnException, IOException {
+ ReservationUpdateRequestProto requestProto =
+ ((ReservationUpdateRequestPBImpl) request).getProto();
+ try {
+ return new ReservationUpdateResponsePBImpl(proxy.updateReservation(null,
+ requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
+
+ @Override
+ public ReservationDeleteResponse deleteReservation(ReservationDeleteRequest request)
+ throws YarnException, IOException {
+ ReservationDeleteRequestProto requestProto =
+ ((ReservationDeleteRequestPBImpl) request).getProto();
+ try {
+ return new ReservationDeleteResponsePBImpl(proxy.deleteReservation(null,
+ requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java
index 1d9bedd..f73202d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java
@@ -45,6 +45,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl;
@@ -78,6 +81,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.MoveApplicationAcrossQ
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.MoveApplicationAcrossQueuesResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RenewDelegationTokenResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationDeleteRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationDeleteResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationSubmissionRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationSubmissionResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationUpdateRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationUpdateResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -99,6 +108,12 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationRequestProt
import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.MoveApplicationAcrossQueuesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.MoveApplicationAcrossQueuesResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto;
@@ -383,4 +398,50 @@ public class ApplicationClientProtocolPBServiceImpl implements ApplicationClient
throw new ServiceException(e);
}
}
+
+ @Override
+ public ReservationSubmissionResponseProto submitReservation(RpcController controller,
+ ReservationSubmissionRequestProto requestProto) throws ServiceException {
+ ReservationSubmissionRequestPBImpl request =
+ new ReservationSubmissionRequestPBImpl(requestProto);
+ try {
+ ReservationSubmissionResponse response = real.submitReservation(request);
+ return ((ReservationSubmissionResponsePBImpl) response).getProto();
+ } catch (YarnException e) {
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public ReservationUpdateResponseProto updateReservation(RpcController controller,
+ ReservationUpdateRequestProto requestProto) throws ServiceException {
+ ReservationUpdateRequestPBImpl request =
+ new ReservationUpdateRequestPBImpl(requestProto);
+ try {
+ ReservationUpdateResponse response = real.updateReservation(request);
+ return ((ReservationUpdateResponsePBImpl) response).getProto();
+ } catch (YarnException e) {
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public ReservationDeleteResponseProto deleteReservation(RpcController controller,
+ ReservationDeleteRequestProto requestProto) throws ServiceException {
+ ReservationDeleteRequestPBImpl request =
+ new ReservationDeleteRequestPBImpl(requestProto);
+ try {
+ ReservationDeleteResponse response = real.deleteReservation(request);
+ return ((ReservationDeleteResponsePBImpl) response).getProto();
+ } catch (YarnException e) {
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationDeleteRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationDeleteRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationDeleteRequestPBImpl.java
new file mode 100644
index 0000000..c235dce
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationDeleteRequestPBImpl.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteRequestProtoOrBuilder;
+
+import com.google.protobuf.TextFormat;
+
+public class ReservationDeleteRequestPBImpl extends ReservationDeleteRequest {
+
+ ReservationDeleteRequestProto proto = ReservationDeleteRequestProto
+ .getDefaultInstance();
+ ReservationDeleteRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ReservationId reservationId;
+
+ public ReservationDeleteRequestPBImpl() {
+ builder = ReservationDeleteRequestProto.newBuilder();
+ }
+
+ public ReservationDeleteRequestPBImpl(ReservationDeleteRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReservationDeleteRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.reservationId != null) {
+ builder.setReservationId(convertToProtoFormat(this.reservationId));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ReservationDeleteRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public ReservationId getReservationId() {
+ ReservationDeleteRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (reservationId != null) {
+ return reservationId;
+ }
+ if (!p.hasReservationId()) {
+ return null;
+ }
+ reservationId = convertFromProtoFormat(p.getReservationId());
+ return reservationId;
+ }
+
+ @Override
+ public void setReservationId(ReservationId reservationId) {
+ maybeInitBuilder();
+ if (reservationId == null) {
+ builder.clearReservationId();
+ return;
+ }
+ this.reservationId = reservationId;
+ }
+
+ private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
+ return new ReservationIdPBImpl(p);
+ }
+
+ private ReservationIdProto convertToProtoFormat(ReservationId t) {
+ return ((ReservationIdPBImpl) t).getProto();
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationDeleteResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationDeleteResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationDeleteResponsePBImpl.java
new file mode 100644
index 0000000..a249ab8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationDeleteResponsePBImpl.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteResponseProto;
+
+import com.google.protobuf.TextFormat;
+
+public class ReservationDeleteResponsePBImpl extends ReservationDeleteResponse {
+
+ ReservationDeleteResponseProto proto = ReservationDeleteResponseProto
+ .getDefaultInstance();
+ ReservationDeleteResponseProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public ReservationDeleteResponsePBImpl() {
+ builder = ReservationDeleteResponseProto.newBuilder();
+ }
+
+ public ReservationDeleteResponsePBImpl(ReservationDeleteResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReservationDeleteResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
[21/50] [abbrv] git commit: HDFS-7073. Allow falling back to a
non-SASL connection on DataTransferProtocol in several edge cases.
Contributed by Chris Nauroth.
Posted by cu...@apache.org.
HDFS-7073. Allow falling back to a non-SASL connection on DataTransferProtocol in several edge cases. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f85cc14e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f85cc14e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f85cc14e
Branch: refs/heads/YARN-1051
Commit: f85cc14eb49a46e81d2edcdc1ffe4d0852f193a5
Parents: 332e2e2
Author: cnauroth <cn...@apache.org>
Authored: Fri Sep 19 21:23:26 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Fri Sep 19 21:23:26 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Client.java | 68 +++++++++++++++---
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 22 ++++--
.../main/java/org/apache/hadoop/ipc/RPC.java | 38 +++++++++-
.../java/org/apache/hadoop/ipc/RpcEngine.java | 9 +++
.../apache/hadoop/ipc/WritableRpcEngine.java | 27 +++++--
.../java/org/apache/hadoop/ipc/TestRPC.java | 12 +++-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 23 +++---
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++
.../java/org/apache/hadoop/hdfs/HAUtil.java | 2 +-
.../org/apache/hadoop/hdfs/NameNodeProxies.java | 74 +++++++++++++++++---
.../sasl/SaslDataTransferClient.java | 48 ++++++++-----
.../sasl/SaslDataTransferServer.java | 26 +++++--
.../hadoop/hdfs/server/balancer/Dispatcher.java | 7 +-
.../hdfs/server/balancer/NameNodeConnector.java | 4 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 17 +++++
.../hadoop/hdfs/server/datanode/DataNode.java | 26 +++----
.../hdfs/server/namenode/NamenodeFsck.java | 20 ++----
.../ha/AbstractNNFailoverProxyProvider.java | 14 ++++
.../ha/ConfiguredFailoverProxyProvider.java | 2 +-
.../datatransfer/sasl/TestSaslDataTransfer.java | 45 +++++++++---
.../namenode/ha/TestRetryCacheWithHA.java | 2 +-
22 files changed, 382 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 2f482c2..84fe552 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -687,7 +687,8 @@ public class Client {
* a header to the server and starts
* the connection thread that waits for responses.
*/
- private synchronized void setupIOstreams() {
+ private synchronized void setupIOstreams(
+ AtomicBoolean fallbackToSimpleAuth) {
if (socket != null || shouldCloseConnection.get()) {
return;
}
@@ -738,11 +739,18 @@ public class Client {
remoteId.saslQop =
(String)saslRpcClient.getNegotiatedProperty(Sasl.QOP);
LOG.debug("Negotiated QOP is :" + remoteId.saslQop);
- } else if (UserGroupInformation.isSecurityEnabled() &&
- !fallbackAllowed) {
- throw new IOException("Server asks us to fall back to SIMPLE " +
- "auth, but this client is configured to only allow secure " +
- "connections.");
+ if (fallbackToSimpleAuth != null) {
+ fallbackToSimpleAuth.set(false);
+ }
+ } else if (UserGroupInformation.isSecurityEnabled()) {
+ if (!fallbackAllowed) {
+ throw new IOException("Server asks us to fall back to SIMPLE " +
+ "auth, but this client is configured to only allow secure " +
+ "connections.");
+ }
+ if (fallbackToSimpleAuth != null) {
+ fallbackToSimpleAuth.set(true);
+ }
}
}
@@ -1375,6 +1383,26 @@ public class Client {
/**
* Make a call, passing <code>rpcRequest</code>, to the IPC server defined by
* <code>remoteId</code>, returning the rpc respond.
+ *
+ * @param rpcKind
+ * @param rpcRequest - contains serialized method and method parameters
+ * @param remoteId - the target rpc server
+ * @param fallbackToSimpleAuth - set to true or false during this method to
+ * indicate if a secure client falls back to simple auth
+ * @returns the rpc response
+ * Throws exceptions if there are network problems or if the remote code
+ * threw an exception.
+ */
+ public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
+ ConnectionId remoteId, AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
+ return call(rpcKind, rpcRequest, remoteId, RPC.RPC_SERVICE_CLASS_DEFAULT,
+ fallbackToSimpleAuth);
+ }
+
+ /**
+ * Make a call, passing <code>rpcRequest</code>, to the IPC server defined by
+ * <code>remoteId</code>, returning the rpc response.
*
* @param rpcKind
* @param rpcRequest - contains serialized method and method parameters
@@ -1386,8 +1414,29 @@ public class Client {
*/
public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
ConnectionId remoteId, int serviceClass) throws IOException {
+ return call(rpcKind, rpcRequest, remoteId, serviceClass, null);
+ }
+
+ /**
+ * Make a call, passing <code>rpcRequest</code>, to the IPC server defined by
+ * <code>remoteId</code>, returning the rpc response.
+ *
+ * @param rpcKind
+ * @param rpcRequest - contains serialized method and method parameters
+ * @param remoteId - the target rpc server
+ * @param serviceClass - service class for RPC
+ * @param fallbackToSimpleAuth - set to true or false during this method to
+ * indicate if a secure client falls back to simple auth
+ * @returns the rpc response
+ * Throws exceptions if there are network problems or if the remote code
+ * threw an exception.
+ */
+ public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
+ ConnectionId remoteId, int serviceClass,
+ AtomicBoolean fallbackToSimpleAuth) throws IOException {
final Call call = createCall(rpcKind, rpcRequest);
- Connection connection = getConnection(remoteId, call, serviceClass);
+ Connection connection = getConnection(remoteId, call, serviceClass,
+ fallbackToSimpleAuth);
try {
connection.sendRpcRequest(call); // send the rpc request
} catch (RejectedExecutionException e) {
@@ -1444,7 +1493,8 @@ public class Client {
/** Get a connection from the pool, or create a new one and add it to the
* pool. Connections to a given ConnectionId are reused. */
private Connection getConnection(ConnectionId remoteId,
- Call call, int serviceClass) throws IOException {
+ Call call, int serviceClass, AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
if (!running.get()) {
// the client is stopped
throw new IOException("The client is stopped");
@@ -1468,7 +1518,7 @@ public class Client {
//block above. The reason for that is if the server happens to be slow,
//it will take longer to establish a connection and that will slow the
//entire system down.
- connection.setupIOstreams();
+ connection.setupIOstreams(fallbackToSimpleAuth);
return connection;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index 0ccdb71..124d835 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -27,6 +27,7 @@ import java.lang.reflect.Proxy;
import java.net.InetSocketAddress;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.SocketFactory;
@@ -84,14 +85,23 @@ public class ProtobufRpcEngine implements RpcEngine {
}
@Override
- @SuppressWarnings("unchecked")
public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy
) throws IOException {
+ return getProxy(protocol, clientVersion, addr, ticket, conf, factory,
+ rpcTimeout, connectionRetryPolicy, null);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
+ InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy,
+ AtomicBoolean fallbackToSimpleAuth) throws IOException {
final Invoker invoker = new Invoker(protocol, addr, ticket, conf, factory,
- rpcTimeout, connectionRetryPolicy);
+ rpcTimeout, connectionRetryPolicy, fallbackToSimpleAuth);
return new ProtocolProxy<T>(protocol, (T) Proxy.newProxyInstance(
protocol.getClassLoader(), new Class[]{protocol}, invoker), false);
}
@@ -115,13 +125,16 @@ public class ProtobufRpcEngine implements RpcEngine {
private final Client client;
private final long clientProtocolVersion;
private final String protocolName;
+ private AtomicBoolean fallbackToSimpleAuth;
private Invoker(Class<?> protocol, InetSocketAddress addr,
UserGroupInformation ticket, Configuration conf, SocketFactory factory,
- int rpcTimeout, RetryPolicy connectionRetryPolicy) throws IOException {
+ int rpcTimeout, RetryPolicy connectionRetryPolicy,
+ AtomicBoolean fallbackToSimpleAuth) throws IOException {
this(protocol, Client.ConnectionId.getConnectionId(
addr, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf),
conf, factory);
+ this.fallbackToSimpleAuth = fallbackToSimpleAuth;
}
/**
@@ -217,7 +230,8 @@ public class ProtobufRpcEngine implements RpcEngine {
final RpcResponseWrapper val;
try {
val = (RpcResponseWrapper) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
- new RpcRequestWrapper(rpcRequestHeader, theRequest), remoteId);
+ new RpcRequestWrapper(rpcRequestHeader, theRequest), remoteId,
+ fallbackToSimpleAuth);
} catch (Throwable e) {
if (LOG.isTraceEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 4ae7956..40f6515 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -33,6 +33,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.SocketFactory;
@@ -524,6 +525,7 @@ public class RPC {
* @param conf configuration
* @param factory socket factory
* @param rpcTimeout max time for each rpc; 0 means no timeout
+ * @param connectionRetryPolicy retry policy
* @return the proxy
* @throws IOException if any error occurs
*/
@@ -535,11 +537,43 @@ public class RPC {
SocketFactory factory,
int rpcTimeout,
RetryPolicy connectionRetryPolicy) throws IOException {
+ return getProtocolProxy(protocol, clientVersion, addr, ticket,
+ conf, factory, rpcTimeout, connectionRetryPolicy, null);
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server
+ *
+ * @param protocol protocol
+ * @param clientVersion client's version
+ * @param addr server address
+ * @param ticket security ticket
+ * @param conf configuration
+ * @param factory socket factory
+ * @param rpcTimeout max time for each rpc; 0 means no timeout
+ * @param connectionRetryPolicy retry policy
+ * @param fallbackToSimpleAuth set to true or false during calls to indicate if
+ * a secure client falls back to simple auth
+ * @return the proxy
+ * @throws IOException if any error occurs
+ */
+ public static <T> ProtocolProxy<T> getProtocolProxy(Class<T> protocol,
+ long clientVersion,
+ InetSocketAddress addr,
+ UserGroupInformation ticket,
+ Configuration conf,
+ SocketFactory factory,
+ int rpcTimeout,
+ RetryPolicy connectionRetryPolicy,
+ AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
if (UserGroupInformation.isSecurityEnabled()) {
SaslRpcServer.init(conf);
}
- return getProtocolEngine(protocol,conf).getProxy(protocol, clientVersion,
- addr, ticket, conf, factory, rpcTimeout, connectionRetryPolicy);
+ return getProtocolEngine(protocol, conf).getProxy(protocol, clientVersion,
+ addr, ticket, conf, factory, rpcTimeout, connectionRetryPolicy,
+ fallbackToSimpleAuth);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
index a8280bd..047722e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ipc;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.SocketFactory;
@@ -43,6 +44,14 @@ public interface RpcEngine {
SocketFactory factory, int rpcTimeout,
RetryPolicy connectionRetryPolicy) throws IOException;
+ /** Construct a client-side proxy object. */
+ <T> ProtocolProxy<T> getProxy(Class<T> protocol,
+ long clientVersion, InetSocketAddress addr,
+ UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory, int rpcTimeout,
+ RetryPolicy connectionRetryPolicy,
+ AtomicBoolean fallbackToSimpleAuth) throws IOException;
+
/**
* Construct a server for a protocol implementation instance.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
index 4b2dfe0..c2d9435 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
@@ -24,6 +24,7 @@ import java.lang.reflect.InvocationTargetException;
import java.net.InetSocketAddress;
import java.io.*;
+import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.SocketFactory;
@@ -212,14 +213,17 @@ public class WritableRpcEngine implements RpcEngine {
private Client.ConnectionId remoteId;
private Client client;
private boolean isClosed = false;
+ private final AtomicBoolean fallbackToSimpleAuth;
public Invoker(Class<?> protocol,
InetSocketAddress address, UserGroupInformation ticket,
Configuration conf, SocketFactory factory,
- int rpcTimeout) throws IOException {
+ int rpcTimeout, AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
this.remoteId = Client.ConnectionId.getConnectionId(address, protocol,
ticket, rpcTimeout, conf);
this.client = CLIENTS.getClient(conf, factory);
+ this.fallbackToSimpleAuth = fallbackToSimpleAuth;
}
@Override
@@ -238,7 +242,8 @@ public class WritableRpcEngine implements RpcEngine {
ObjectWritable value;
try {
value = (ObjectWritable)
- client.call(RPC.RpcKind.RPC_WRITABLE, new Invocation(method, args), remoteId);
+ client.call(RPC.RpcKind.RPC_WRITABLE, new Invocation(method, args),
+ remoteId, fallbackToSimpleAuth);
} finally {
if (traceScope != null) traceScope.close();
}
@@ -275,11 +280,25 @@ public class WritableRpcEngine implements RpcEngine {
* talking to a server at the named address.
* @param <T>*/
@Override
- @SuppressWarnings("unchecked")
public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
InetSocketAddress addr, UserGroupInformation ticket,
Configuration conf, SocketFactory factory,
int rpcTimeout, RetryPolicy connectionRetryPolicy)
+ throws IOException {
+ return getProxy(protocol, clientVersion, addr, ticket, conf, factory,
+ rpcTimeout, connectionRetryPolicy, null);
+ }
+
+ /** Construct a client-side proxy object that implements the named protocol,
+ * talking to a server at the named address.
+ * @param <T>*/
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
+ InetSocketAddress addr, UserGroupInformation ticket,
+ Configuration conf, SocketFactory factory,
+ int rpcTimeout, RetryPolicy connectionRetryPolicy,
+ AtomicBoolean fallbackToSimpleAuth)
throws IOException {
if (connectionRetryPolicy != null) {
@@ -289,7 +308,7 @@ public class WritableRpcEngine implements RpcEngine {
T proxy = (T) Proxy.newProxyInstance(protocol.getClassLoader(),
new Class[] { protocol }, new Invoker(protocol, addr, ticket, conf,
- factory, rpcTimeout));
+ factory, rpcTimeout, fallbackToSimpleAuth));
return new ProtocolProxy<T>(protocol, proxy, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
index f0e389f..c1b1bfb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
@@ -276,12 +276,22 @@ public class TestRPC {
*/
private static class StoppedRpcEngine implements RpcEngine {
- @SuppressWarnings("unchecked")
@Override
public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy
) throws IOException {
+ return getProxy(protocol, clientVersion, addr, ticket, conf, factory,
+ rpcTimeout, connectionRetryPolicy, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
+ InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory, int rpcTimeout,
+ RetryPolicy connectionRetryPolicy, AtomicBoolean fallbackToSimpleAuth
+ ) throws IOException {
T proxy = (T) Proxy.newProxyInstance(protocol.getClassLoader(),
new Class[] { protocol }, new StoppedInvocationHandler());
return new ProtocolProxy<T>(protocol, proxy, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fa9bbcd..9babe96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -900,6 +900,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7105. Fix TestJournalNode#testFailToStartWithBadConfig to match log
output change. (Ray Chiang via cnauroth)
+ HDFS-7105. Allow falling back to a non-SASL connection on
+ DataTransferProtocol in several edge cases. (cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 3f978fb..ed08be0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -22,8 +22,6 @@ import static org.apache.hadoop.crypto.key.KeyProviderCryptoExtension
.EncryptedKeyVersion;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT;
@@ -90,6 +88,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -616,13 +615,15 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY,
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT);
NameNodeProxies.ProxyAndInfo<ClientProtocol> proxyInfo = null;
+ AtomicBoolean nnFallbackToSimpleAuth = new AtomicBoolean(false);
if (numResponseToDrop > 0) {
// This case is used for testing.
LOG.warn(DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY
+ " is set to " + numResponseToDrop
+ ", this hacked client will proactively drop responses");
proxyInfo = NameNodeProxies.createProxyWithLossyRetryHandler(conf,
- nameNodeUri, ClientProtocol.class, numResponseToDrop);
+ nameNodeUri, ClientProtocol.class, numResponseToDrop,
+ nnFallbackToSimpleAuth);
}
if (proxyInfo != null) {
@@ -637,7 +638,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
Preconditions.checkArgument(nameNodeUri != null,
"null URI");
proxyInfo = NameNodeProxies.createProxy(conf, nameNodeUri,
- ClientProtocol.class);
+ ClientProtocol.class, nnFallbackToSimpleAuth);
this.dtService = proxyInfo.getDelegationTokenService();
this.namenode = proxyInfo.getProxy();
}
@@ -675,10 +676,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
this.saslClient = new SaslDataTransferClient(
DataTransferSaslUtil.getSaslPropertiesResolver(conf),
- TrustedChannelResolver.getInstance(conf),
- conf.getBoolean(
- IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
- IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT));
+ TrustedChannelResolver.getInstance(conf), nnFallbackToSimpleAuth);
}
/**
@@ -3113,4 +3111,13 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
public void setKeyProvider(KeyProviderCryptoExtension provider) {
this.provider = provider;
}
+
+ /**
+ * Returns the SaslDataTransferClient configured for this DFSClient.
+ *
+ * @return SaslDataTransferClient configured for this DFSClient
+ */
+ public SaslDataTransferClient getSaslDataTransferClient() {
+ return saslClient;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 609b4c6..3c5358f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -589,6 +589,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_DATA_ENCRYPTION_ALGORITHM_KEY = "dfs.encrypt.data.transfer.algorithm";
public static final String DFS_TRUSTEDCHANNEL_RESOLVER_CLASS = "dfs.trustedchannel.resolver.class";
public static final String DFS_DATA_TRANSFER_PROTECTION_KEY = "dfs.data.transfer.protection";
+ public static final String DFS_DATA_TRANSFER_PROTECTION_DEFAULT = "";
public static final String DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY = "dfs.data.transfer.saslproperties.resolver.class";
public static final int DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES_DEFAULT = 100;
public static final String DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES = "dfs.namenode.list.encryption.zones.num.responses";
@@ -703,4 +704,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_DEFAULT =
1000;
+ public static final String IGNORE_SECURE_PORTS_FOR_TESTING_KEY =
+ "ignore.secure.ports.for.testing";
+ public static final boolean IGNORE_SECURE_PORTS_FOR_TESTING_DEFAULT = false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
index 90acede..f91f709 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
@@ -244,7 +244,7 @@ public class HAUtil {
// Create the proxy provider. Actual proxy is not created.
AbstractNNFailoverProxyProvider<ClientProtocol> provider = NameNodeProxies
.createFailoverProxyProvider(conf, nameNodeUri, ClientProtocol.class,
- false);
+ false, null);
// No need to use logical URI since failover is not configured.
if (provider == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
index 1765334..fcc2f5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
@@ -36,6 +36,7 @@ import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -145,13 +146,37 @@ public class NameNodeProxies {
@SuppressWarnings("unchecked")
public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
URI nameNodeUri, Class<T> xface) throws IOException {
+ return createProxy(conf, nameNodeUri, xface, null);
+ }
+
+ /**
+ * Creates the namenode proxy with the passed protocol. This will handle
+ * creation of either HA- or non-HA-enabled proxy objects, depending upon
+ * if the provided URI is a configured logical URI.
+ *
+ * @param conf the configuration containing the required IPC
+ * properties, client failover configurations, etc.
+ * @param nameNodeUri the URI pointing either to a specific NameNode
+ * or to a logical nameservice.
+ * @param xface the IPC interface which should be created
+ * @param fallbackToSimpleAuth set to true or false during calls to indicate if
+ * a secure client falls back to simple auth
+ * @return an object containing both the proxy and the associated
+ * delegation token service it corresponds to
+ * @throws IOException if there is an error creating the proxy
+ **/
+ @SuppressWarnings("unchecked")
+ public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
+ URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
AbstractNNFailoverProxyProvider<T> failoverProxyProvider =
- createFailoverProxyProvider(conf, nameNodeUri, xface, true);
+ createFailoverProxyProvider(conf, nameNodeUri, xface, true,
+ fallbackToSimpleAuth);
if (failoverProxyProvider == null) {
// Non-HA case
return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
- UserGroupInformation.getCurrentUser(), true);
+ UserGroupInformation.getCurrentUser(), true, fallbackToSimpleAuth);
} else {
// HA case
Conf config = new Conf(conf);
@@ -187,6 +212,8 @@ public class NameNodeProxies {
* or to a logical nameservice.
* @param xface the IPC interface which should be created
* @param numResponseToDrop The number of responses to drop for each RPC call
+ * @param fallbackToSimpleAuth set to true or false during calls to indicate if
+ * a secure client falls back to simple auth
* @return an object containing both the proxy and the associated
* delegation token service it corresponds to. Will return null of the
* given configuration does not support HA.
@@ -195,10 +222,12 @@ public class NameNodeProxies {
@SuppressWarnings("unchecked")
public static <T> ProxyAndInfo<T> createProxyWithLossyRetryHandler(
Configuration config, URI nameNodeUri, Class<T> xface,
- int numResponseToDrop) throws IOException {
+ int numResponseToDrop, AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
Preconditions.checkArgument(numResponseToDrop > 0);
AbstractNNFailoverProxyProvider<T> failoverProxyProvider =
- createFailoverProxyProvider(config, nameNodeUri, xface, true);
+ createFailoverProxyProvider(config, nameNodeUri, xface, true,
+ fallbackToSimpleAuth);
if (failoverProxyProvider != null) { // HA case
int delay = config.getInt(
@@ -257,12 +286,35 @@ public class NameNodeProxies {
public static <T> ProxyAndInfo<T> createNonHAProxy(
Configuration conf, InetSocketAddress nnAddr, Class<T> xface,
UserGroupInformation ugi, boolean withRetries) throws IOException {
+ return createNonHAProxy(conf, nnAddr, xface, ugi, withRetries, null);
+ }
+
+ /**
+ * Creates an explicitly non-HA-enabled proxy object. Most of the time you
+ * don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
+ *
+ * @param conf the configuration object
+ * @param nnAddr address of the remote NN to connect to
+ * @param xface the IPC interface which should be created
+ * @param ugi the user who is making the calls on the proxy object
+ * @param withRetries certain interfaces have a non-standard retry policy
+ * @param fallbackToSimpleAuth - set to true or false during this method to
+ * indicate if a secure client falls back to simple auth
+ * @return an object containing both the proxy and the associated
+ * delegation token service it corresponds to
+ * @throws IOException
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> ProxyAndInfo<T> createNonHAProxy(
+ Configuration conf, InetSocketAddress nnAddr, Class<T> xface,
+ UserGroupInformation ugi, boolean withRetries,
+ AtomicBoolean fallbackToSimpleAuth) throws IOException {
Text dtService = SecurityUtil.buildTokenService(nnAddr);
T proxy;
if (xface == ClientProtocol.class) {
proxy = (T) createNNProxyWithClientProtocol(nnAddr, conf, ugi,
- withRetries);
+ withRetries, fallbackToSimpleAuth);
} else if (xface == JournalProtocol.class) {
proxy = (T) createNNProxyWithJournalProtocol(nnAddr, conf, ugi);
} else if (xface == NamenodeProtocol.class) {
@@ -351,7 +403,8 @@ public class NameNodeProxies {
private static ClientProtocol createNNProxyWithClientProtocol(
InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
- boolean withRetries) throws IOException {
+ boolean withRetries, AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);
final RetryPolicy defaultPolicy =
@@ -367,8 +420,8 @@ public class NameNodeProxies {
ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
ClientNamenodeProtocolPB.class, version, address, ugi, conf,
NetUtils.getDefaultSocketFactory(conf),
- org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy)
- .getProxy();
+ org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy,
+ fallbackToSimpleAuth).getProxy();
if (withRetries) { // create the proxy with retries
@@ -440,8 +493,8 @@ public class NameNodeProxies {
/** Creates the Failover proxy provider instance*/
@VisibleForTesting
public static <T> AbstractNNFailoverProxyProvider<T> createFailoverProxyProvider(
- Configuration conf, URI nameNodeUri, Class<T> xface, boolean checkPort)
- throws IOException {
+ Configuration conf, URI nameNodeUri, Class<T> xface, boolean checkPort,
+ AtomicBoolean fallbackToSimpleAuth) throws IOException {
Class<FailoverProxyProvider<T>> failoverProxyProviderClass = null;
AbstractNNFailoverProxyProvider<T> providerNN;
Preconditions.checkArgument(
@@ -490,6 +543,7 @@ public class NameNodeProxies {
+ " and does not use port information.");
}
}
+ providerNN.setFallbackToSimpleAuth(fallbackToSimpleAuth);
return providerNN;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
index 643af4a..9df9929 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
@@ -28,6 +28,7 @@ import java.io.OutputStream;
import java.net.InetAddress;
import java.net.Socket;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import javax.security.auth.callback.Callback;
import javax.security.auth.callback.CallbackHandler;
@@ -71,21 +72,38 @@ public class SaslDataTransferClient {
private static final Logger LOG = LoggerFactory.getLogger(
SaslDataTransferClient.class);
- private final boolean fallbackToSimpleAuthAllowed;
+ private final AtomicBoolean fallbackToSimpleAuth;
private final SaslPropertiesResolver saslPropsResolver;
private final TrustedChannelResolver trustedChannelResolver;
/**
+ * Creates a new SaslDataTransferClient. This constructor is used in cases
+ * where it is not relevant to track if a secure client did a fallback to
+ * simple auth. For intra-cluster connections between data nodes in the same
+ * cluster, we can assume that all run under the same security configuration.
+ *
+ * @param saslPropsResolver for determining properties of SASL negotiation
+ * @param trustedChannelResolver for identifying trusted connections that do
+ * not require SASL negotiation
+ */
+ public SaslDataTransferClient(SaslPropertiesResolver saslPropsResolver,
+ TrustedChannelResolver trustedChannelResolver) {
+ this(saslPropsResolver, trustedChannelResolver, null);
+ }
+
+ /**
* Creates a new SaslDataTransferClient.
*
* @param saslPropsResolver for determining properties of SASL negotiation
* @param trustedChannelResolver for identifying trusted connections that do
* not require SASL negotiation
+ * @param fallbackToSimpleAuth checked on each attempt at general SASL
+ * handshake, if true forces use of simple auth
*/
public SaslDataTransferClient(SaslPropertiesResolver saslPropsResolver,
TrustedChannelResolver trustedChannelResolver,
- boolean fallbackToSimpleAuthAllowed) {
- this.fallbackToSimpleAuthAllowed = fallbackToSimpleAuthAllowed;
+ AtomicBoolean fallbackToSimpleAuth) {
+ this.fallbackToSimpleAuth = fallbackToSimpleAuth;
this.saslPropsResolver = saslPropsResolver;
this.trustedChannelResolver = trustedChannelResolver;
}
@@ -221,22 +239,26 @@ public class SaslDataTransferClient {
"SASL client skipping handshake in secured configuration with "
+ "privileged port for addr = {}, datanodeId = {}", addr, datanodeId);
return null;
- } else if (accessToken.getIdentifier().length == 0) {
- if (!fallbackToSimpleAuthAllowed) {
- throw new IOException(
- "No block access token was provided (insecure cluster), but this " +
- "client is configured to allow only secure connections.");
- }
+ } else if (fallbackToSimpleAuth != null && fallbackToSimpleAuth.get()) {
LOG.debug(
"SASL client skipping handshake in secured configuration with "
+ "unsecured cluster for addr = {}, datanodeId = {}", addr, datanodeId);
return null;
- } else {
+ } else if (saslPropsResolver != null) {
LOG.debug(
"SASL client doing general handshake for addr = {}, datanodeId = {}",
addr, datanodeId);
return getSaslStreams(addr, underlyingOut, underlyingIn, accessToken,
datanodeId);
+ } else {
+ // It's a secured cluster using non-privileged ports, but no SASL. The
+ // only way this can happen is if the DataNode has
+ // ignore.secure.ports.for.testing configured, so this is a rare edge case.
+ LOG.debug(
+ "SASL client skipping handshake in secured configuration with no SASL "
+ + "protection configured for addr = {}, datanodeId = {}",
+ addr, datanodeId);
+ return null;
}
}
@@ -348,12 +370,6 @@ public class SaslDataTransferClient {
OutputStream underlyingOut, InputStream underlyingIn,
Token<BlockTokenIdentifier> accessToken, DatanodeID datanodeId)
throws IOException {
- if (saslPropsResolver == null) {
- throw new IOException(String.format("Cannot create a secured " +
- "connection if DataNode listens on unprivileged port (%d) and no " +
- "protection is defined in configuration property %s.",
- datanodeId.getXferPort(), DFS_DATA_TRANSFER_PROTECTION_KEY));
- }
Map<String, String> saslProps = saslPropsResolver.getClientProperties(addr);
String userName = buildUserName(accessToken);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
index 7857057..2b82c82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
@@ -112,11 +112,29 @@ public class SaslDataTransferServer {
"SASL server skipping handshake in unsecured configuration for "
+ "peer = {}, datanodeId = {}", peer, datanodeId);
return new IOStreamPair(underlyingIn, underlyingOut);
- } else {
+ } else if (dnConf.getSaslPropsResolver() != null) {
LOG.debug(
"SASL server doing general handshake for peer = {}, datanodeId = {}",
peer, datanodeId);
return getSaslStreams(peer, underlyingOut, underlyingIn, datanodeId);
+ } else if (dnConf.getIgnoreSecurePortsForTesting()) {
+ // It's a secured cluster using non-privileged ports, but no SASL. The
+ // only way this can happen is if the DataNode has
+ // ignore.secure.ports.for.testing configured, so this is a rare edge case.
+ LOG.debug(
+ "SASL server skipping handshake in secured configuration with no SASL "
+ + "protection configured for peer = {}, datanodeId = {}",
+ peer, datanodeId);
+ return new IOStreamPair(underlyingIn, underlyingOut);
+ } else {
+ // The error message here intentionally does not mention
+ // ignore.secure.ports.for.testing. That's intended for dev use only.
+ // This code path is not expected to execute ever, because DataNode startup
+ // checks for invalid configuration and aborts.
+ throw new IOException(String.format("Cannot create a secured " +
+ "connection if DataNode listens on unprivileged port (%d) and no " +
+ "protection is defined in configuration property %s.",
+ datanodeId.getXferPort(), DFS_DATA_TRANSFER_PROTECTION_KEY));
}
}
@@ -257,12 +275,6 @@ public class SaslDataTransferServer {
private IOStreamPair getSaslStreams(Peer peer, OutputStream underlyingOut,
InputStream underlyingIn, final DatanodeID datanodeId) throws IOException {
SaslPropertiesResolver saslPropsResolver = dnConf.getSaslPropsResolver();
- if (saslPropsResolver == null) {
- throw new IOException(String.format("Cannot create a secured " +
- "connection if DataNode listens on unprivileged port (%d) and no " +
- "protection is defined in configuration property %s.",
- datanodeId.getXferPort(), DFS_DATA_TRANSFER_PROTECTION_KEY));
- }
Map<String, String> saslProps = saslPropsResolver.getServerProperties(
getPeerAddress(peer));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index df6aa99..cea1ab7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -48,8 +48,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.StorageType;
@@ -787,12 +785,9 @@ public class Dispatcher {
: Executors.newFixedThreadPool(dispatcherThreads);
this.maxConcurrentMovesPerNode = maxConcurrentMovesPerNode;
- final boolean fallbackToSimpleAuthAllowed = conf.getBoolean(
- CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
- CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT);
this.saslClient = new SaslDataTransferClient(
DataTransferSaslUtil.getSaslPropertiesResolver(conf),
- TrustedChannelResolver.getInstance(conf), fallbackToSimpleAuthAllowed);
+ TrustedChannelResolver.getInstance(conf), nnc.fallbackToSimpleAuth);
}
public DistributedFileSystem getDistributedFileSystem() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
index d27f33f..9162531 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -101,6 +102,7 @@ public class NameNodeConnector implements Closeable {
private final NamenodeProtocol namenode;
private final ClientProtocol client;
private final KeyManager keyManager;
+ final AtomicBoolean fallbackToSimpleAuth = new AtomicBoolean(false);
private final DistributedFileSystem fs;
private final Path idPath;
@@ -120,7 +122,7 @@ public class NameNodeConnector implements Closeable {
this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri,
NamenodeProtocol.class).getProxy();
this.client = NameNodeProxies.createProxy(conf, nameNodeUri,
- ClientProtocol.class).getProxy();
+ ClientProtocol.class, fallbackToSimpleAuth).getProxy();
this.fs = (DistributedFileSystem)FileSystem.get(nameNodeUri, conf);
final NamespaceInfo namespaceinfo = namenode.versionRequest();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 4a36472..3127682 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -48,6 +48,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_DEF
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATA_ENCRYPTION_ALGORITHM_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_RESTART_REPLICA_EXPIRY_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_RESTART_REPLICA_EXPIRY_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_DEFAULT;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -90,6 +92,7 @@ public class DNConf {
final String encryptionAlgorithm;
final SaslPropertiesResolver saslPropsResolver;
final TrustedChannelResolver trustedChannelResolver;
+ private final boolean ignoreSecurePortsForTesting;
final long xceiverStopTimeout;
final long restartReplicaExpiry;
@@ -173,6 +176,9 @@ public class DNConf {
this.trustedChannelResolver = TrustedChannelResolver.getInstance(conf);
this.saslPropsResolver = DataTransferSaslUtil.getSaslPropertiesResolver(
conf);
+ this.ignoreSecurePortsForTesting = conf.getBoolean(
+ IGNORE_SECURE_PORTS_FOR_TESTING_KEY,
+ IGNORE_SECURE_PORTS_FOR_TESTING_DEFAULT);
this.xceiverStopTimeout = conf.getLong(
DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY,
@@ -238,4 +244,15 @@ public class DNConf {
public TrustedChannelResolver getTrustedChannelResolver() {
return trustedChannelResolver;
}
+
+ /**
+ * Returns true if configuration is set to skip checking for proper
+ * port configuration in a secured cluster. This is only intended for use in
+ * dev testing.
+ *
+ * @return true if configured to skip checking secured port configuration
+ */
+ public boolean getIgnoreSecurePortsForTesting() {
+ return ignoreSecurePortsForTesting;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 57ff2fa..b1ef186 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY;
@@ -46,9 +44,12 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATA_TRANSFER_PROTECTION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
import java.io.BufferedOutputStream;
@@ -170,6 +171,7 @@ import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.unix.DomainSocket;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.SaslPropertiesResolver;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -967,8 +969,6 @@ public class DataNode extends ReconfigurableBase
SecureResources resources
) throws IOException {
- checkSecureConfig(conf, resources);
-
// settings global for all BPs in the Data Node
this.secureResources = resources;
synchronized (this) {
@@ -976,6 +976,8 @@ public class DataNode extends ReconfigurableBase
}
this.conf = conf;
this.dnConf = new DNConf(conf);
+ checkSecureConfig(dnConf, conf, resources);
+
this.spanReceiverHost = SpanReceiverHost.getInstance(conf);
if (dnConf.maxLockedMemory > 0) {
@@ -1031,10 +1033,7 @@ public class DataNode extends ReconfigurableBase
// exit without having to explicitly shutdown its thread pool.
readaheadPool = ReadaheadPool.getInstance();
saslClient = new SaslDataTransferClient(dnConf.saslPropsResolver,
- dnConf.trustedChannelResolver,
- conf.getBoolean(
- IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
- IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT));
+ dnConf.trustedChannelResolver);
saslServer = new SaslDataTransferServer(dnConf, blockPoolTokenSecretManager);
}
@@ -1054,23 +1053,24 @@ public class DataNode extends ReconfigurableBase
* must check if the target port is a privileged port, and if so, skip the
* SASL handshake.
*
+ * @param dnConf DNConf to check
* @param conf Configuration to check
* @param resources SecuredResources obtained for DataNode
* @throws RuntimeException if security enabled, but configuration is insecure
*/
- private static void checkSecureConfig(Configuration conf,
+ private static void checkSecureConfig(DNConf dnConf, Configuration conf,
SecureResources resources) throws RuntimeException {
if (!UserGroupInformation.isSecurityEnabled()) {
return;
}
- String dataTransferProtection = conf.get(DFS_DATA_TRANSFER_PROTECTION_KEY);
- if (resources != null && dataTransferProtection == null) {
+ SaslPropertiesResolver saslPropsResolver = dnConf.getSaslPropsResolver();
+ if (resources != null && saslPropsResolver == null) {
return;
}
- if (conf.getBoolean("ignore.secure.ports.for.testing", false)) {
+ if (dnConf.getIgnoreSecurePortsForTesting()) {
return;
}
- if (dataTransferProtection != null &&
+ if (saslPropsResolver != null &&
DFSUtil.getHttpPolicy(conf) == HttpConfig.Policy.HTTPS_ONLY &&
resources == null) {
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 5cc8a47..a187123 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY;
-
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
@@ -59,10 +56,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
-import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
-import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory;
-import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
@@ -161,7 +155,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
private List<String> snapshottableDirs = null;
private final BlockPlacementPolicy bpPolicy;
- private final SaslDataTransferClient saslClient;
/**
* Filesystem checker.
@@ -188,12 +181,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
networktopology,
namenode.getNamesystem().getBlockManager().getDatanodeManager()
.getHost2DatanodeMap());
- this.saslClient = new SaslDataTransferClient(
- DataTransferSaslUtil.getSaslPropertiesResolver(conf),
- TrustedChannelResolver.getInstance(conf),
- conf.getBoolean(
- IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
- IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT));
for (Iterator<String> it = pmap.keySet().iterator(); it.hasNext();) {
String key = it.next();
@@ -594,7 +581,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
* bad. Both places should be refactored to provide a method to copy blocks
* around.
*/
- private void copyBlock(DFSClient dfs, LocatedBlock lblock,
+ private void copyBlock(final DFSClient dfs, LocatedBlock lblock,
OutputStream fos) throws Exception {
int failures = 0;
InetSocketAddress targetAddr = null;
@@ -647,8 +634,9 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
try {
s.connect(addr, HdfsServerConstants.READ_TIMEOUT);
s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
- peer = TcpPeerServer.peerFromSocketAndKey(saslClient, s,
- NamenodeFsck.this, blockToken, datanodeId);
+ peer = TcpPeerServer.peerFromSocketAndKey(
+ dfs.getSaslDataTransferClient(), s, NamenodeFsck.this,
+ blockToken, datanodeId);
} finally {
if (peer == null) {
IOUtils.closeQuietly(s);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
index 3c0edfd..08e82be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
@@ -18,12 +18,16 @@
package org.apache.hadoop.hdfs.server.namenode.ha;
+import java.util.concurrent.atomic.AtomicBoolean;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.retry.FailoverProxyProvider;
public abstract class AbstractNNFailoverProxyProvider<T> implements
FailoverProxyProvider <T> {
+ protected AtomicBoolean fallbackToSimpleAuth;
+
/**
* Inquire whether logical HA URI is used for the implementation. If it is
* used, a special token handling may be needed to make sure a token acquired
@@ -32,4 +36,14 @@ public abstract class AbstractNNFailoverProxyProvider<T> implements
* @return true if logical HA URI is used. false, if not used.
*/
public abstract boolean useLogicalURI();
+
+ /**
+ * Set for tracking if a secure client falls back to simple auth.
+ *
+ * @param fallbackToSimpleAuth - set to true or false during this method to
+ * indicate if a secure client falls back to simple auth
+ */
+ public void setFallbackToSimpleAuth(AtomicBoolean fallbackToSimpleAuth) {
+ this.fallbackToSimpleAuth = fallbackToSimpleAuth;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
index 4d196a2..06aa8fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
@@ -122,7 +122,7 @@ public class ConfiguredFailoverProxyProvider<T> extends
if (current.namenode == null) {
try {
current.namenode = NameNodeProxies.createNonHAProxy(conf,
- current.address, xface, ugi, false).getProxy();
+ current.address, xface, ugi, false, fallbackToSimpleAuth).getProxy();
} catch (IOException e) {
LOG.error("Failed to create RPC proxy to NameNode", e);
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
index 7602f44..0d860b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.hdfs.protocol.datatransfer.sasl;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY;
+
import static org.junit.Assert.*;
import java.io.IOException;
@@ -29,11 +32,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
public class TestSaslDataTransfer extends SaslDataTransferTestCase {
@@ -49,6 +54,9 @@ public class TestSaslDataTransfer extends SaslDataTransferTestCase {
@Rule
public ExpectedException exception = ExpectedException.none();
+ @Rule
+ public Timeout timeout = new Timeout(60000);
+
@After
public void shutdown() {
IOUtils.cleanup(null, fs);
@@ -99,17 +107,6 @@ public class TestSaslDataTransfer extends SaslDataTransferTestCase {
}
@Test
- public void testClientSaslNoServerSasl() throws Exception {
- HdfsConfiguration clusterConf = createSecureConfig("");
- startCluster(clusterConf);
- HdfsConfiguration clientConf = new HdfsConfiguration(clusterConf);
- clientConf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, "authentication");
- exception.expect(IOException.class);
- exception.expectMessage("could only be replicated to 0 nodes");
- doTest(clientConf);
- }
-
- @Test
public void testServerSaslNoClientSasl() throws Exception {
HdfsConfiguration clusterConf = createSecureConfig(
"authentication,integrity,privacy");
@@ -121,6 +118,32 @@ public class TestSaslDataTransfer extends SaslDataTransferTestCase {
doTest(clientConf);
}
+ @Test
+ public void testDataNodeAbortsIfNoSasl() throws Exception {
+ HdfsConfiguration clusterConf = createSecureConfig("");
+ exception.expect(RuntimeException.class);
+ exception.expectMessage("Cannot start secure DataNode");
+ startCluster(clusterConf);
+ }
+
+ @Test
+ public void testDataNodeAbortsIfNotHttpsOnly() throws Exception {
+ HdfsConfiguration clusterConf = createSecureConfig("authentication");
+ clusterConf.set(DFS_HTTP_POLICY_KEY,
+ HttpConfig.Policy.HTTP_AND_HTTPS.name());
+ exception.expect(RuntimeException.class);
+ exception.expectMessage("Cannot start secure DataNode");
+ startCluster(clusterConf);
+ }
+
+ @Test
+ public void testNoSaslAndSecurePortsIgnored() throws Exception {
+ HdfsConfiguration clusterConf = createSecureConfig("");
+ clusterConf.setBoolean(IGNORE_SECURE_PORTS_FOR_TESTING_KEY, true);
+ startCluster(clusterConf);
+ doTest(clusterConf);
+ }
+
/**
* Tests DataTransferProtocol with the given client configuration.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85cc14e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
index 899b888..8f7d11a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
@@ -194,7 +194,7 @@ public class TestRetryCacheWithHA {
URI nnUri = dfs.getUri();
FailoverProxyProvider<ClientProtocol> failoverProxyProvider =
NameNodeProxies.createFailoverProxyProvider(conf,
- nnUri, ClientProtocol.class, true);
+ nnUri, ClientProtocol.class, true, null);
InvocationHandler dummyHandler = new DummyRetryInvocationHandler(
failoverProxyProvider, RetryPolicies
.failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL,
[22/50] [abbrv] git commit: HADOOP-11017. KMS delegation token secret
manager should be able to use zookeeper as store. (asuresh via tucu)
Posted by cu...@apache.org.
HADOOP-11017. KMS delegation token secret manager should be able to use zookeeper as store. (asuresh via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db890eef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db890eef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db890eef
Branch: refs/heads/YARN-1051
Commit: db890eef3208cc557476fa510f7a253ba22bc68a
Parents: f85cc14
Author: Alejandro Abdelnur <tu...@apache.org>
Authored: Sat Sep 20 08:20:34 2014 -0700
Committer: Alejandro Abdelnur <tu...@apache.org>
Committed: Sat Sep 20 08:21:44 2014 -0700
----------------------------------------------------------------------
.../util/ZKSignerSecretProvider.java | 2 +
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
hadoop-common-project/hadoop-common/pom.xml | 13 +
.../AbstractDelegationTokenSecretManager.java | 132 +++-
.../ZKDelegationTokenSecretManager.java | 727 +++++++++++++++++++
.../DelegationTokenAuthenticationFilter.java | 10 +
.../DelegationTokenAuthenticationHandler.java | 26 +-
.../delegation/web/DelegationTokenManager.java | 76 +-
.../token/delegation/TestDelegationToken.java | 4 +-
.../TestZKDelegationTokenSecretManager.java | 68 ++
.../web/TestDelegationTokenManager.java | 17 +-
hadoop-project/pom.xml | 10 +
12 files changed, 1013 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
index a17b6d4..6c0fbbb 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
@@ -197,6 +197,8 @@ public class ZKSignerSecretProvider extends RolloverSignerSecretProvider {
client = (CuratorFramework) curatorClientObj;
} else {
client = createCuratorClient(config);
+ servletContext.setAttribute(
+ ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE, client);
}
this.tokenValidity = tokenValidity;
shouldDisconnect = Boolean.parseBoolean(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 70579c3..2b07f8d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -537,6 +537,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10970. Cleanup KMS configuration keys. (wang)
+ HADOOP-11017. KMS delegation token secret manager should be able to use
+ zookeeper as store. (asuresh via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 0183e29..32e9525 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -219,6 +219,19 @@
<artifactId>jsch</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<scope>compile</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
index b9e26b5..f5e7bc9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
@@ -127,7 +127,7 @@ extends AbstractDelegationTokenIdentifier>
public synchronized void reset() {
currentId = 0;
allKeys.clear();
- delegationTokenSequenceNumber = 0;
+ setDelegationTokenSeqNum(0);
currentTokens.clear();
}
@@ -141,7 +141,7 @@ extends AbstractDelegationTokenIdentifier>
if (key.getKeyId() > currentId) {
currentId = key.getKeyId();
}
- allKeys.put(key.getKeyId(), key);
+ storeDelegationKey(key);
}
public synchronized DelegationKey[] getAllKeys() {
@@ -163,25 +163,109 @@ extends AbstractDelegationTokenIdentifier>
return;
}
+ // for ZK based secretManager
+ protected void updateMasterKey(DelegationKey key) throws IOException{
+ return;
+ }
+
// RM
protected void removeStoredMasterKey(DelegationKey key) {
return;
}
// RM
- protected void storeNewToken(TokenIdent ident, long renewDate) {
+ protected void storeNewToken(TokenIdent ident, long renewDate) throws IOException{
return;
}
+
// RM
protected void removeStoredToken(TokenIdent ident) throws IOException {
}
// RM
- protected void updateStoredToken(TokenIdent ident, long renewDate) {
+ protected void updateStoredToken(TokenIdent ident, long renewDate) throws IOException {
return;
}
/**
+ * For subclasses externalizing the storage, for example Zookeeper
+ * based implementations
+ */
+ protected int getDelegationTokenSeqNum() {
+ return delegationTokenSequenceNumber;
+ }
+
+ /**
+ * For subclasses externalizing the storage, for example Zookeeper
+ * based implementations
+ */
+ protected int incrementDelegationTokenSeqNum() {
+ return ++delegationTokenSequenceNumber;
+ }
+
+ /**
+ * For subclasses externalizing the storage, for example Zookeeper
+ * based implementations
+ */
+ protected void setDelegationTokenSeqNum(int seqNum) {
+ delegationTokenSequenceNumber = seqNum;
+ }
+
+ /**
+ * For subclasses externalizing the storage, for example Zookeeper
+ * based implementations
+ */
+ protected DelegationKey getDelegationKey(int keyId) {
+ return allKeys.get(keyId);
+ }
+
+ /**
+ * For subclasses externalizing the storage, for example Zookeeper
+ * based implementations
+ */
+ protected void storeDelegationKey(DelegationKey key) throws IOException {
+ allKeys.put(key.getKeyId(), key);
+ storeNewMasterKey(key);
+ }
+
+ /**
+ * For subclasses externalizing the storage, for example Zookeeper
+ * based implementations
+ */
+ protected void updateDelegationKey(DelegationKey key) throws IOException {
+ allKeys.put(key.getKeyId(), key);
+ updateMasterKey(key);
+ }
+
+ /**
+ * For subclasses externalizing the storage, for example Zookeeper
+ * based implementations
+ */
+ protected DelegationTokenInformation getTokenInfo(TokenIdent ident) {
+ return currentTokens.get(ident);
+ }
+
+ /**
+ * For subclasses externalizing the storage, for example Zookeeper
+ * based implementations
+ */
+ protected void storeToken(TokenIdent ident,
+ DelegationTokenInformation tokenInfo) throws IOException {
+ currentTokens.put(ident, tokenInfo);
+ storeNewToken(ident, tokenInfo.getRenewDate());
+ }
+
+ /**
+ * For subclasses externalizing the storage, for example Zookeeper
+ * based implementations
+ */
+ protected void updateToken(TokenIdent ident,
+ DelegationTokenInformation tokenInfo) throws IOException {
+ currentTokens.put(ident, tokenInfo);
+ updateStoredToken(ident, tokenInfo.getRenewDate());
+ }
+
+ /**
* This method is intended to be used for recovering persisted delegation
* tokens
* @param identifier identifier read from persistent storage
@@ -196,17 +280,18 @@ extends AbstractDelegationTokenIdentifier>
"Can't add persisted delegation token to a running SecretManager.");
}
int keyId = identifier.getMasterKeyId();
- DelegationKey dKey = allKeys.get(keyId);
+ DelegationKey dKey = getDelegationKey(keyId);
if (dKey == null) {
LOG.warn("No KEY found for persisted identifier " + identifier.toString());
return;
}
byte[] password = createPassword(identifier.getBytes(), dKey.getKey());
- if (identifier.getSequenceNumber() > this.delegationTokenSequenceNumber) {
- this.delegationTokenSequenceNumber = identifier.getSequenceNumber();
+ int delegationTokenSeqNum = getDelegationTokenSeqNum();
+ if (identifier.getSequenceNumber() > delegationTokenSeqNum) {
+ setDelegationTokenSeqNum(identifier.getSequenceNumber());
}
- if (currentTokens.get(identifier) == null) {
- currentTokens.put(identifier, new DelegationTokenInformation(renewDate,
+ if (getTokenInfo(identifier) == null) {
+ storeToken(identifier, new DelegationTokenInformation(renewDate,
password, getTrackingIdIfEnabled(identifier)));
} else {
throw new IOException("Same delegation token being added twice.");
@@ -234,7 +319,7 @@ extends AbstractDelegationTokenIdentifier>
synchronized (this) {
currentId = newKey.getKeyId();
currentKey = newKey;
- allKeys.put(currentKey.getKeyId(), currentKey);
+ storeDelegationKey(currentKey);
}
}
@@ -252,7 +337,7 @@ extends AbstractDelegationTokenIdentifier>
* updateMasterKey() isn't called at expected interval. Add it back to
* allKeys just in case.
*/
- allKeys.put(currentKey.getKeyId(), currentKey);
+ updateDelegationKey(currentKey);
}
updateCurrentKey();
}
@@ -276,19 +361,25 @@ extends AbstractDelegationTokenIdentifier>
protected synchronized byte[] createPassword(TokenIdent identifier) {
int sequenceNum;
long now = Time.now();
- sequenceNum = ++delegationTokenSequenceNumber;
+ sequenceNum = incrementDelegationTokenSeqNum();
identifier.setIssueDate(now);
identifier.setMaxDate(now + tokenMaxLifetime);
identifier.setMasterKeyId(currentId);
identifier.setSequenceNumber(sequenceNum);
LOG.info("Creating password for identifier: " + identifier);
byte[] password = createPassword(identifier.getBytes(), currentKey.getKey());
- storeNewToken(identifier, now + tokenRenewInterval);
- currentTokens.put(identifier, new DelegationTokenInformation(now
- + tokenRenewInterval, password, getTrackingIdIfEnabled(identifier)));
+ DelegationTokenInformation tokenInfo = new DelegationTokenInformation(now
+ + tokenRenewInterval, password, getTrackingIdIfEnabled(identifier));
+ try {
+ storeToken(identifier, tokenInfo);
+ } catch (IOException ioe) {
+ LOG.error("Could not store token !!", ioe);
+ }
return password;
}
+
+
/**
* Find the DelegationTokenInformation for the given token id, and verify that
* if the token is expired. Note that this method should be called with
@@ -297,7 +388,7 @@ extends AbstractDelegationTokenIdentifier>
protected DelegationTokenInformation checkToken(TokenIdent identifier)
throws InvalidToken {
assert Thread.holdsLock(this);
- DelegationTokenInformation info = currentTokens.get(identifier);
+ DelegationTokenInformation info = getTokenInfo(identifier);
if (info == null) {
throw new InvalidToken("token (" + identifier.toString()
+ ") can't be found in cache");
@@ -322,7 +413,7 @@ extends AbstractDelegationTokenIdentifier>
}
public synchronized String getTokenTrackingId(TokenIdent identifier) {
- DelegationTokenInformation info = currentTokens.get(identifier);
+ DelegationTokenInformation info = getTokenInfo(identifier);
if (info == null) {
return null;
}
@@ -373,7 +464,7 @@ extends AbstractDelegationTokenIdentifier>
throw new AccessControlException(renewer +
" tries to renew a token with renewer " + id.getRenewer());
}
- DelegationKey key = allKeys.get(id.getMasterKeyId());
+ DelegationKey key = getDelegationKey(id.getMasterKeyId());
if (key == null) {
throw new InvalidToken("Unable to find master key for keyId="
+ id.getMasterKeyId()
@@ -390,11 +481,10 @@ extends AbstractDelegationTokenIdentifier>
DelegationTokenInformation info = new DelegationTokenInformation(renewTime,
password, trackingId);
- if (currentTokens.get(id) == null) {
+ if (getTokenInfo(id) == null) {
throw new InvalidToken("Renewal request for unknown token");
}
- currentTokens.put(id, info);
- updateStoredToken(id, renewTime);
+ updateToken(id, info);
return renewTime;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
new file mode 100644
index 0000000..23c7144
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
@@ -0,0 +1,727 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.token.delegation;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.security.auth.login.AppConfigurationEntry;
+
+import org.apache.curator.ensemble.fixed.FixedEnsembleProvider;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.CuratorFrameworkFactory.Builder;
+import org.apache.curator.framework.api.ACLProvider;
+import org.apache.curator.framework.imps.DefaultACLProvider;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache.StartMode;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.curator.framework.recipes.shared.SharedCount;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs.Perms;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+/**
+ * An implementation of {@link AbstractDelegationTokenSecretManager} that
+ * persists TokenIdentifiers and DelegationKeys in Zookeeper. This class can
+ * be used by HA (Highly available) services that consists of multiple nodes.
+ * This class ensures that Identifiers and Keys are replicated to all nodes of
+ * the service.
+ */
+@InterfaceAudience.Private
+public abstract class ZKDelegationTokenSecretManager<TokenIdent extends AbstractDelegationTokenIdentifier>
+ extends AbstractDelegationTokenSecretManager<TokenIdent> {
+
+ private static final String ZK_CONF_PREFIX = "zk-dt-secret-manager.";
+ public static final String ZK_DTSM_ZK_NUM_RETRIES = ZK_CONF_PREFIX
+ + "zkNumRetries";
+ public static final String ZK_DTSM_ZK_SESSION_TIMEOUT = ZK_CONF_PREFIX
+ + "zkSessionTimeout";
+ public static final String ZK_DTSM_ZK_CONNECTION_TIMEOUT = ZK_CONF_PREFIX
+ + "zkConnectionTimeout";
+ public static final String ZK_DTSM_ZNODE_WORKING_PATH = ZK_CONF_PREFIX
+ + "znodeWorkingPath";
+ public static final String ZK_DTSM_ZK_AUTH_TYPE = ZK_CONF_PREFIX
+ + "zkAuthType";
+ public static final String ZK_DTSM_ZK_CONNECTION_STRING = ZK_CONF_PREFIX
+ + "zkConnectionString";
+ public static final String ZK_DTSM_ZK_KERBEROS_KEYTAB = ZK_CONF_PREFIX
+ + "kerberos.keytab";
+ public static final String ZK_DTSM_ZK_KERBEROS_PRINCIPAL = ZK_CONF_PREFIX
+ + "kerberos.principal";
+
+ public static final int ZK_DTSM_ZK_NUM_RETRIES_DEFAULT = 3;
+ public static final int ZK_DTSM_ZK_SESSION_TIMEOUT_DEFAULT = 10000;
+ public static final int ZK_DTSM_ZK_CONNECTION_TIMEOUT_DEFAULT = 10000;
+ public static final String ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT = "zkdtsm";
+
+ private static Logger LOG = LoggerFactory
+ .getLogger(ZKDelegationTokenSecretManager.class);
+
+ private static final String JAAS_LOGIN_ENTRY_NAME =
+ "ZKDelegationTokenSecretManagerClient";
+
+ private static final String ZK_DTSM_NAMESPACE = "ZKDTSMRoot";
+ private static final String ZK_DTSM_SEQNUM_ROOT = "ZKDTSMSeqNumRoot";
+ private static final String ZK_DTSM_TOKENS_ROOT = "ZKDTSMTokensRoot";
+ private static final String ZK_DTSM_MASTER_KEY_ROOT = "ZKDTSMMasterKeyRoot";
+
+ private static final String DELEGATION_KEY_PREFIX = "DK_";
+ private static final String DELEGATION_TOKEN_PREFIX = "DT_";
+
+ private static final ThreadLocal<CuratorFramework> CURATOR_TL =
+ new ThreadLocal<CuratorFramework>();
+
+ public static void setCurator(CuratorFramework curator) {
+ CURATOR_TL.set(curator);
+ }
+
+ private final boolean isExternalClient;
+ private final CuratorFramework zkClient;
+ private SharedCount seqCounter;
+ private PathChildrenCache keyCache;
+ private PathChildrenCache tokenCache;
+ private ExecutorService listenerThreadPool;
+
+ public ZKDelegationTokenSecretManager(Configuration conf) {
+ super(conf.getLong(DelegationTokenManager.UPDATE_INTERVAL,
+ DelegationTokenManager.UPDATE_INTERVAL_DEFAULT) * 1000,
+ conf.getLong(DelegationTokenManager.MAX_LIFETIME,
+ DelegationTokenManager.MAX_LIFETIME_DEFAULT) * 1000,
+ conf.getLong(DelegationTokenManager.RENEW_INTERVAL,
+ DelegationTokenManager.RENEW_INTERVAL_DEFAULT * 1000),
+ conf.getLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL,
+ DelegationTokenManager.REMOVAL_SCAN_INTERVAL_DEFAULT) * 1000);
+ if (CURATOR_TL.get() != null) {
+ zkClient = CURATOR_TL.get();
+ isExternalClient = true;
+ } else {
+ String connString = conf.get(ZK_DTSM_ZK_CONNECTION_STRING);
+ Preconditions.checkNotNull(connString,
+ "Zookeeper connection string cannot be null");
+ String authType = conf.get(ZK_DTSM_ZK_AUTH_TYPE);
+
+ // AuthType has to be explicitly set to 'none' or 'sasl'
+ Preconditions.checkNotNull(authType, "Zookeeper authType cannot be null !!");
+ Preconditions.checkArgument(
+ authType.equals("sasl") || authType.equals("none"),
+ "Zookeeper authType must be one of [none, sasl]");
+
+ Builder builder = null;
+ try {
+ ACLProvider aclProvider = null;
+ if (authType.equals("sasl")) {
+ LOG.info("Connecting to ZooKeeper with SASL/Kerberos"
+ + "and using 'sasl' ACLs");
+ String principal = setJaasConfiguration(conf);
+ System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
+ JAAS_LOGIN_ENTRY_NAME);
+ System.setProperty("zookeeper.authProvider.1",
+ "org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
+ aclProvider = new SASLOwnerACLProvider(principal);
+ } else { // "none"
+ LOG.info("Connecting to ZooKeeper without authentication");
+ aclProvider = new DefaultACLProvider(); // open to everyone
+ }
+ int sessionT =
+ conf.getInt(ZK_DTSM_ZK_SESSION_TIMEOUT,
+ ZK_DTSM_ZK_SESSION_TIMEOUT_DEFAULT);
+ int numRetries =
+ conf.getInt(ZK_DTSM_ZK_NUM_RETRIES, ZK_DTSM_ZK_NUM_RETRIES_DEFAULT);
+ builder =
+ CuratorFrameworkFactory
+ .builder()
+ .aclProvider(aclProvider)
+ .namespace(
+ conf.get(ZK_DTSM_ZNODE_WORKING_PATH,
+ ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT)
+ + "/"
+ + ZK_DTSM_NAMESPACE
+ )
+ .sessionTimeoutMs(sessionT)
+ .connectionTimeoutMs(
+ conf.getInt(ZK_DTSM_ZK_CONNECTION_TIMEOUT,
+ ZK_DTSM_ZK_CONNECTION_TIMEOUT_DEFAULT)
+ )
+ .retryPolicy(
+ new RetryNTimes(numRetries, sessionT / numRetries));
+ } catch (Exception ex) {
+ throw new RuntimeException("Could not Load ZK acls or auth");
+ }
+ zkClient = builder.ensembleProvider(new FixedEnsembleProvider(connString))
+ .build();
+ isExternalClient = false;
+ }
+ listenerThreadPool = Executors.newFixedThreadPool(2);
+ }
+
+ private String setJaasConfiguration(Configuration config) throws Exception {
+ String keytabFile =
+ config.get(ZK_DTSM_ZK_KERBEROS_KEYTAB, "").trim();
+ if (keytabFile == null || keytabFile.length() == 0) {
+ throw new IllegalArgumentException(ZK_DTSM_ZK_KERBEROS_KEYTAB
+ + " must be specified");
+ }
+ String principal =
+ config.get(ZK_DTSM_ZK_KERBEROS_PRINCIPAL, "").trim();
+ if (principal == null || principal.length() == 0) {
+ throw new IllegalArgumentException(ZK_DTSM_ZK_KERBEROS_PRINCIPAL
+ + " must be specified");
+ }
+
+ JaasConfiguration jConf =
+ new JaasConfiguration(JAAS_LOGIN_ENTRY_NAME, principal, keytabFile);
+ javax.security.auth.login.Configuration.setConfiguration(jConf);
+ return principal.split("[/@]")[0];
+ }
+
+ /**
+ * Creates a programmatic version of a jaas.conf file. This can be used
+ * instead of writing a jaas.conf file and setting the system property,
+ * "java.security.auth.login.config", to point to that file. It is meant to be
+ * used for connecting to ZooKeeper.
+ */
+ @InterfaceAudience.Private
+ public static class JaasConfiguration extends
+ javax.security.auth.login.Configuration {
+
+ private static AppConfigurationEntry[] entry;
+ private String entryName;
+
+ /**
+ * Add an entry to the jaas configuration with the passed in name,
+ * principal, and keytab. The other necessary options will be set for you.
+ *
+ * @param entryName
+ * The name of the entry (e.g. "Client")
+ * @param principal
+ * The principal of the user
+ * @param keytab
+ * The location of the keytab
+ */
+ public JaasConfiguration(String entryName, String principal, String keytab) {
+ this.entryName = entryName;
+ Map<String, String> options = new HashMap<String, String>();
+ options.put("keyTab", keytab);
+ options.put("principal", principal);
+ options.put("useKeyTab", "true");
+ options.put("storeKey", "true");
+ options.put("useTicketCache", "false");
+ options.put("refreshKrb5Config", "true");
+ String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
+ if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
+ options.put("debug", "true");
+ }
+ entry = new AppConfigurationEntry[] {
+ new AppConfigurationEntry(getKrb5LoginModuleName(),
+ AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
+ options) };
+ }
+
+ @Override
+ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
+ return (entryName.equals(name)) ? entry : null;
+ }
+
+ private String getKrb5LoginModuleName() {
+ String krb5LoginModuleName;
+ if (System.getProperty("java.vendor").contains("IBM")) {
+ krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
+ } else {
+ krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
+ }
+ return krb5LoginModuleName;
+ }
+ }
+
+ @Override
+ public void startThreads() throws IOException {
+ if (!isExternalClient) {
+ try {
+ zkClient.start();
+ } catch (Exception e) {
+ throw new IOException("Could not start Curator Framework", e);
+ }
+ }
+ try {
+ seqCounter = new SharedCount(zkClient, ZK_DTSM_SEQNUM_ROOT, 0);
+ if (seqCounter != null) {
+ seqCounter.start();
+ }
+ } catch (Exception e) {
+ throw new IOException("Could not start Sequence Counter", e);
+ }
+ try {
+ createPersistentNode(ZK_DTSM_MASTER_KEY_ROOT);
+ createPersistentNode(ZK_DTSM_TOKENS_ROOT);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not create ZK paths");
+ }
+ try {
+ keyCache = new PathChildrenCache(zkClient, ZK_DTSM_MASTER_KEY_ROOT, true);
+ if (keyCache != null) {
+ keyCache.start(StartMode.POST_INITIALIZED_EVENT);
+ keyCache.getListenable().addListener(new PathChildrenCacheListener() {
+ @Override
+ public void childEvent(CuratorFramework client,
+ PathChildrenCacheEvent event)
+ throws Exception {
+ switch (event.getType()) {
+ case CHILD_ADDED:
+ processKeyAddOrUpdate(event.getData().getData());
+ break;
+ case CHILD_UPDATED:
+ processKeyAddOrUpdate(event.getData().getData());
+ break;
+ case CHILD_REMOVED:
+ processKeyRemoved(event.getData().getPath());
+ break;
+ default:
+ break;
+ }
+ }
+ }, listenerThreadPool);
+ }
+ } catch (Exception e) {
+ throw new IOException("Could not start PathChildrenCache for keys", e);
+ }
+ try {
+ tokenCache = new PathChildrenCache(zkClient, ZK_DTSM_TOKENS_ROOT, true);
+ if (tokenCache != null) {
+ tokenCache.start(StartMode.POST_INITIALIZED_EVENT);
+ tokenCache.getListenable().addListener(new PathChildrenCacheListener() {
+
+ @Override
+ public void childEvent(CuratorFramework client,
+ PathChildrenCacheEvent event) throws Exception {
+ switch (event.getType()) {
+ case CHILD_ADDED:
+ processTokenAddOrUpdate(event.getData().getData());
+ break;
+ case CHILD_UPDATED:
+ processTokenAddOrUpdate(event.getData().getData());
+ break;
+ case CHILD_REMOVED:
+ processTokenRemoved(event.getData().getData());
+ break;
+ default:
+ break;
+ }
+ }
+ }, listenerThreadPool);
+ }
+ } catch (Exception e) {
+ throw new IOException("Could not start PathChildrenCache for tokens", e);
+ }
+ super.startThreads();
+ }
+
+ private void processKeyAddOrUpdate(byte[] data) throws IOException {
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+ allKeys.put(key.getKeyId(), key);
+ }
+
+ private void processKeyRemoved(String path) {
+ int i = path.lastIndexOf('/');
+ if (i > 0) {
+ String tokSeg = path.substring(i + 1);
+ int j = tokSeg.indexOf('_');
+ if (j > 0) {
+ int keyId = Integer.parseInt(tokSeg.substring(j + 1));
+ allKeys.remove(keyId);
+ }
+ }
+ }
+
+ private void processTokenAddOrUpdate(byte[] data) throws IOException {
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ TokenIdent ident = createIdentifier();
+ ident.readFields(din);
+ long renewDate = din.readLong();
+ int pwdLen = din.readInt();
+ byte[] password = new byte[pwdLen];
+ int numRead = din.read(password, 0, pwdLen);
+ if (numRead > -1) {
+ DelegationTokenInformation tokenInfo =
+ new DelegationTokenInformation(renewDate, password);
+ currentTokens.put(ident, tokenInfo);
+ }
+ }
+
+ private void processTokenRemoved(byte[] data) throws IOException {
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ TokenIdent ident = createIdentifier();
+ ident.readFields(din);
+ currentTokens.remove(ident);
+ }
+
+ @Override
+ public void stopThreads() {
+ try {
+ if (!isExternalClient && (zkClient != null)) {
+ zkClient.close();
+ }
+ if (seqCounter != null) {
+ seqCounter.close();
+ }
+ if (keyCache != null) {
+ keyCache.close();
+ }
+ if (tokenCache != null) {
+ tokenCache.close();
+ }
+ } catch (Exception e) {
+ LOG.error("Could not stop Curator Framework", e);
+ // Ignore
+ }
+ super.stopThreads();
+ }
+
+ private void createPersistentNode(String nodePath) throws Exception {
+ try {
+ zkClient.create().withMode(CreateMode.PERSISTENT).forPath(nodePath);
+ } catch (KeeperException.NodeExistsException ne) {
+ LOG.debug(nodePath + " znode already exists !!");
+ } catch (Exception e) {
+ throw new IOException(nodePath + " znode could not be created !!", e);
+ }
+ }
+
+ @Override
+ protected int getDelegationTokenSeqNum() {
+ return seqCounter.getCount();
+ }
+
+ @Override
+ protected int incrementDelegationTokenSeqNum() {
+ try {
+ while (!seqCounter.trySetCount(seqCounter.getCount() + 1)) {
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Could not increment shared counter !!", e);
+ }
+ return seqCounter.getCount();
+ }
+
+ @Override
+ protected void setDelegationTokenSeqNum(int seqNum) {
+ delegationTokenSequenceNumber = seqNum;
+ }
+
+ @Override
+ protected DelegationKey getDelegationKey(int keyId) {
+ // First check if its I already have this key
+ DelegationKey key = allKeys.get(keyId);
+ // Then query ZK
+ if (key == null) {
+ try {
+ key = getKeyFromZK(keyId);
+ if (key != null) {
+ allKeys.put(keyId, key);
+ }
+ } catch (IOException e) {
+ LOG.error("Error retrieving key [" + keyId + "] from ZK", e);
+ }
+ }
+ return key;
+ }
+
+ private DelegationKey getKeyFromZK(int keyId) throws IOException {
+ String nodePath =
+ getNodePath(ZK_DTSM_MASTER_KEY_ROOT, DELEGATION_KEY_PREFIX + keyId);
+ try {
+ byte[] data = zkClient.getData().forPath(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+ return key;
+ } catch (KeeperException.NoNodeException e) {
+ LOG.error("No node in path [" + nodePath + "]");
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ return null;
+ }
+
+ @Override
+ protected DelegationTokenInformation getTokenInfo(TokenIdent ident) {
+ // First check if I have this..
+ DelegationTokenInformation tokenInfo = currentTokens.get(ident);
+ // Then query ZK
+ if (tokenInfo == null) {
+ try {
+ tokenInfo = getTokenInfoFromZK(ident);
+ if (tokenInfo != null) {
+ currentTokens.put(ident, tokenInfo);
+ }
+ } catch (IOException e) {
+ LOG.error("Error retrieving tokenInfo [" + ident.getSequenceNumber()
+ + "] from ZK", e);
+ }
+ }
+ return tokenInfo;
+ }
+
+ private DelegationTokenInformation getTokenInfoFromZK(TokenIdent ident)
+ throws IOException {
+ String nodePath =
+ getNodePath(ZK_DTSM_TOKENS_ROOT,
+ DELEGATION_TOKEN_PREFIX + ident.getSequenceNumber());
+ try {
+ byte[] data = zkClient.getData().forPath(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ createIdentifier().readFields(din);
+ long renewDate = din.readLong();
+ int pwdLen = din.readInt();
+ byte[] password = new byte[pwdLen];
+ int numRead = din.read(password, 0, pwdLen);
+ if (numRead > -1) {
+ DelegationTokenInformation tokenInfo =
+ new DelegationTokenInformation(renewDate, password);
+ return tokenInfo;
+ }
+ } catch (KeeperException.NoNodeException e) {
+ LOG.error("No node in path [" + nodePath + "]");
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ return null;
+ }
+
+ @Override
+ protected void storeDelegationKey(DelegationKey key) throws IOException {
+ allKeys.put(key.getKeyId(), key);
+ addOrUpdateDelegationKey(key, false);
+ }
+
+ @Override
+ protected void updateDelegationKey(DelegationKey key) throws IOException {
+ allKeys.put(key.getKeyId(), key);
+ addOrUpdateDelegationKey(key, true);
+ }
+
+ private void addOrUpdateDelegationKey(DelegationKey key, boolean isUpdate)
+ throws IOException {
+ String nodeCreatePath =
+ getNodePath(ZK_DTSM_MASTER_KEY_ROOT,
+ DELEGATION_KEY_PREFIX + key.getKeyId());
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing ZKDTSMDelegationKey_" + key.getKeyId());
+ }
+ key.write(fsOut);
+ try {
+ if (zkClient.checkExists().forPath(nodeCreatePath) != null) {
+ zkClient.setData().forPath(nodeCreatePath, os.toByteArray())
+ .setVersion(-1);
+ if (!isUpdate) {
+ LOG.debug("Key with path [" + nodeCreatePath
+ + "] already exists.. Updating !!");
+ }
+ } else {
+ zkClient.create().withMode(CreateMode.PERSISTENT)
+ .forPath(nodeCreatePath, os.toByteArray());
+ if (isUpdate) {
+ LOG.debug("Updating non existent Key path [" + nodeCreatePath
+ + "].. Adding new !!");
+ }
+ }
+ } catch (KeeperException.NodeExistsException ne) {
+ LOG.debug(nodeCreatePath + " znode already exists !!");
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ } finally {
+ os.close();
+ }
+ }
+
+ @Override
+ protected void removeStoredMasterKey(DelegationKey key) {
+ String nodeRemovePath =
+ getNodePath(ZK_DTSM_MASTER_KEY_ROOT,
+ DELEGATION_KEY_PREFIX + key.getKeyId());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removing ZKDTSMDelegationKey_" + key.getKeyId());
+ }
+ try {
+ if (zkClient.checkExists().forPath(nodeRemovePath) != null) {
+ zkClient.delete().forPath(nodeRemovePath);
+ } else {
+ LOG.debug("Attempted to delete a non-existing znode " + nodeRemovePath);
+ }
+ } catch (Exception e) {
+ LOG.debug(nodeRemovePath + " znode could not be removed!!");
+ }
+ }
+
+ @Override
+ protected void storeToken(TokenIdent ident,
+ DelegationTokenInformation tokenInfo) throws IOException {
+ currentTokens.put(ident, tokenInfo);
+ try {
+ addOrUpdateToken(ident, tokenInfo, false);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ protected void updateToken(TokenIdent ident,
+ DelegationTokenInformation tokenInfo) throws IOException {
+ currentTokens.put(ident, tokenInfo);
+ String nodeRemovePath =
+ getNodePath(ZK_DTSM_TOKENS_ROOT, DELEGATION_TOKEN_PREFIX
+ + ident.getSequenceNumber());
+ try {
+ if (zkClient.checkExists().forPath(nodeRemovePath) != null) {
+ addOrUpdateToken(ident, tokenInfo, false);
+ LOG.debug("Attempted to update a non-existing znode " + nodeRemovePath);
+ } else {
+ addOrUpdateToken(ident, tokenInfo, true);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Could not update Stored Token ZKDTSMDelegationToken_"
+ + ident.getSequenceNumber(), e);
+ }
+ }
+
+ @Override
+ protected void removeStoredToken(TokenIdent ident)
+ throws IOException {
+ String nodeRemovePath =
+ getNodePath(ZK_DTSM_TOKENS_ROOT, DELEGATION_TOKEN_PREFIX
+ + ident.getSequenceNumber());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removing ZKDTSMDelegationToken_"
+ + ident.getSequenceNumber());
+ }
+ try {
+ if (zkClient.checkExists().forPath(nodeRemovePath) != null) {
+ LOG.debug("Attempted to remove a non-existing znode " + nodeRemovePath);
+ } else {
+ zkClient.delete().forPath(nodeRemovePath);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Could not remove Stored Token ZKDTSMDelegationToken_"
+ + ident.getSequenceNumber(), e);
+ }
+ }
+
+ private void addOrUpdateToken(TokenIdent ident,
+ DelegationTokenInformation info, boolean isUpdate) throws Exception {
+ String nodeCreatePath =
+ getNodePath(ZK_DTSM_TOKENS_ROOT, DELEGATION_TOKEN_PREFIX
+ + ident.getSequenceNumber());
+ ByteArrayOutputStream tokenOs = new ByteArrayOutputStream();
+ DataOutputStream tokenOut = new DataOutputStream(tokenOs);
+ ByteArrayOutputStream seqOs = new ByteArrayOutputStream();
+
+ try {
+ ident.write(tokenOut);
+ tokenOut.writeLong(info.getRenewDate());
+ tokenOut.writeInt(info.getPassword().length);
+ tokenOut.write(info.getPassword());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug((isUpdate ? "Storing " : "Updating ")
+ + "ZKDTSMDelegationToken_" +
+ ident.getSequenceNumber());
+ }
+ if (isUpdate) {
+ zkClient.setData().forPath(nodeCreatePath, tokenOs.toByteArray())
+ .setVersion(-1);
+ } else {
+ zkClient.create().withMode(CreateMode.PERSISTENT)
+ .forPath(nodeCreatePath, tokenOs.toByteArray());
+ }
+ } finally {
+ seqOs.close();
+ }
+ }
+
+ /**
+ * Simple implementation of an {@link ACLProvider} that simply returns an ACL
+ * that gives all permissions only to a single principal.
+ */
+ private static class SASLOwnerACLProvider implements ACLProvider {
+
+ private final List<ACL> saslACL;
+
+ private SASLOwnerACLProvider(String principal) {
+ this.saslACL = Collections.singletonList(
+ new ACL(Perms.ALL, new Id("sasl", principal)));
+ }
+
+ @Override
+ public List<ACL> getDefaultAcl() {
+ return saslACL;
+ }
+
+ @Override
+ public List<ACL> getAclForPath(String path) {
+ return saslACL;
+ }
+ }
+
+ @VisibleForTesting
+ @Private
+ @Unstable
+ static String getNodePath(String root, String nodeName) {
+ return (root + "/" + nodeName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
index 64a5622..aa9ec99 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.security.token.delegation.web;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -28,9 +29,11 @@ import org.apache.hadoop.security.authentication.server.AuthenticationHandler;
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.security.authentication.util.ZKSignerSecretProvider;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
+import org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager;
import org.apache.hadoop.util.HttpExceptionUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
@@ -153,7 +156,14 @@ public class DelegationTokenAuthenticationFilter
@Override
public void init(FilterConfig filterConfig) throws ServletException {
+ // A single CuratorFramework should be used for a ZK cluster.
+ // If the ZKSignerSecretProvider has already created it, it has to
+ // be set here... to be used by the ZKDelegationTokenSecretManager
+ ZKDelegationTokenSecretManager.setCurator((CuratorFramework)
+ filterConfig.getServletContext().getAttribute(ZKSignerSecretProvider.
+ ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE));
super.init(filterConfig);
+ ZKDelegationTokenSecretManager.setCurator(null);
AuthenticationHandler handler = getAuthenticationHandler();
AbstractDelegationTokenSecretManager dtSecretManager =
(AbstractDelegationTokenSecretManager) filterConfig.getServletContext().
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index f41f892..5a31d6d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -78,19 +78,6 @@ public abstract class DelegationTokenAuthenticationHandler
public static final String TOKEN_KIND = PREFIX + "token-kind";
- public static final String UPDATE_INTERVAL = PREFIX + "update-interval.sec";
- public static final long UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60;
-
- public static final String MAX_LIFETIME = PREFIX + "max-lifetime.sec";
- public static final long MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60;
-
- public static final String RENEW_INTERVAL = PREFIX + "renew-interval.sec";
- public static final long RENEW_INTERVAL_DEFAULT = 24 * 60 * 60;
-
- public static final String REMOVAL_SCAN_INTERVAL = PREFIX +
- "removal-scan-interval.sec";
- public static final long REMOVAL_SCAN_INTERVAL_DEFAULT = 60 * 60;
-
private static final Set<String> DELEGATION_TOKEN_OPS = new HashSet<String>();
static final String DELEGATION_TOKEN_UGI_ATTRIBUTE =
@@ -142,7 +129,6 @@ public abstract class DelegationTokenAuthenticationHandler
@VisibleForTesting
@SuppressWarnings("unchecked")
public void initTokenManager(Properties config) {
- String configPrefix = authHandler.getType() + ".";
Configuration conf = new Configuration(false);
for (Map.Entry entry : config.entrySet()) {
conf.set((String) entry.getKey(), (String) entry.getValue());
@@ -153,17 +139,7 @@ public abstract class DelegationTokenAuthenticationHandler
"The configuration does not define the token kind");
}
tokenKind = tokenKind.trim();
- long updateInterval = conf.getLong(configPrefix + UPDATE_INTERVAL,
- UPDATE_INTERVAL_DEFAULT);
- long maxLifeTime = conf.getLong(configPrefix + MAX_LIFETIME,
- MAX_LIFETIME_DEFAULT);
- long renewInterval = conf.getLong(configPrefix + RENEW_INTERVAL,
- RENEW_INTERVAL_DEFAULT);
- long removalScanInterval = conf.getLong(
- configPrefix + REMOVAL_SCAN_INTERVAL, REMOVAL_SCAN_INTERVAL_DEFAULT);
- tokenManager = new DelegationTokenManager(new Text(tokenKind),
- updateInterval * 1000, maxLifeTime * 1000, renewInterval * 1000,
- removalScanInterval * 1000);
+ tokenManager = new DelegationTokenManager(conf, new Text(tokenKind));
tokenManager.init();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
index 2e6b46e..dbde0a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
@@ -17,16 +17,20 @@
*/
package org.apache.hadoop.security.token.delegation.web;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
+import org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
+import com.google.common.annotations.VisibleForTesting;
/**
* Delegation Token Manager used by the
@@ -35,20 +39,36 @@ import java.io.IOException;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-class DelegationTokenManager {
+public class DelegationTokenManager {
+
+ public static final String ENABLE_ZK_KEY = "zk-dt-secret-manager.enable";
+
+ public static final String PREFIX = "delegation-token.";
+
+ public static final String UPDATE_INTERVAL = PREFIX + "update-interval.sec";
+ public static final long UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60;
+
+ public static final String MAX_LIFETIME = PREFIX + "max-lifetime.sec";
+ public static final long MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60;
+
+ public static final String RENEW_INTERVAL = PREFIX + "renew-interval.sec";
+ public static final long RENEW_INTERVAL_DEFAULT = 24 * 60 * 60;
+
+ public static final String REMOVAL_SCAN_INTERVAL = PREFIX +
+ "removal-scan-interval.sec";
+ public static final long REMOVAL_SCAN_INTERVAL_DEFAULT = 60 * 60;
private static class DelegationTokenSecretManager
extends AbstractDelegationTokenSecretManager<DelegationTokenIdentifier> {
private Text tokenKind;
- public DelegationTokenSecretManager(Text tokenKind,
- long delegationKeyUpdateInterval,
- long delegationTokenMaxLifetime,
- long delegationTokenRenewInterval,
- long delegationTokenRemoverScanInterval) {
- super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
- delegationTokenRenewInterval, delegationTokenRemoverScanInterval);
+ public DelegationTokenSecretManager(Configuration conf, Text tokenKind) {
+ super(conf.getLong(UPDATE_INTERVAL, UPDATE_INTERVAL_DEFAULT) * 1000,
+ conf.getLong(MAX_LIFETIME, MAX_LIFETIME_DEFAULT) * 1000,
+ conf.getLong(RENEW_INTERVAL, RENEW_INTERVAL_DEFAULT) * 1000,
+ conf.getLong(REMOVAL_SCAN_INTERVAL,
+ REMOVAL_SCAN_INTERVAL_DEFAULT * 1000));
this.tokenKind = tokenKind;
}
@@ -56,21 +76,34 @@ class DelegationTokenManager {
public DelegationTokenIdentifier createIdentifier() {
return new DelegationTokenIdentifier(tokenKind);
}
+ }
+
+ private static class ZKSecretManager
+ extends ZKDelegationTokenSecretManager<DelegationTokenIdentifier> {
+
+ private Text tokenKind;
+
+ public ZKSecretManager(Configuration conf, Text tokenKind) {
+ super(conf);
+ this.tokenKind = tokenKind;
+ }
+ @Override
+ public DelegationTokenIdentifier createIdentifier() {
+ return new DelegationTokenIdentifier(tokenKind);
+ }
}
private AbstractDelegationTokenSecretManager secretManager = null;
private boolean managedSecretManager;
private Text tokenKind;
- public DelegationTokenManager(Text tokenKind,
- long delegationKeyUpdateInterval,
- long delegationTokenMaxLifetime,
- long delegationTokenRenewInterval,
- long delegationTokenRemoverScanInterval) {
- this.secretManager = new DelegationTokenSecretManager(tokenKind,
- delegationKeyUpdateInterval, delegationTokenMaxLifetime,
- delegationTokenRenewInterval, delegationTokenRemoverScanInterval);
+ public DelegationTokenManager(Configuration conf, Text tokenKind) {
+ if (conf.getBoolean(ENABLE_ZK_KEY, false)) {
+ this.secretManager = new ZKSecretManager(conf, tokenKind);
+ } else {
+ this.secretManager = new DelegationTokenSecretManager(conf, tokenKind);
+ }
this.tokenKind = tokenKind;
managedSecretManager = true;
}
@@ -150,4 +183,9 @@ class DelegationTokenManager {
return id.getUser();
}
+ @VisibleForTesting
+ @SuppressWarnings("rawtypes")
+ public AbstractDelegationTokenSecretManager getDelegationTokenSecretManager() {
+ return secretManager;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
index 4f83a57..239b841 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
@@ -121,7 +121,7 @@ public class TestDelegationToken {
@Override
protected void storeNewToken(TestDelegationTokenIdentifier ident,
- long renewDate) {
+ long renewDate) throws IOException {
super.storeNewToken(ident, renewDate);
isStoreNewTokenCalled = true;
}
@@ -135,7 +135,7 @@ public class TestDelegationToken {
@Override
protected void updateStoredToken(TestDelegationTokenIdentifier ident,
- long renewDate) {
+ long renewDate) throws IOException {
super.updateStoredToken(ident, renewDate);
isUpdateStoredTokenCalled = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
new file mode 100644
index 0000000..076c87a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.token.delegation;
+
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestZKDelegationTokenSecretManager {
+
+ private static final long DAY_IN_SECS = 86400;
+
+ @Test
+ public void testZKDelTokSecretManager() throws Exception {
+ TestingServer zkServer = new TestingServer();
+ DelegationTokenManager tm1, tm2 = null;
+ zkServer.start();
+ try {
+ String connectString = zkServer.getConnectString();
+ Configuration conf = new Configuration();
+ conf.setBoolean(DelegationTokenManager.ENABLE_ZK_KEY, true);
+ conf.set(ZKDelegationTokenSecretManager.ZK_DTSM_ZK_CONNECTION_STRING, connectString);
+ conf.set(ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH, "testPath");
+ conf.set(ZKDelegationTokenSecretManager.ZK_DTSM_ZK_AUTH_TYPE, "none");
+ conf.setLong(DelegationTokenManager.UPDATE_INTERVAL, DAY_IN_SECS);
+ conf.setLong(DelegationTokenManager.MAX_LIFETIME, DAY_IN_SECS);
+ conf.setLong(DelegationTokenManager.RENEW_INTERVAL, DAY_IN_SECS);
+ conf.setLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, DAY_IN_SECS);
+ tm1 = new DelegationTokenManager(conf, new Text("foo"));
+ tm1.init();
+ tm2 = new DelegationTokenManager(conf, new Text("foo"));
+ tm2.init();
+
+ Token<DelegationTokenIdentifier> token =
+ tm1.createToken(UserGroupInformation.getCurrentUser(), "foo");
+ Assert.assertNotNull(token);
+ tm2.verifyToken(token);
+
+ token = tm2.createToken(UserGroupInformation.getCurrentUser(), "bar");
+ Assert.assertNotNull(token);
+ tm1.verifyToken(token);
+ } finally {
+ zkServer.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenManager.java
index 4a0e834..496b762 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenManager.java
@@ -17,27 +17,28 @@
*/
package org.apache.hadoop.security.token.delegation.web;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.StringUtils;
import org.junit.Assert;
import org.junit.Test;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.util.Arrays;
-
public class TestDelegationTokenManager {
private static final long DAY_IN_SECS = 86400;
@Test
public void testDTManager() throws Exception {
- DelegationTokenManager tm = new DelegationTokenManager(new Text("foo"),
- DAY_IN_SECS, DAY_IN_SECS, DAY_IN_SECS, DAY_IN_SECS);
+ Configuration conf = new Configuration(false);
+ conf.setLong(DelegationTokenManager.UPDATE_INTERVAL, DAY_IN_SECS);
+ conf.setLong(DelegationTokenManager.MAX_LIFETIME, DAY_IN_SECS);
+ conf.setLong(DelegationTokenManager.RENEW_INTERVAL, DAY_IN_SECS);
+ conf.setLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, DAY_IN_SECS);
+ DelegationTokenManager tm =
+ new DelegationTokenManager(conf, new Text("foo"));
tm.init();
Token<DelegationTokenIdentifier> token =
tm.createToken(UserGroupInformation.getCurrentUser(), "foo");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a121faf..3bc1a57 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -866,6 +866,16 @@
<dependency>
<groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>2.6.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ <version>2.6.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>2.6.0</version>
</dependency>
[05/50] [abbrv] git commit: HDFS-6727. Refresh data volumes on
DataNode based on configuration changes (Lei Xu via Colin Patrick McCabe)
Posted by cu...@apache.org.
HDFS-6727. Refresh data volumes on DataNode based on configuration changes (Lei Xu via Colin Patrick McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe38d2e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe38d2e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe38d2e9
Branch: refs/heads/YARN-1051
Commit: fe38d2e9b5ac7e13f97cd2d3d2a984ab6bbaaf77
Parents: 5d01a68
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Sep 18 16:46:01 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Thu Sep 18 16:52:46 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/common/HdfsServerConstants.java | 7 +-
.../hadoop/hdfs/server/common/Storage.java | 10 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 161 ++++++-
.../server/datanode/fsdataset/FsDatasetSpi.java | 4 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 141 +++++--
.../server/datanode/SimulatedFSDataset.java | 3 +-
.../datanode/TestDataNodeHotSwapVolumes.java | 423 +++++++++++++++++++
.../fsdataset/impl/TestFsDatasetImpl.java | 12 +-
9 files changed, 725 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe38d2e9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fc22282..c99207b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -559,6 +559,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7003. Add NFS Gateway support for reading and writing to
encryption zones. (clamb via wang)
+ HDFS-6727. Refresh data volumes on DataNode based on configuration changes
+ (Lei Xu via cmccabe)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe38d2e9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index 106f489..767c1b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -94,7 +94,12 @@ public final class HdfsServerConstants {
NONINTERACTIVE("-nonInteractive"),
RENAMERESERVED("-renameReserved"),
METADATAVERSION("-metadataVersion"),
- UPGRADEONLY("-upgradeOnly");
+ UPGRADEONLY("-upgradeOnly"),
+ // The -hotswap constant should not be used as a startup option, it is
+ // only used for StorageDirectory.analyzeStorage() in hot swap drive scenario.
+ // TODO refactor StorageDirectory.analyzeStorage() so that we can do away with
+ // this in StartupOption.
+ HOTSWAP("-hotswap");
private static final Pattern ENUM_WITH_ROLLING_UPGRADE_OPTION = Pattern.compile(
"(\\w+)\\((\\w+)\\)");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe38d2e9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
index 1d68727..7933fed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
@@ -464,17 +464,20 @@ public abstract class Storage extends StorageInfo {
public StorageState analyzeStorage(StartupOption startOpt, Storage storage)
throws IOException {
assert root != null : "root is null";
+ boolean hadMkdirs = false;
String rootPath = root.getCanonicalPath();
try { // check that storage exists
if (!root.exists()) {
// storage directory does not exist
- if (startOpt != StartupOption.FORMAT) {
+ if (startOpt != StartupOption.FORMAT &&
+ startOpt != StartupOption.HOTSWAP) {
LOG.warn("Storage directory " + rootPath + " does not exist");
return StorageState.NON_EXISTENT;
}
LOG.info(rootPath + " does not exist. Creating ...");
if (!root.mkdirs())
throw new IOException("Cannot create directory " + rootPath);
+ hadMkdirs = true;
}
// or is inaccessible
if (!root.isDirectory()) {
@@ -492,7 +495,10 @@ public abstract class Storage extends StorageInfo {
this.lock(); // lock storage if it exists
- if (startOpt == HdfsServerConstants.StartupOption.FORMAT)
+ // If startOpt is HOTSWAP, it returns NOT_FORMATTED for empty directory,
+ // while it also checks the layout version.
+ if (startOpt == HdfsServerConstants.StartupOption.FORMAT ||
+ (startOpt == StartupOption.HOTSWAP && hadMkdirs))
return StorageState.NOT_FORMATTED;
if (startOpt != HdfsServerConstants.StartupOption.IMPORT) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe38d2e9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 9b57262..57ff2fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -70,8 +70,10 @@ import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -80,11 +82,13 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.management.ObjectName;
+import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.conf.ReconfigurableBase;
+import org.apache.hadoop.conf.ReconfigurationException;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
@@ -137,6 +141,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -220,7 +225,7 @@ import com.google.protobuf.BlockingService;
*
**********************************************************/
@InterfaceAudience.Private
-public class DataNode extends Configured
+public class DataNode extends ReconfigurableBase
implements InterDatanodeProtocol, ClientDatanodeProtocol,
DataNodeMXBean {
public static final Log LOG = LogFactory.getLog(DataNode.class);
@@ -305,6 +310,7 @@ public class DataNode extends Configured
private JvmPauseMonitor pauseMonitor;
private SecureResources secureResources = null;
+ // dataDirs must be accessed while holding the DataNode lock.
private List<StorageLocation> dataDirs;
private Configuration conf;
private final String confVersion;
@@ -386,6 +392,149 @@ public class DataNode extends Configured
}
}
+ @Override
+ public void reconfigurePropertyImpl(String property, String newVal)
+ throws ReconfigurationException {
+ if (property.equals(DFS_DATANODE_DATA_DIR_KEY)) {
+ try {
+ LOG.info("Reconfiguring " + property + " to " + newVal);
+ this.refreshVolumes(newVal);
+ } catch (Exception e) {
+ throw new ReconfigurationException(property, newVal,
+ getConf().get(property), e);
+ }
+ } else {
+ throw new ReconfigurationException(
+ property, newVal, getConf().get(property));
+ }
+ }
+
+ /**
+ * Get a list of the keys of the re-configurable properties in configuration.
+ */
+ @Override
+ public Collection<String> getReconfigurableProperties() {
+ List<String> reconfigurable =
+ Collections.unmodifiableList(Arrays.asList(DFS_DATANODE_DATA_DIR_KEY));
+ return reconfigurable;
+ }
+
+ /**
+ * Contains the StorageLocations for changed data volumes.
+ */
+ @VisibleForTesting
+ static class ChangedVolumes {
+ List<StorageLocation> newLocations = Lists.newArrayList();
+ List<StorageLocation> deactivateLocations = Lists.newArrayList();
+ }
+
+ /**
+ * Parse the new DFS_DATANODE_DATA_DIR value in the configuration to detect
+ * changed volumes.
+ * @return changed volumes.
+ * @throws IOException if none of the directories are specified in the
+ * configuration.
+ */
+ @VisibleForTesting
+ ChangedVolumes parseChangedVolumes() throws IOException {
+ List<StorageLocation> locations = getStorageLocations(getConf());
+
+ if (locations.isEmpty()) {
+ throw new IOException("No directory is specified.");
+ }
+
+ ChangedVolumes results = new ChangedVolumes();
+ results.newLocations.addAll(locations);
+
+ for (Iterator<Storage.StorageDirectory> it = storage.dirIterator();
+ it.hasNext(); ) {
+ Storage.StorageDirectory dir = it.next();
+ boolean found = false;
+ for (Iterator<StorageLocation> sl = results.newLocations.iterator();
+ sl.hasNext(); ) {
+ if (sl.next().getFile().getCanonicalPath().equals(
+ dir.getRoot().getCanonicalPath())) {
+ sl.remove();
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ results.deactivateLocations.add(
+ StorageLocation.parse(dir.getRoot().toString()));
+ }
+ }
+
+ return results;
+ }
+
+ /**
+ * Attempts to reload data volumes with new configuration.
+ * @param newVolumes a comma separated string that specifies the data volumes.
+ * @throws Exception
+ */
+ private synchronized void refreshVolumes(String newVolumes) throws Exception {
+ Configuration conf = getConf();
+ String oldVolumes = conf.get(DFS_DATANODE_DATA_DIR_KEY);
+ conf.set(DFS_DATANODE_DATA_DIR_KEY, newVolumes);
+ List<StorageLocation> locations = getStorageLocations(conf);
+
+ final int numOldDataDirs = dataDirs.size();
+ dataDirs = locations;
+ ChangedVolumes changedVolumes = parseChangedVolumes();
+
+ try {
+ if (numOldDataDirs + changedVolumes.newLocations.size() -
+ changedVolumes.deactivateLocations.size() <= 0) {
+ throw new IOException("Attempt to remove all volumes.");
+ }
+ if (!changedVolumes.newLocations.isEmpty()) {
+ LOG.info("Adding new volumes: " +
+ Joiner.on(",").join(changedVolumes.newLocations));
+
+ // Add volumes for each Namespace
+ for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) {
+ NamespaceInfo nsInfo = bpos.getNamespaceInfo();
+ LOG.info("Loading volumes for namesapce: " + nsInfo.getNamespaceID());
+ storage.addStorageLocations(
+ this, nsInfo, changedVolumes.newLocations, StartupOption.HOTSWAP);
+ }
+ List<String> bpids = Lists.newArrayList();
+ for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) {
+ bpids.add(bpos.getBlockPoolId());
+ }
+ List<StorageLocation> succeedVolumes =
+ data.addVolumes(changedVolumes.newLocations, bpids);
+
+ if (succeedVolumes.size() < changedVolumes.newLocations.size()) {
+ List<StorageLocation> failedVolumes = Lists.newArrayList();
+ // Clean all failed volumes.
+ for (StorageLocation location : changedVolumes.newLocations) {
+ if (!succeedVolumes.contains(location)) {
+ failedVolumes.add(location);
+ }
+ }
+ storage.removeVolumes(failedVolumes);
+ data.removeVolumes(failedVolumes);
+ }
+ }
+
+ if (!changedVolumes.deactivateLocations.isEmpty()) {
+ LOG.info("Deactivating volumes: " +
+ Joiner.on(",").join(changedVolumes.deactivateLocations));
+
+ data.removeVolumes(changedVolumes.deactivateLocations);
+ storage.removeVolumes(changedVolumes.deactivateLocations);
+ }
+ } catch (IOException e) {
+ LOG.warn("There is IOException when refreshing volumes! "
+ + "Recover configurations: " + DFS_DATANODE_DATA_DIR_KEY
+ + " = " + oldVolumes, e);
+ throw e;
+ }
+ }
+
private synchronized void setClusterId(final String nsCid, final String bpid
) throws IOException {
if(clusterId != null && !clusterId.equals(nsCid)) {
@@ -822,7 +971,9 @@ public class DataNode extends Configured
// settings global for all BPs in the Data Node
this.secureResources = resources;
- this.dataDirs = dataDirs;
+ synchronized (this) {
+ this.dataDirs = dataDirs;
+ }
this.conf = conf;
this.dnConf = new DNConf(conf);
this.spanReceiverHost = SpanReceiverHost.getInstance(conf);
@@ -1112,7 +1263,9 @@ public class DataNode extends Configured
}
final String bpid = nsInfo.getBlockPoolID();
//read storage info, lock data dirs and transition fs state if necessary
- storage.recoverTransitionRead(this, bpid, nsInfo, dataDirs, startOpt);
+ synchronized (this) {
+ storage.recoverTransitionRead(this, bpid, nsInfo, dataDirs, startOpt);
+ }
final StorageInfo bpStorage = storage.getBPStorage(bpid);
LOG.info("Setting up storage: nsid=" + bpStorage.getNamespaceID()
+ ";bpid=" + bpid + ";lv=" + storage.getLayoutVersion()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe38d2e9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 553208e..4c03151 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -94,8 +94,8 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
public List<V> getVolumes();
/** Add an array of StorageLocation to FsDataset. */
- public void addVolumes(Collection<StorageLocation> volumes)
- throws IOException;
+ public List<StorageLocation> addVolumes(List<StorageLocation> volumes,
+ final Collection<String> bpids);
/** Removes a collection of volumes from FsDataset. */
public void removeVolumes(Collection<StorageLocation> volumes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe38d2e9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 05c9914..d28d616 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -28,19 +28,23 @@ import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
import javax.management.StandardMBean;
+import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -85,6 +89,7 @@ import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlo
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.util.DataChecksum;
@@ -245,7 +250,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
+ ", volume failures tolerated: " + volFailuresTolerated);
}
- storageMap = new HashMap<String, DatanodeStorage>();
+ storageMap = new ConcurrentHashMap<String, DatanodeStorage>();
volumeMap = new ReplicaMap(this);
@SuppressWarnings("unchecked")
final VolumeChoosingPolicy<FsVolumeImpl> blockChooserImpl =
@@ -275,45 +280,124 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// storageMap and asyncDiskService, consistent.
FsVolumeImpl fsVolume = new FsVolumeImpl(
this, sd.getStorageUuid(), dir, this.conf, storageType);
- fsVolume.getVolumeMap(volumeMap);
+ ReplicaMap tempVolumeMap = new ReplicaMap(this);
+ fsVolume.getVolumeMap(tempVolumeMap);
+ volumeMap.addAll(tempVolumeMap);
volumes.addVolume(fsVolume);
storageMap.put(sd.getStorageUuid(),
new DatanodeStorage(sd.getStorageUuid(),
- DatanodeStorage.State.NORMAL,
- storageType));
+ DatanodeStorage.State.NORMAL,
+ storageType));
asyncDiskService.addVolume(sd.getCurrentDir());
LOG.info("Added volume - " + dir + ", StorageType: " + storageType);
}
+ private void addVolumeAndBlockPool(Collection<StorageLocation> dataLocations,
+ Storage.StorageDirectory sd, final Collection<String> bpids)
+ throws IOException {
+ final File dir = sd.getCurrentDir();
+ final StorageType storageType =
+ getStorageTypeFromLocations(dataLocations, sd.getRoot());
+
+ final FsVolumeImpl fsVolume = new FsVolumeImpl(
+ this, sd.getStorageUuid(), dir, this.conf, storageType);
+ final ReplicaMap tempVolumeMap = new ReplicaMap(fsVolume);
+
+ List<IOException> exceptions = Lists.newArrayList();
+ for (final String bpid : bpids) {
+ try {
+ fsVolume.addBlockPool(bpid, this.conf);
+ fsVolume.getVolumeMap(bpid, tempVolumeMap);
+ } catch (IOException e) {
+ LOG.warn("Caught exception when adding " + fsVolume +
+ ". Will throw later.", e);
+ exceptions.add(e);
+ }
+ }
+ if (!exceptions.isEmpty()) {
+ // The states of FsDatasteImpl are not modified, thus no need to rolled back.
+ throw MultipleIOException.createIOException(exceptions);
+ }
+
+ volumeMap.addAll(tempVolumeMap);
+ storageMap.put(sd.getStorageUuid(),
+ new DatanodeStorage(sd.getStorageUuid(),
+ DatanodeStorage.State.NORMAL,
+ storageType));
+ asyncDiskService.addVolume(sd.getCurrentDir());
+ volumes.addVolume(fsVolume);
+
+ LOG.info("Added volume - " + dir + ", StorageType: " + storageType);
+ }
+
/**
* Add an array of StorageLocation to FsDataset.
*
* @pre dataStorage must have these volumes.
- * @param volumes
- * @throws IOException
+ * @param volumes an array of storage locations for adding volumes.
+ * @param bpids block pool IDs.
+ * @return an array of successfully loaded volumes.
*/
@Override
- public synchronized void addVolumes(Collection<StorageLocation> volumes)
- throws IOException {
+ public synchronized List<StorageLocation> addVolumes(
+ final List<StorageLocation> volumes, final Collection<String> bpids) {
final Collection<StorageLocation> dataLocations =
DataNode.getStorageLocations(this.conf);
- Map<String, Storage.StorageDirectory> allStorageDirs =
+ final Map<String, Storage.StorageDirectory> allStorageDirs =
new HashMap<String, Storage.StorageDirectory>();
- for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) {
- Storage.StorageDirectory sd = dataStorage.getStorageDir(idx);
- allStorageDirs.put(sd.getRoot().getAbsolutePath(), sd);
+ List<StorageLocation> succeedVolumes = Lists.newArrayList();
+ try {
+ for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) {
+ Storage.StorageDirectory sd = dataStorage.getStorageDir(idx);
+ allStorageDirs.put(sd.getRoot().getCanonicalPath(), sd);
+ }
+ } catch (IOException ioe) {
+ LOG.warn("Caught exception when parsing storage URL.", ioe);
+ return succeedVolumes;
+ }
+
+ final boolean[] successFlags = new boolean[volumes.size()];
+ Arrays.fill(successFlags, false);
+ List<Thread> volumeAddingThreads = Lists.newArrayList();
+ for (int i = 0; i < volumes.size(); i++) {
+ final int idx = i;
+ Thread t = new Thread() {
+ public void run() {
+ StorageLocation vol = volumes.get(idx);
+ try {
+ String key = vol.getFile().getCanonicalPath();
+ if (!allStorageDirs.containsKey(key)) {
+ LOG.warn("Attempt to add an invalid volume: " + vol.getFile());
+ } else {
+ addVolumeAndBlockPool(dataLocations, allStorageDirs.get(key),
+ bpids);
+ successFlags[idx] = true;
+ }
+ } catch (IOException e) {
+ LOG.warn("Caught exception when adding volume " + vol, e);
+ }
+ }
+ };
+ volumeAddingThreads.add(t);
+ t.start();
}
- for (StorageLocation vol : volumes) {
- String key = vol.getFile().getAbsolutePath();
- if (!allStorageDirs.containsKey(key)) {
- LOG.warn("Attempt to add an invalid volume: " + vol.getFile());
- } else {
- addVolume(dataLocations, allStorageDirs.get(key));
+ for (Thread t : volumeAddingThreads) {
+ try {
+ t.join();
+ } catch (InterruptedException e) {
+ LOG.warn("Caught InterruptedException when adding volume.", e);
}
}
+
+ for (int i = 0; i < volumes.size(); i++) {
+ if (successFlags[i]) {
+ succeedVolumes.add(volumes.get(i));
+ }
+ }
+ return succeedVolumes;
}
/**
@@ -335,9 +419,9 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
String volume = sd.getRoot().toString();
LOG.info("Removing " + volume + " from FsDataset.");
- this.volumes.removeVolume(volume);
- storageMap.remove(sd.getStorageUuid());
+ // Disable the volume from the service.
asyncDiskService.removeVolume(sd.getCurrentDir());
+ this.volumes.removeVolume(volume);
// Removed all replica information for the blocks on the volume. Unlike
// updating the volumeMap in addVolume(), this operation does not scan
@@ -348,7 +432,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
it.hasNext(); ) {
ReplicaInfo block = it.next();
if (block.getVolume().getBasePath().equals(volume)) {
- invalidate(bpid, block.getBlockId());
+ invalidate(bpid, block);
blocks.add(block);
it.remove();
}
@@ -357,6 +441,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
datanode.getBlockScanner().deleteBlocks(bpid,
blocks.toArray(new Block[blocks.size()]));
}
+
+ storageMap.remove(sd.getStorageUuid());
}
}
}
@@ -1345,23 +1431,26 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
/**
* Invalidate a block but does not delete the actual on-disk block file.
*
- * It should only be used for decommissioning disks.
+ * It should only be used when deactivating disks.
*
* @param bpid the block pool ID.
- * @param blockId the ID of the block.
+ * @param block The block to be invalidated.
*/
- public void invalidate(String bpid, long blockId) {
+ public void invalidate(String bpid, ReplicaInfo block) {
// If a DFSClient has the replica in its cache of short-circuit file
// descriptors (and the client is using ShortCircuitShm), invalidate it.
// The short-circuit registry is null in the unit tests, because the
// datanode is mock object.
if (datanode.getShortCircuitRegistry() != null) {
datanode.getShortCircuitRegistry().processBlockInvalidation(
- new ExtendedBlockId(blockId, bpid));
+ new ExtendedBlockId(block.getBlockId(), bpid));
// If the block is cached, start uncaching it.
- cacheManager.uncacheBlock(bpid, blockId);
+ cacheManager.uncacheBlock(bpid, block.getBlockId());
}
+
+ datanode.notifyNamenodeDeletedBlock(new ExtendedBlock(bpid, block),
+ block.getStorageUuid());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe38d2e9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index d0fad6e..83d93f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -1093,7 +1093,8 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
@Override
- public void addVolumes(Collection<StorageLocation> volumes) {
+ public List<StorageLocation> addVolumes(List<StorageLocation> volumes,
+ final Collection<String> bpids) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe38d2e9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
new file mode 100644
index 0000000..d2b2995
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -0,0 +1,423 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.ReconfigurationException;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.BlockMissingException;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestDataNodeHotSwapVolumes {
+ private static final int BLOCK_SIZE = 512;
+ private MiniDFSCluster cluster;
+
+ @After
+ public void tearDown() {
+ shutdown();
+ }
+
+ private void startDFSCluster(int numNameNodes, int numDataNodes)
+ throws IOException {
+ shutdown();
+ Configuration conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+
+ /*
+ * Lower the DN heartbeat, DF rate, and recheck interval to one second
+ * so state about failures and datanode death propagates faster.
+ */
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_DF_INTERVAL_KEY, 1000);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
+ 1000);
+
+ MiniDFSNNTopology nnTopology =
+ MiniDFSNNTopology.simpleFederatedTopology(numNameNodes);
+
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(nnTopology)
+ .numDataNodes(numDataNodes)
+ .build();
+ cluster.waitActive();
+ }
+
+ private void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ private void createFile(Path path, int numBlocks)
+ throws IOException, InterruptedException, TimeoutException {
+ final short replicateFactor = 1;
+ createFile(path, numBlocks, replicateFactor);
+ }
+
+ private void createFile(Path path, int numBlocks, short replicateFactor)
+ throws IOException, InterruptedException, TimeoutException {
+ createFile(0, path, numBlocks, replicateFactor);
+ }
+
+ private void createFile(int fsIdx, Path path, int numBlocks)
+ throws IOException, InterruptedException, TimeoutException {
+ final short replicateFactor = 1;
+ createFile(fsIdx, path, numBlocks, replicateFactor);
+ }
+
+ private void createFile(int fsIdx, Path path, int numBlocks,
+ short replicateFactor)
+ throws IOException, TimeoutException, InterruptedException {
+ final int seed = 0;
+ final DistributedFileSystem fs = cluster.getFileSystem(fsIdx);
+ DFSTestUtil.createFile(fs, path, BLOCK_SIZE * numBlocks,
+ replicateFactor, seed);
+ DFSTestUtil.waitReplication(fs, path, replicateFactor);
+ }
+
+ /**
+ * Verify whether a file has enough content.
+ */
+ private static void verifyFileLength(FileSystem fs, Path path, int numBlocks)
+ throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ assertEquals(numBlocks * BLOCK_SIZE, status.getLen());
+ }
+
+ /** Return the number of replicas for a given block in the file. */
+ private static int getNumReplicas(FileSystem fs, Path file,
+ int blockIdx) throws IOException {
+ BlockLocation locs[] = fs.getFileBlockLocations(file, 0, Long.MAX_VALUE);
+ return locs.length < blockIdx + 1 ? 0 : locs[blockIdx].getNames().length;
+ }
+
+ /**
+ * Wait the block to have the exact number of replicas as expected.
+ */
+ private static void waitReplication(FileSystem fs, Path file, int blockIdx,
+ int numReplicas)
+ throws IOException, TimeoutException, InterruptedException {
+ int attempts = 50; // Wait 5 seconds.
+ while (attempts > 0) {
+ if (getNumReplicas(fs, file, blockIdx) == numReplicas) {
+ return;
+ }
+ Thread.sleep(100);
+ attempts--;
+ }
+ throw new TimeoutException("Timed out waiting the " + blockIdx + "-th block"
+ + " of " + file + " to have " + numReplicas + " replicas.");
+ }
+
+ /** Parses data dirs from DataNode's configuration. */
+ private static Collection<String> getDataDirs(DataNode datanode) {
+ return datanode.getConf().getTrimmedStringCollection(
+ DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
+ }
+
+ @Test
+ public void testParseChangedVolumes() throws IOException {
+ startDFSCluster(1, 1);
+ DataNode dn = cluster.getDataNodes().get(0);
+ Configuration conf = dn.getConf();
+
+ String oldPaths = conf.get(DFS_DATANODE_DATA_DIR_KEY);
+ List<StorageLocation> oldLocations = new ArrayList<StorageLocation>();
+ for (String path : oldPaths.split(",")) {
+ oldLocations.add(StorageLocation.parse(path));
+ }
+ assertFalse(oldLocations.isEmpty());
+
+ String newPaths = "/foo/path1,/foo/path2";
+ conf.set(DFS_DATANODE_DATA_DIR_KEY, newPaths);
+
+ DataNode.ChangedVolumes changedVolumes =dn.parseChangedVolumes();
+ List<StorageLocation> newVolumes = changedVolumes.newLocations;
+ assertEquals(2, newVolumes.size());
+ assertEquals("/foo/path1", newVolumes.get(0).getFile().getAbsolutePath());
+ assertEquals("/foo/path2", newVolumes.get(1).getFile().getAbsolutePath());
+
+ List<StorageLocation> removedVolumes = changedVolumes.deactivateLocations;
+ assertEquals(oldLocations.size(), removedVolumes.size());
+ for (int i = 0; i < removedVolumes.size(); i++) {
+ assertEquals(oldLocations.get(i).getFile(),
+ removedVolumes.get(i).getFile());
+ }
+ }
+
+ @Test
+ public void testParseChangedVolumesFailures() throws IOException {
+ startDFSCluster(1, 1);
+ DataNode dn = cluster.getDataNodes().get(0);
+ Configuration conf = dn.getConf();
+ try {
+ conf.set(DFS_DATANODE_DATA_DIR_KEY, "");
+ dn.parseChangedVolumes();
+ fail("Should throw IOException: empty inputs.");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("No directory is specified.", e);
+ }
+ }
+
+ /** Add volumes to the first DataNode. */
+ private void addVolumes(int numNewVolumes) throws ReconfigurationException {
+ File dataDir = new File(cluster.getDataDirectory());
+ DataNode dn = cluster.getDataNodes().get(0); // First DataNode.
+ Configuration conf = dn.getConf();
+ String oldDataDir = conf.get(DFS_DATANODE_DATA_DIR_KEY);
+
+ List<File> newVolumeDirs = new ArrayList<File>();
+ StringBuilder newDataDirBuf = new StringBuilder(oldDataDir);
+ int startIdx = oldDataDir.split(",").length + 1;
+ // Find the first available (non-taken) directory name for data volume.
+ while (true) {
+ File volumeDir = new File(dataDir, "data" + startIdx);
+ if (!volumeDir.exists()) {
+ break;
+ }
+ startIdx++;
+ }
+ for (int i = startIdx; i < startIdx + numNewVolumes; i++) {
+ File volumeDir = new File(dataDir, "data" + String.valueOf(i));
+ newVolumeDirs.add(volumeDir);
+ volumeDir.mkdirs();
+ newDataDirBuf.append(",");
+ newDataDirBuf.append(volumeDir.toURI());
+ }
+
+ String newDataDir = newDataDirBuf.toString();
+ dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, newDataDir);
+ assertEquals(newDataDir, conf.get(DFS_DATANODE_DATA_DIR_KEY));
+
+ // Check that all newly created volumes are appropriately formatted.
+ for (File volumeDir : newVolumeDirs) {
+ File curDir = new File(volumeDir, "current");
+ assertTrue(curDir.exists());
+ assertTrue(curDir.isDirectory());
+ }
+ }
+
+ private List<List<Integer>> getNumBlocksReport(int namesystemIdx) {
+ List<List<Integer>> results = new ArrayList<List<Integer>>();
+ final String bpid = cluster.getNamesystem(namesystemIdx).getBlockPoolId();
+ List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
+ cluster.getAllBlockReports(bpid);
+ for (Map<DatanodeStorage, BlockListAsLongs> datanodeReport : blockReports) {
+ List<Integer> numBlocksPerDN = new ArrayList<Integer>();
+ for (BlockListAsLongs blocks : datanodeReport.values()) {
+ numBlocksPerDN.add(blocks.getNumberOfBlocks());
+ }
+ results.add(numBlocksPerDN);
+ }
+ return results;
+ }
+
+ /**
+ * Test adding one volume on a running MiniDFSCluster with only one NameNode.
+ */
+ @Test
+ public void testAddOneNewVolume()
+ throws IOException, ReconfigurationException,
+ InterruptedException, TimeoutException {
+ startDFSCluster(1, 1);
+ String bpid = cluster.getNamesystem().getBlockPoolId();
+ final int numBlocks = 10;
+
+ addVolumes(1);
+
+ Path testFile = new Path("/test");
+ createFile(testFile, numBlocks);
+
+ List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
+ cluster.getAllBlockReports(bpid);
+ assertEquals(1, blockReports.size()); // 1 DataNode
+ assertEquals(3, blockReports.get(0).size()); // 3 volumes
+
+ // FSVolumeList uses Round-Robin block chooser by default. Thus the new
+ // blocks should be evenly located in all volumes.
+ int minNumBlocks = Integer.MAX_VALUE;
+ int maxNumBlocks = Integer.MIN_VALUE;
+ for (BlockListAsLongs blockList : blockReports.get(0).values()) {
+ minNumBlocks = Math.min(minNumBlocks, blockList.getNumberOfBlocks());
+ maxNumBlocks = Math.max(maxNumBlocks, blockList.getNumberOfBlocks());
+ }
+ assertTrue(Math.abs(maxNumBlocks - maxNumBlocks) <= 1);
+ verifyFileLength(cluster.getFileSystem(), testFile, numBlocks);
+ }
+
+ @Test(timeout = 60000)
+ public void testAddVolumesDuringWrite()
+ throws IOException, InterruptedException, TimeoutException,
+ ReconfigurationException {
+ startDFSCluster(1, 1);
+ String bpid = cluster.getNamesystem().getBlockPoolId();
+ Path testFile = new Path("/test");
+ createFile(testFile, 4); // Each volume has 2 blocks.
+
+ addVolumes(2);
+
+ // Continue to write the same file, thus the new volumes will have blocks.
+ DFSTestUtil.appendFile(cluster.getFileSystem(), testFile, BLOCK_SIZE * 8);
+ verifyFileLength(cluster.getFileSystem(), testFile, 8 + 4);
+ // After appending data, there should be [2, 2, 4, 4] blocks in each volume
+ // respectively.
+ List<Integer> expectedNumBlocks = Arrays.asList(2, 2, 4, 4);
+
+ List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
+ cluster.getAllBlockReports(bpid);
+ assertEquals(1, blockReports.size()); // 1 DataNode
+ assertEquals(4, blockReports.get(0).size()); // 4 volumes
+ Map<DatanodeStorage, BlockListAsLongs> dnReport =
+ blockReports.get(0);
+ List<Integer> actualNumBlocks = new ArrayList<Integer>();
+ for (BlockListAsLongs blockList : dnReport.values()) {
+ actualNumBlocks.add(blockList.getNumberOfBlocks());
+ }
+ Collections.sort(actualNumBlocks);
+ assertEquals(expectedNumBlocks, actualNumBlocks);
+ }
+
+ @Test
+ public void testAddVolumesToFederationNN()
+ throws IOException, TimeoutException, InterruptedException,
+ ReconfigurationException {
+ // Starts a Cluster with 2 NameNode and 3 DataNodes. Each DataNode has 2
+ // volumes.
+ final int numNameNodes = 2;
+ final int numDataNodes = 1;
+ startDFSCluster(numNameNodes, numDataNodes);
+ Path testFile = new Path("/test");
+ // Create a file on the first namespace with 4 blocks.
+ createFile(0, testFile, 4);
+ // Create a file on the second namespace with 4 blocks.
+ createFile(1, testFile, 4);
+
+ // Add 2 volumes to the first DataNode.
+ final int numNewVolumes = 2;
+ addVolumes(numNewVolumes);
+
+ // Append to the file on the first namespace.
+ DFSTestUtil.appendFile(cluster.getFileSystem(0), testFile, BLOCK_SIZE * 8);
+
+ List<List<Integer>> actualNumBlocks = getNumBlocksReport(0);
+ assertEquals(cluster.getDataNodes().size(), actualNumBlocks.size());
+ List<Integer> blocksOnFirstDN = actualNumBlocks.get(0);
+ Collections.sort(blocksOnFirstDN);
+ assertEquals(Arrays.asList(2, 2, 4, 4), blocksOnFirstDN);
+
+ // Verify the second namespace also has the new volumes and they are empty.
+ actualNumBlocks = getNumBlocksReport(1);
+ assertEquals(4, actualNumBlocks.get(0).size());
+ assertEquals(numNewVolumes,
+ Collections.frequency(actualNumBlocks.get(0), 0));
+ }
+
+ @Test
+ public void testRemoveOneVolume()
+ throws ReconfigurationException, InterruptedException, TimeoutException,
+ IOException {
+ startDFSCluster(1, 1);
+ final short replFactor = 1;
+ Path testFile = new Path("/test");
+ createFile(testFile, 10, replFactor);
+
+ DataNode dn = cluster.getDataNodes().get(0);
+ Collection<String> oldDirs = getDataDirs(dn);
+ String newDirs = oldDirs.iterator().next(); // Keep the first volume.
+ dn.reconfigurePropertyImpl(
+ DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
+ dn.scheduleAllBlockReport(0);
+
+ try {
+ DFSTestUtil.readFile(cluster.getFileSystem(), testFile);
+ fail("Expect to throw BlockMissingException.");
+ } catch (BlockMissingException e) {
+ GenericTestUtils.assertExceptionContains("Could not obtain block", e);
+ }
+
+ Path newFile = new Path("/newFile");
+ createFile(newFile, 6);
+
+ String bpid = cluster.getNamesystem().getBlockPoolId();
+ List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
+ cluster.getAllBlockReports(bpid);
+ assertEquals((int)replFactor, blockReports.size());
+
+ BlockListAsLongs blocksForVolume1 =
+ blockReports.get(0).values().iterator().next();
+ // The first volume has half of the testFile and full of newFile.
+ assertEquals(10 / 2 + 6, blocksForVolume1.getNumberOfBlocks());
+ }
+
+ @Test
+ public void testReplicatingAfterRemoveVolume()
+ throws InterruptedException, TimeoutException, IOException,
+ ReconfigurationException {
+ startDFSCluster(1, 2);
+ final DistributedFileSystem fs = cluster.getFileSystem();
+ final short replFactor = 2;
+ Path testFile = new Path("/test");
+ createFile(testFile, 4, replFactor);
+
+ DataNode dn = cluster.getDataNodes().get(0);
+ Collection<String> oldDirs = getDataDirs(dn);
+ String newDirs = oldDirs.iterator().next(); // Keep the first volume.
+ dn.reconfigurePropertyImpl(
+ DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
+
+ // Force DataNode to report missing blocks.
+ dn.scheduleAllBlockReport(0);
+ DataNodeTestUtils.triggerDeletionReport(dn);
+
+ // The 2nd block only has 1 replica due to the removed data volume.
+ waitReplication(fs, testFile, 1, 1);
+
+ // Wait NameNode to replica missing blocks.
+ DFSTestUtil.waitReplication(fs, testFile, replFactor);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe38d2e9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index 2c4c401..10b9f7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -40,7 +40,10 @@ import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -117,6 +120,7 @@ public class TestFsDatasetImpl {
final int numExistingVolumes = dataset.getVolumes().size();
final int totalVolumes = numNewVolumes + numExistingVolumes;
List<StorageLocation> newLocations = new ArrayList<StorageLocation>();
+ Set<String> expectedVolumes = new HashSet<String>();
for (int i = 0; i < numNewVolumes; i++) {
String path = BASE_DIR + "/newData" + i;
newLocations.add(StorageLocation.parse(path));
@@ -125,13 +129,15 @@ public class TestFsDatasetImpl {
}
when(storage.getNumStorageDirs()).thenReturn(totalVolumes);
- dataset.addVolumes(newLocations);
+ dataset.addVolumes(newLocations, Arrays.asList(BLOCK_POOL_IDS));
assertEquals(totalVolumes, dataset.getVolumes().size());
assertEquals(totalVolumes, dataset.storageMap.size());
+
+ Set<String> actualVolumes = new HashSet<String>();
for (int i = 0; i < numNewVolumes; i++) {
- assertEquals(newLocations.get(i).getFile().getPath(),
- dataset.getVolumes().get(numExistingVolumes + i).getBasePath());
+ dataset.getVolumes().get(numExistingVolumes + i).getBasePath();
}
+ assertEquals(actualVolumes, expectedVolumes);
}
@Test
[29/50] [abbrv] git commit: HADOOP-10131.
NetWorkTopology#countNumOfAvailableNodes() is returning wrong value if
excluded nodes passed are not part of the cluster tree (Contributed by
Vinayakumar B)
Posted by cu...@apache.org.
HADOOP-10131. NetWorkTopology#countNumOfAvailableNodes() is returning wrong value if excluded nodes passed are not part of the cluster tree (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eab15af1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eab15af1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eab15af1
Branch: refs/heads/YARN-1051
Commit: eab15af12c114eef4e9abd9af2ba03b0ab2cc441
Parents: 1737950
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Sep 22 11:25:56 2014 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Mon Sep 22 11:25:56 2014 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 4 +
.../org/apache/hadoop/net/NetworkTopology.java | 30 +++--
.../apache/hadoop/net/TestClusterTopology.java | 122 +++++++++++++++++++
3 files changed, 146 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eab15af1/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2b07f8d..0b37577 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -847,6 +847,10 @@ Release 2.6.0 - UNRELEASED
HADOOP-10946. Fix a bunch of typos in log messages (Ray Chiang via aw)
+ HADOOP-10131. NetWorkTopology#countNumOfAvailableNodes() is returning
+ wrong value if excluded nodes passed are not part of the cluster tree
+ (vinayakumarb)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eab15af1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 50d5116..5f11367 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -782,25 +782,35 @@ public class NetworkTopology {
scope=scope.substring(1);
}
scope = NodeBase.normalize(scope);
- int count=0; // the number of nodes in both scope & excludedNodes
+ int excludedCountInScope = 0; // the number of nodes in both scope & excludedNodes
+ int excludedCountOffScope = 0; // the number of nodes outside scope & excludedNodes
netlock.readLock().lock();
try {
- for(Node node:excludedNodes) {
- if ((NodeBase.getPath(node)+NodeBase.PATH_SEPARATOR_STR).
- startsWith(scope+NodeBase.PATH_SEPARATOR_STR)) {
- count++;
+ for (Node node : excludedNodes) {
+ node = getNode(NodeBase.getPath(node));
+ if (node == null) {
+ continue;
}
+ if ((NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR)
+ .startsWith(scope + NodeBase.PATH_SEPARATOR_STR)) {
+ excludedCountInScope++;
+ } else {
+ excludedCountOffScope++;
+ }
+ }
+ Node n = getNode(scope);
+ int scopeNodeCount = 0;
+ if (n != null) {
+ scopeNodeCount++;
}
- Node n=getNode(scope);
- int scopeNodeCount=1;
if (n instanceof InnerNode) {
scopeNodeCount=((InnerNode)n).getNumOfLeaves();
}
if (isExcluded) {
- return clusterMap.getNumOfLeaves()-
- scopeNodeCount-excludedNodes.size()+count;
+ return clusterMap.getNumOfLeaves() - scopeNodeCount
+ - excludedCountOffScope;
} else {
- return scopeNodeCount-count;
+ return scopeNodeCount - excludedCountInScope;
}
} finally {
netlock.readLock().unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eab15af1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
new file mode 100644
index 0000000..3ab663f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.net;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestClusterTopology extends Assert {
+
+ public static class NodeElement implements Node {
+ private String location;
+ private String name;
+ private Node parent;
+ private int level;
+
+ public NodeElement(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getNetworkLocation() {
+ return location;
+ }
+
+ @Override
+ public void setNetworkLocation(String location) {
+ this.location = location;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Node getParent() {
+ return parent;
+ }
+
+ @Override
+ public void setParent(Node parent) {
+ this.parent = parent;
+ }
+
+ @Override
+ public int getLevel() {
+ return level;
+ }
+
+ @Override
+ public void setLevel(int i) {
+ this.level = i;
+ }
+
+ }
+
+ /**
+ * Test the count of nodes with exclude list
+ */
+ @Test
+ public void testCountNumNodes() throws Exception {
+ // create the topology
+ NetworkTopology cluster = new NetworkTopology();
+ cluster.add(getNewNode("node1", "/d1/r1"));
+ NodeElement node2 = getNewNode("node2", "/d1/r2");
+ cluster.add(node2);
+ cluster.add(getNewNode("node3", "/d1/r3"));
+ NodeElement node3 = getNewNode("node4", "/d1/r4");
+ cluster.add(node3);
+ // create exclude list
+ List<Node> excludedNodes = new ArrayList<Node>();
+
+ assertEquals("4 nodes should be available", 4,
+ cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes));
+ NodeElement deadNode = getNewNode("node5", "/d1/r2");
+ excludedNodes.add(deadNode);
+ assertEquals("4 nodes should be available with extra excluded Node", 4,
+ cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes));
+ // add one existing node to exclude list
+ excludedNodes.add(node3);
+ assertEquals("excluded nodes with ROOT scope should be considered", 3,
+ cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes));
+ assertEquals("excluded nodes without ~ scope should be considered", 2,
+ cluster.countNumOfAvailableNodes("~" + deadNode.getNetworkLocation(),
+ excludedNodes));
+ assertEquals("excluded nodes with rack scope should be considered", 1,
+ cluster.countNumOfAvailableNodes(deadNode.getNetworkLocation(),
+ excludedNodes));
+ // adding the node in excluded scope to excluded list
+ excludedNodes.add(node2);
+ assertEquals("excluded nodes with ~ scope should be considered", 2,
+ cluster.countNumOfAvailableNodes("~" + deadNode.getNetworkLocation(),
+ excludedNodes));
+ // getting count with non-exist scope.
+ assertEquals("No nodes should be considered for non-exist scope", 0,
+ cluster.countNumOfAvailableNodes("/non-exist", excludedNodes));
+ }
+
+ private NodeElement getNewNode(String name, String rackLocation) {
+ NodeElement node = new NodeElement(name);
+ node.setNetworkLocation(rackLocation);
+ return node;
+ }
+}
[26/50] [abbrv] git commit: HDFS-7110. Skip tests related to
short-circuit read on platforms that do not currently implement short-circuit
read. Contributed by Chris Nauroth.
Posted by cu...@apache.org.
HDFS-7110. Skip tests related to short-circuit read on platforms that do not currently implement short-circuit read. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6fd5ccb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6fd5ccb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6fd5ccb
Branch: refs/heads/YARN-1051
Commit: b6fd5ccbce25f390c0e348d296b4fe1f8b9b738f
Parents: 0795fe3
Author: cnauroth <cn...@apache.org>
Authored: Sun Sep 21 12:36:57 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Sun Sep 21 12:36:57 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/datanode/TestFsDatasetCacheRevocation.java | 5 +++++
.../apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java | 4 ++++
3 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fd5ccb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5a69d9d..f29f7c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -911,6 +911,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7109. TestDataStorage does not release file locks between tests.
(cnauroth)
+ HDFS-7110. Skip tests related to short-circuit read on platforms that do not
+ currently implement short-circuit read. (cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fd5ccb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCacheRevocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCacheRevocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCacheRevocation.java
index af28ed7..d5531db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCacheRevocation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCacheRevocation.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
+import static org.junit.Assume.assumeTrue;
+
import java.io.File;
import java.nio.ByteBuffer;
import java.util.EnumSet;
@@ -41,6 +43,7 @@ import org.apache.hadoop.io.nativeio.NativeIO.POSIX.CacheManipulator;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator;
import org.apache.hadoop.net.unix.DomainSocket;
import org.apache.hadoop.net.unix.TemporarySocketDirectory;
+import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -94,6 +97,7 @@ public class TestFsDatasetCacheRevocation {
*/
@Test(timeout=120000)
public void testPinning() throws Exception {
+ assumeTrue(NativeCodeLoader.isNativeCodeLoaded() && !Path.WINDOWS);
Configuration conf = getDefaultConf();
// Set a really long revocation timeout, so that we won't reach it during
// this test.
@@ -143,6 +147,7 @@ public class TestFsDatasetCacheRevocation {
*/
@Test(timeout=120000)
public void testRevocation() throws Exception {
+ assumeTrue(NativeCodeLoader.isNativeCodeLoaded() && !Path.WINDOWS);
BlockReaderTestUtil.enableHdfsCachingTracing();
BlockReaderTestUtil.enableShortCircuitShmTracing();
Configuration conf = getDefaultConf();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fd5ccb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
index 7fe8a1e..800cc6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.tracing;
+import static org.junit.Assume.assumeTrue;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
@@ -27,6 +29,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.unix.DomainSocket;
import org.apache.hadoop.net.unix.TemporarySocketDirectory;
+import org.apache.hadoop.util.NativeCodeLoader;
import org.htrace.Sampler;
import org.htrace.Span;
import org.htrace.Trace;
@@ -59,6 +62,7 @@ public class TestTracingShortCircuitLocalRead {
@Test
public void testShortCircuitTraceHooks() throws IOException {
+ assumeTrue(NativeCodeLoader.isNativeCodeLoaded() && !Path.WINDOWS);
conf = new Configuration();
conf.set(SpanReceiverHost.SPAN_RECEIVERS_CONF_KEY,
TestTracing.SetSpanReceiver.class.getName());
[11/50] [abbrv] git commit: HDFS-7065. Pipeline close recovery race
can cause block corruption. Contributed by Kihwal Lee.
Posted by cu...@apache.org.
HDFS-7065. Pipeline close recovery race can cause block corruption.
Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf27b9ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf27b9ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf27b9ca
Branch: refs/heads/YARN-1051
Commit: bf27b9ca574592ef603e126bacb9b6a37c9eb5c6
Parents: f02d934
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Sep 19 09:00:13 2014 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Sep 19 09:00:13 2014 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf27b9ca/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9d76c37..ba1a272 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -777,6 +777,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6840. Clients are always sent to the same datanode when read
is off rack. (wang)
+ HDFS-7065. Pipeline close recovery race can cause block corruption (kihwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf27b9ca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index a2179dc..6e81082 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -904,7 +904,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
@Override // FsDatasetSpi
- public String recoverClose(ExtendedBlock b, long newGS,
+ public synchronized String recoverClose(ExtendedBlock b, long newGS,
long expectedBlockLen) throws IOException {
LOG.info("Recover failed close " + b);
// check replica's state
[02/50] [abbrv] git commit: Merge branch 'HDFS-6584' into trunk
Posted by cu...@apache.org.
Merge branch 'HDFS-6584' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98f7bfe9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98f7bfe9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98f7bfe9
Branch: refs/heads/YARN-1051
Commit: 98f7bfe99395bb27d97a4f6076fc10ede50383f0
Parents: fad4cd8 76ccb80
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Fri Sep 19 07:11:19 2014 +0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Fri Sep 19 07:11:19 2014 +0800
----------------------------------------------------------------------
.../hadoop-common/src/main/conf/hadoop-env.sh | 9 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 76 ++
.../hadoop-hdfs/src/main/bin/hdfs | 7 +
.../hadoop-hdfs/src/main/bin/hdfs.cmd | 8 +-
.../src/main/conf/blockStoragePolicy-site.xml | 21 +
.../hadoop-hdfs/src/main/conf/hdfs-site.xml | 3 +-
.../apache/hadoop/hdfs/BlockStoragePolicy.java | 419 +++++++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 19 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 15 +
.../hadoop/hdfs/DistributedFileSystem.java | 34 +-
.../org/apache/hadoop/hdfs/StorageType.java | 6 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 14 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 5 +-
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 10 +-
.../hdfs/protocol/HdfsLocatedFileStatus.java | 10 +-
.../protocol/SnapshottableDirectoryStatus.java | 3 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 16 +
.../ClientNamenodeProtocolTranslatorPB.java | 16 +
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 37 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 19 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 241 ++--
.../hadoop/hdfs/server/balancer/Matcher.java | 15 +
.../hdfs/server/balancer/MovedBlocks.java | 2 +-
.../hdfs/server/balancer/NameNodeConnector.java | 79 +-
.../server/blockmanagement/BlockCollection.java | 5 +
.../server/blockmanagement/BlockManager.java | 89 +-
.../blockmanagement/BlockPlacementPolicy.java | 22 +-
.../BlockPlacementPolicyDefault.java | 377 +++---
.../BlockPlacementPolicyWithNodeGroup.java | 73 +-
.../blockmanagement/DatanodeDescriptor.java | 52 +-
.../blockmanagement/DatanodeStorageInfo.java | 24 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 4 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 644 +++++++++++
.../hdfs/server/namenode/FSDirectory.java | 118 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 12 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 12 +-
.../hdfs/server/namenode/FSEditLogOp.java | 67 ++
.../hdfs/server/namenode/FSEditLogOpCodes.java | 1 +
.../hdfs/server/namenode/FSImageFormat.java | 4 +-
.../server/namenode/FSImageFormatPBINode.java | 6 +-
.../server/namenode/FSImageSerialization.java | 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 89 +-
.../hadoop/hdfs/server/namenode/INode.java | 17 +-
.../hdfs/server/namenode/INodeDirectory.java | 27 +
.../namenode/INodeDirectoryAttributes.java | 4 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 49 +-
.../server/namenode/INodeFileAttributes.java | 14 +-
.../hadoop/hdfs/server/namenode/INodeMap.java | 11 +
.../hdfs/server/namenode/INodeReference.java | 10 +
.../hdfs/server/namenode/INodeSymlink.java | 12 +
.../server/namenode/NameNodeLayoutVersion.java | 3 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 8 +-
.../snapshot/FSImageFormatPBSnapshot.java | 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 8 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 75 +-
.../apache/hadoop/hdfs/util/EnumCounters.java | 9 +
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 10 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 10 +
.../hadoop-hdfs/src/main/proto/fsimage.proto | 1 +
.../hadoop-hdfs/src/main/proto/hdfs.proto | 3 +
.../resources/blockStoragePolicy-default.xml | 118 ++
.../src/main/resources/hdfs-default.xml | 3 +-
.../src/site/apt/ArchivalStorage.apt.vm | 302 +++++
.../src/site/apt/HDFSCommands.apt.vm | 43 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 119 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 12 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 1075 ++++++++++++++++++
.../hadoop/hdfs/TestDFSClientRetries.java | 4 +-
.../hdfs/TestDFSInotifyEventInputStream.java | 2 +-
.../java/org/apache/hadoop/hdfs/TestLease.java | 9 +-
.../hadoop/hdfs/TestSnapshotCommands.java | 152 +--
.../hadoop/hdfs/TestStoragePolicyCommands.java | 81 ++
.../hdfs/server/balancer/TestBalancer.java | 7 +
.../balancer/TestBalancerWithHANameNodes.java | 2 +-
.../TestBalancerWithMultipleNameNodes.java | 2 +-
.../balancer/TestBalancerWithNodeGroup.java | 2 +-
.../blockmanagement/TestBlockManager.java | 20 +-
.../blockmanagement/TestReplicationPolicy.java | 27 +-
.../TestReplicationPolicyConsiderLoad.java | 28 +-
.../TestReplicationPolicyWithNodeGroup.java | 17 +-
.../hadoop/hdfs/server/mover/TestMover.java | 222 ++++
.../hdfs/server/mover/TestStorageMover.java | 766 +++++++++++++
.../hdfs/server/namenode/CreateEditsLog.java | 4 +-
.../hdfs/server/namenode/TestAddBlockRetry.java | 4 +-
.../hdfs/server/namenode/TestDeleteRace.java | 6 +-
.../hdfs/server/namenode/TestEditLog.java | 2 +-
.../namenode/TestFSPermissionChecker.java | 35 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 3 +-
.../hdfs/server/namenode/TestINodeFile.java | 35 +-
.../hdfs/server/namenode/ha/TestDNFencing.java | 4 +-
.../apache/hadoop/hdfs/web/TestJsonUtil.java | 2 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 5252 -> 4999 bytes
.../src/test/resources/editsStored.xml | 422 ++++---
hadoop-project/src/site/site.xml | 1 +
94 files changed, 5666 insertions(+), 824 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98f7bfe9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98f7bfe9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
[08/50] [abbrv] git commit: HADOOP-11109. Site build is broken.
Contributed by Jian He.
Posted by cu...@apache.org.
HADOOP-11109. Site build is broken. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64345722
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64345722
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64345722
Branch: refs/heads/YARN-1051
Commit: 64345722975a671869fcfd66a7263f831b36d068
Parents: 8e73084
Author: Aaron T. Myers <at...@apache.org>
Authored: Thu Sep 18 17:59:36 2014 -0700
Committer: Aaron T. Myers <at...@apache.org>
Committed: Thu Sep 18 18:00:39 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 4 +++-
hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64345722/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index f21771b..90053fc 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -834,7 +834,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11105. MetricsSystemImpl could leak memory in registered callbacks.
(Chuan Liu via cnauroth)
- KMS: Support for multiple Kerberos principals. (tucu)
+ HADOOP-10982. KMS: Support for multiple Kerberos principals. (tucu)
+
+ HADOOP-11109. Site build is broken. (Jian He via atm)
Release 2.5.1 - 2014-09-05
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64345722/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index cf7a557..e32893b 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -626,7 +626,7 @@ $ keytool -genkey -alias tomcat -keyalg RSA
<<NOTE:>> If using HTTPS, the SSL certificate used by the KMS instance must
be configured to support multiple hostnames (see Java 7
- <<<keytool>> SAN extension support for details on how to do this).
+ <<<keytool>>> SAN extension support for details on how to do this).
*** HTTP Authentication Signature
[16/50] [abbrv] git commit: YARN-2460. Remove obsolete entries from
yarn-default.xml (Ray Chiang via aw)
Posted by cu...@apache.org.
YARN-2460. Remove obsolete entries from yarn-default.xml (Ray Chiang via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa1052c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa1052c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa1052c3
Branch: refs/heads/YARN-1051
Commit: aa1052c34b78b5b8b6a1498c8c842d21b07fceca
Parents: 951847b
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 19 14:07:38 2014 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Sep 19 14:07:38 2014 -0700
----------------------------------------------------------------------
.../resources/job_1329348432655_0001_conf.xml | 1 -
.../src/main/data/2jobs2min-rumen-jh.json | 6 +--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../src/main/resources/yarn-default.xml | 39 +-------------------
4 files changed, 6 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa1052c3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
index e34518c..bd9c9c5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
@@ -152,7 +152,6 @@
<property><!--Loaded from job.xml--><name>mapreduce.map.speculative</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.acl-view-job</name><value> </value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.output.key.class</name><value>org.apache.hadoop.io.IntWritable</value></property>
-<property><!--Loaded from job.xml--><name>yarn.ipc.serializer.type</name><value>protocolbuffers</value></property>
<property><!--Loaded from mapred-default.xml--><name>mapreduce.job.end-notification.max.retry.interval</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>ftp.blocksize</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.http.threads</name><value>40</value></property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa1052c3/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
index 230e4fd..59ae8d7 100644
--- a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
+++ b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
@@ -4716,7 +4716,6 @@
"dfs.journalnode.http-address" : "0.0.0.0:8480",
"mapreduce.job.acl-view-job" : " ",
"mapreduce.reduce.shuffle.retry-delay.max.ms" : "60000",
- "yarn.ipc.serializer.type" : "protocolbuffers",
"mapreduce.job.end-notification.max.retry.interval" : "5",
"ftp.blocksize" : "67108864",
"mapreduce.tasktracker.http.threads" : "80",
@@ -4841,7 +4840,7 @@
"yarn.ipc.rpc.class" : "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC",
"mapreduce.job.name" : "TeraGen",
"kfs.blocksize" : "67108864",
- "yarn.resourcemanager.application-tokens.master-key-rolling-interval-secs" : "86400",
+ "yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs" : "86400",
"mapreduce.job.ubertask.maxmaps" : "9",
"yarn.scheduler.maximum-allocation-mb" : "8192",
"yarn.nodemanager.heartbeat.interval-ms" : "1000",
@@ -9830,7 +9829,6 @@
"dfs.journalnode.http-address" : "0.0.0.0:8480",
"mapreduce.job.acl-view-job" : " ",
"mapreduce.reduce.shuffle.retry-delay.max.ms" : "60000",
- "yarn.ipc.serializer.type" : "protocolbuffers",
"mapreduce.job.end-notification.max.retry.interval" : "5",
"ftp.blocksize" : "67108864",
"mapreduce.tasktracker.http.threads" : "80",
@@ -9955,7 +9953,7 @@
"yarn.ipc.rpc.class" : "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC",
"mapreduce.job.name" : "TeraGen",
"kfs.blocksize" : "67108864",
- "yarn.resourcemanager.application-tokens.master-key-rolling-interval-secs" : "86400",
+ "yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs" : "86400",
"mapreduce.job.ubertask.maxmaps" : "9",
"yarn.scheduler.maximum-allocation-mb" : "8192",
"yarn.nodemanager.heartbeat.interval-ms" : "1000",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa1052c3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7175b14..9c484c7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -403,6 +403,9 @@ Release 2.6.0 - UNRELEASED
YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless
explicitly set. (Zhijie Shen via jianhe)
+ YARN-2460. Remove obsolete entries from yarn-default.xml (Ray Chiang via
+ aw)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa1052c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a2c3fd0..e642d05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -31,22 +31,11 @@
</property>
<property>
- <description>Type of serialization to use.</description>
- <name>yarn.ipc.serializer.type</name>
- <value>protocolbuffers</value>
- </property>
-
- <property>
<description>Factory to create server IPC classes.</description>
<name>yarn.ipc.server.factory.class</name>
</property>
<property>
- <description>Factory to create IPC exceptions.</description>
- <name>yarn.ipc.exception.factory.class</name>
- </property>
-
- <property>
<description>Factory to create serializeable records.</description>
<name>yarn.ipc.record.factory.class</name>
</property>
@@ -163,12 +152,6 @@
</property>
<property>
- <description>How often should the RM check that the AM is still alive.</description>
- <name>yarn.resourcemanager.amliveliness-monitor.interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
<description>Maximum time to wait to establish connection to
ResourceManager.</description>
<name>yarn.resourcemanager.connect.max-wait.ms</name>
@@ -221,12 +204,6 @@
</property>
<property>
- <description>How often to check that node managers are still alive.</description>
- <name>yarn.resourcemanager.nm.liveness-monitor.interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
<description>Path to file with nodes to include.</description>
<name>yarn.resourcemanager.nodes.include-path</name>
<value></value>
@@ -580,7 +557,7 @@
<description>Interval for the roll over for the master key used to generate
application tokens
</description>
- <name>yarn.resourcemanager.application-tokens.master-key-rolling-interval-secs</name>
+ <name>yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs</name>
<value>86400</value>
</property>
@@ -1091,20 +1068,6 @@
</property>
<property>
- <description>Max time, in seconds, to wait to establish a connection to RM when NM starts.
- The NM will shutdown if it cannot connect to RM within the specified max time period.
- If the value is set as -1, then NM will retry forever.</description>
- <name>yarn.nodemanager.resourcemanager.connect.wait.secs</name>
- <value>900</value>
- </property>
-
- <property>
- <description>Time interval, in seconds, between each NM attempt to connect to RM.</description>
- <name>yarn.nodemanager.resourcemanager.connect.retry_interval.secs</name>
- <value>30</value>
- </property>
-
- <property>
<description>The minimum allowed version of a resourcemanager that a nodemanager will connect to.
The valid values are NONE (no version checking), EqualToNM (the resourcemanager's version is
equal to or greater than the NM version), or a Version String.</description>
[17/50] [abbrv] git commit: HADOOP-10970. Cleanup KMS configuration
keys. (wang)
Posted by cu...@apache.org.
HADOOP-10970. Cleanup KMS configuration keys. (wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adf0b67a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adf0b67a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adf0b67a
Branch: refs/heads/YARN-1051
Commit: adf0b67a7104bd457b20c95ff78dd48753dcd699
Parents: aa1052c
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Sep 19 14:54:37 2014 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Sep 19 14:59:25 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../hadoop-kms/src/main/conf/kms-acls.xml | 12 ++---
.../hadoop-kms/src/main/conf/kms-site.xml | 47 +++++++++++++++++++-
.../hadoop/crypto/key/kms/server/KMSAudit.java | 14 ++++--
.../crypto/key/kms/server/KMSConfiguration.java | 6 +--
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 4 +-
6 files changed, 70 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adf0b67a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7fc3b5b..0603304 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -535,6 +535,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11106. Document considerations of HAR and Encryption. (clamb via wang)
+ HADOOP-10970. Cleanup KMS configuration keys. (wang)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adf0b67a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
index 24a46b8..1d5b649 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
@@ -23,7 +23,7 @@
<value>*</value>
<description>
ACL for create-key operations.
- If the user does is not in the GET ACL, the key material is not returned
+ If the user is not in the GET ACL, the key material is not returned
as part of the response.
</description>
</property>
@@ -58,7 +58,7 @@
<name>hadoop.kms.acl.GET_KEYS</name>
<value>*</value>
<description>
- ACL for get-keys operation.
+ ACL for get-keys operations.
</description>
</property>
@@ -66,7 +66,7 @@
<name>hadoop.kms.acl.GET_METADATA</name>
<value>*</value>
<description>
- ACL for get-key-metadata an get-keys-metadata operations.
+ ACL for get-key-metadata and get-keys-metadata operations.
</description>
</property>
@@ -74,7 +74,7 @@
<name>hadoop.kms.acl.SET_KEY_MATERIAL</name>
<value>*</value>
<description>
- Complimentary ACL for CREATE and ROLLOVER operation to allow the client
+ Complementary ACL for CREATE and ROLLOVER operations to allow the client
to provide the key material when creating or rolling a key.
</description>
</property>
@@ -83,7 +83,7 @@
<name>hadoop.kms.acl.GENERATE_EEK</name>
<value>*</value>
<description>
- ACL for generateEncryptedKey CryptoExtension operations
+ ACL for generateEncryptedKey CryptoExtension operations.
</description>
</property>
@@ -91,7 +91,7 @@
<name>hadoop.kms.acl.DECRYPT_EEK</name>
<value>*</value>
<description>
- ACL for decrypt EncryptedKey CryptoExtension operations
+ ACL for decryptEncryptedKey CryptoExtension operations.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adf0b67a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
index 4f4694c..a810ca4 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
@@ -15,10 +15,12 @@
<configuration>
<!-- KMS Backend KeyProvider -->
+
<property>
<name>hadoop.kms.key.provider.uri</name>
<value>jceks://file@/${user.home}/kms.keystore</value>
<description>
+ URI of the backing KeyProvider for the KMS.
</description>
</property>
@@ -26,14 +28,52 @@
<name>hadoop.security.keystore.JavaKeyStoreProvider.password</name>
<value>none</value>
<description>
+ If using the JavaKeyStoreProvider, the password for the keystore file.
</description>
</property>
<!-- KMS Cache -->
+
+ <property>
+ <name>hadoop.kms.cache.enable</name>
+ <value>true</value>
+ <description>
+ Whether the KMS will act as a cache for the backing KeyProvider.
+ When the cache is enabled, operations like getKeyVersion, getMetadata,
+ and getCurrentKey will sometimes return cached data without consulting
+ the backing KeyProvider. Cached values are flushed when keys are deleted
+ or modified.
+ </description>
+ </property>
+
<property>
<name>hadoop.kms.cache.timeout.ms</name>
+ <value>600000</value>
+ <description>
+ Expiry time for the KMS key version and key metadata cache, in
+ milliseconds. This affects getKeyVersion and getMetadata.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.current.key.cache.timeout.ms</name>
+ <value>30000</value>
+ <description>
+ Expiry time for the KMS current key cache, in milliseconds. This
+ affects getCurrentKey operations.
+ </description>
+ </property>
+
+ <!-- KMS Audit -->
+
+ <property>
+ <name>hadoop.kms.audit.aggregation.window.ms</name>
<value>10000</value>
<description>
+ Duplicate audit log events within the aggregation window (specified in
+ ms) are quashed to reduce log traffic. A single message for aggregated
+ events is printed at the end of the window, along with a count of the
+ number of aggregated events.
</description>
</property>
@@ -43,7 +83,8 @@
<name>hadoop.kms.authentication.type</name>
<value>simple</value>
<description>
- simple or kerberos
+ Authentication type for the KMS. Can be either "simple"
+ or "kerberos".
</description>
</property>
@@ -51,6 +92,7 @@
<name>hadoop.kms.authentication.kerberos.keytab</name>
<value>${user.home}/kms.keytab</value>
<description>
+ Path to the keytab with credentials for the configured Kerberos principal.
</description>
</property>
@@ -58,6 +100,8 @@
<name>hadoop.kms.authentication.kerberos.principal</name>
<value>HTTP/localhost</value>
<description>
+ The Kerberos principal to use for the HTTP endpoint.
+ The principal must start with 'HTTP/' as per the Kerberos HTTP SPNEGO specification.
</description>
</property>
@@ -65,6 +109,7 @@
<name>hadoop.kms.authentication.kerberos.name.rules</name>
<value>DEFAULT</value>
<description>
+ Rules used to resolve Kerberos principal names.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adf0b67a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
index dc55a84..7ff76e5 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
@@ -103,9 +103,17 @@ public class KMSAudit {
private static Logger AUDIT_LOG = LoggerFactory.getLogger(KMS_LOGGER_NAME);
- KMSAudit(long delay) {
+ /**
+ * Create a new KMSAudit.
+ *
+ * @param windowMs Duplicate events within the aggregation window are quashed
+ * to reduce log traffic. A single message for aggregated
+ * events is printed at the end of the window, along with a
+ * count of the number of aggregated events.
+ */
+ KMSAudit(long windowMs) {
cache = CacheBuilder.newBuilder()
- .expireAfterWrite(delay, TimeUnit.MILLISECONDS)
+ .expireAfterWrite(windowMs, TimeUnit.MILLISECONDS)
.removalListener(
new RemovalListener<String, AuditEvent>() {
@Override
@@ -126,7 +134,7 @@ public class KMSAudit {
public void run() {
cache.cleanUp();
}
- }, delay / 10, delay / 10, TimeUnit.MILLISECONDS);
+ }, windowMs / 10, windowMs / 10, TimeUnit.MILLISECONDS);
}
private void logEvent(AuditEvent event) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adf0b67a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index c9b0491..bd61ca7 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -54,8 +54,8 @@ public class KMSConfiguration {
public static final String CURR_KEY_CACHE_TIMEOUT_KEY = CONFIG_PREFIX +
"current.key.cache.timeout.ms";
// Delay for Audit logs that need aggregation
- public static final String KMS_AUDIT_AGGREGATION_DELAY = CONFIG_PREFIX +
- "aggregation.delay.ms";
+ public static final String KMS_AUDIT_AGGREGATION_WINDOW = CONFIG_PREFIX +
+ "audit.aggregation.window.ms";
public static final boolean KEY_CACHE_ENABLE_DEFAULT = true;
// 10 mins
@@ -63,7 +63,7 @@ public class KMSConfiguration {
// 30 secs
public static final long CURR_KEY_CACHE_TIMEOUT_DEFAULT = 30 * 1000;
// 10 secs
- public static final long KMS_AUDIT_AGGREGATION_DELAY_DEFAULT = 10000;
+ public static final long KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT = 10000;
// Property to Enable/Disable per Key authorization
public static final String KEY_AUTHORIZATION_ENABLE = CONFIG_PREFIX +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adf0b67a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
index c9eeb1d..325f8db 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -148,8 +148,8 @@ public class KMSWebApp implements ServletContextListener {
kmsAudit =
new KMSAudit(kmsConf.getLong(
- KMSConfiguration.KMS_AUDIT_AGGREGATION_DELAY,
- KMSConfiguration.KMS_AUDIT_AGGREGATION_DELAY_DEFAULT));
+ KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW,
+ KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT));
// this is required for the the JMXJsonServlet to work properly.
// the JMXJsonServlet is behind the authentication filter,
[09/50] [abbrv] git commit: YARN-2568. Fixed the potential test
failures due to race conditions when RM work-preserving recovery is enabled.
Contributed by Jian He.
Posted by cu...@apache.org.
YARN-2568. Fixed the potential test failures due to race conditions when RM work-preserving recovery is enabled. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fe5c6b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fe5c6b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fe5c6b7
Branch: refs/heads/YARN-1051
Commit: 6fe5c6b746a40019b9a43676c33efec0f971c4b9
Parents: 6434572
Author: Zhijie Shen <zj...@apache.org>
Authored: Thu Sep 18 21:56:56 2014 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Thu Sep 18 21:56:56 2014 -0700
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java | 1 +
.../yarn/server/resourcemanager/TestWorkPreservingRMRestart.java | 1 +
4 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fe5c6b7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 9664b8f..4c03b31 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -2095,7 +2095,7 @@ public class TestRMContainerAllocator {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
-
+ conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0);
conf.setBoolean(MRJobConfig.MR_AM_JOB_NODE_BLACKLISTING_ENABLE, true);
conf.setInt(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, 1);
conf.setInt(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fe5c6b7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 759f2bf..8f717ae 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -397,6 +397,9 @@ Release 2.6.0 - UNRELEASED
YARN-2563. Fixed YarnClient to call getTimeLineDelegationToken only if the
Token is not present. (Zhijie Shen via jianhe)
+ YARN-2568. Fixed the potential test failures due to race conditions when RM
+ work-preserving recovery is enabled. (Jian He via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fe5c6b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
index dfad6d6..ce3086f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
@@ -84,6 +84,7 @@ public class TestAMRMClientOnRMRestart {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
+ conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0);
}
// Test does major 6 steps verification.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fe5c6b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index f1b5f14..5f00f31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -102,6 +102,7 @@ public class TestWorkPreservingRMRestart {
conf.setClass(YarnConfiguration.RM_SCHEDULER, schedulerClass,
ResourceScheduler.class);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
+ conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0);
DefaultMetricsSystem.setMiniClusterMode(true);
}
[36/50] [abbrv] git commit: HDFS-7106. Reconfiguring DataNode volumes
does not release the lock files in removed volumes. (cnauroth via cmccabe)
Posted by cu...@apache.org.
HDFS-7106. Reconfiguring DataNode volumes does not release the lock files in removed volumes. (cnauroth via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/912ad32b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/912ad32b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/912ad32b
Branch: refs/heads/YARN-1051
Commit: 912ad32b03c1e023ab88918bfa8cb356d1851545
Parents: 0a64149
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Sep 22 11:51:31 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Mon Sep 22 11:51:31 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/datanode/DataStorage.java | 7 +++
.../datanode/TestDataNodeHotSwapVolumes.java | 64 +++++++++++++++++++-
3 files changed, 71 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/912ad32b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c4e3e27..9d74b38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -789,6 +789,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7046. HA NN can NPE upon transition to active. (kihwal)
+ HDFS-7106. Reconfiguring DataNode volumes does not release the lock files
+ in removed volumes. (cnauroth via cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/912ad32b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 4383e56..965b655 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -375,6 +375,13 @@ public class DataStorage extends Storage {
StorageDirectory sd = it.next();
if (dataDirs.contains(sd.getRoot())) {
it.remove();
+ try {
+ sd.unlock();
+ } catch (IOException e) {
+ LOG.warn(String.format(
+ "I/O error attempting to unlock storage directory %s.",
+ sd.getRoot()), e);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/912ad32b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index d2b2995..f6e984b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -31,13 +31,19 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -46,13 +52,19 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestDataNodeHotSwapVolumes {
+ private static final Log LOG = LogFactory.getLog(
+ TestDataNodeHotSwapVolumes.class);
private static final int BLOCK_SIZE = 512;
private MiniDFSCluster cluster;
@@ -179,8 +191,10 @@ public class TestDataNodeHotSwapVolumes {
DataNode.ChangedVolumes changedVolumes =dn.parseChangedVolumes();
List<StorageLocation> newVolumes = changedVolumes.newLocations;
assertEquals(2, newVolumes.size());
- assertEquals("/foo/path1", newVolumes.get(0).getFile().getAbsolutePath());
- assertEquals("/foo/path2", newVolumes.get(1).getFile().getAbsolutePath());
+ assertEquals(new File("/foo/path1").getAbsolutePath(),
+ newVolumes.get(0).getFile().getAbsolutePath());
+ assertEquals(new File("/foo/path2").getAbsolutePath(),
+ newVolumes.get(1).getFile().getAbsolutePath());
List<StorageLocation> removedVolumes = changedVolumes.deactivateLocations;
assertEquals(oldLocations.size(), removedVolumes.size());
@@ -371,6 +385,8 @@ public class TestDataNodeHotSwapVolumes {
String newDirs = oldDirs.iterator().next(); // Keep the first volume.
dn.reconfigurePropertyImpl(
DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
+ assertFileLocksReleased(
+ new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));
dn.scheduleAllBlockReport(0);
try {
@@ -409,6 +425,8 @@ public class TestDataNodeHotSwapVolumes {
String newDirs = oldDirs.iterator().next(); // Keep the first volume.
dn.reconfigurePropertyImpl(
DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
+ assertFileLocksReleased(
+ new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));
// Force DataNode to report missing blocks.
dn.scheduleAllBlockReport(0);
@@ -420,4 +438,44 @@ public class TestDataNodeHotSwapVolumes {
// Wait NameNode to replica missing blocks.
DFSTestUtil.waitReplication(fs, testFile, replFactor);
}
-}
\ No newline at end of file
+
+ /**
+ * Asserts that the storage lock file in each given directory has been
+ * released. This method works by trying to acquire the lock file itself. If
+ * locking fails here, then the main code must have failed to release it.
+ *
+ * @param dirs every storage directory to check
+ * @throws IOException if there is an unexpected I/O error
+ */
+ private static void assertFileLocksReleased(Collection<String> dirs)
+ throws IOException {
+ for (String dir: dirs) {
+ StorageLocation sl = StorageLocation.parse(dir);
+ File lockFile = new File(sl.getFile(), Storage.STORAGE_FILE_LOCK);
+ RandomAccessFile raf = null;
+ FileChannel channel = null;
+ FileLock lock = null;
+ try {
+ raf = new RandomAccessFile(lockFile, "rws");
+ channel = raf.getChannel();
+ lock = channel.tryLock();
+ assertNotNull(String.format(
+ "Lock file at %s appears to be held by a different process.",
+ lockFile.getAbsolutePath()), lock);
+ } catch (OverlappingFileLockException e) {
+ fail(String.format("Must release lock file at %s.",
+ lockFile.getAbsolutePath()));
+ } finally {
+ if (lock != null) {
+ try {
+ lock.release();
+ } catch (IOException e) {
+ LOG.warn(String.format("I/O error releasing file lock %s.",
+ lockFile.getAbsolutePath()), e);
+ }
+ }
+ IOUtils.cleanup(null, channel, raf);
+ }
+ }
+ }
+}
[23/50] [abbrv] git commit: HDFS-7095. Archival Storage:
TestStorageMover often fails in Jenkins. Contributed by Jing Zhao.
Posted by cu...@apache.org.
HDFS-7095. Archival Storage: TestStorageMover often fails in Jenkins. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84a0a629
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84a0a629
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84a0a629
Branch: refs/heads/YARN-1051
Commit: 84a0a629d3c63589b9aed7889e2a764538676471
Parents: db890ee
Author: Jing Zhao <ji...@hortonworks.com>
Authored: Sat Sep 20 13:44:53 2014 -0700
Committer: Jing Zhao <ji...@hortonworks.com>
Committed: Sat Sep 20 13:44:53 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../datanode/fsdataset/impl/FsVolumeImpl.java | 4 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 26 ++--
.../hdfs/server/mover/TestStorageMover.java | 150 ++++++++-----------
4 files changed, 80 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84a0a629/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9babe96..9fa06d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -340,6 +340,8 @@ Trunk (Unreleased)
HDFS-7088. Archival Storage: fix TestBalancer and
TestBalancerWithMultipleNameNodes. (szetszwo via jing9)
+ HDFS-7095. TestStorageMover often fails in Jenkins. (jing9)
+
Release 2.6.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84a0a629/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 3952c39..63bc6a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -69,7 +69,7 @@ public class FsVolumeImpl implements FsVolumeSpi {
// Capacity configured. This is useful when we want to
// limit the visible capacity for tests. If negative, then we just
// query from the filesystem.
- protected long configuredCapacity;
+ protected volatile long configuredCapacity;
/**
* Per-volume worker pool that processes new blocks to cache.
@@ -129,7 +129,7 @@ public class FsVolumeImpl implements FsVolumeSpi {
}
}
}
-
+
long getDfsUsed() throws IOException {
long dfsUsed = 0;
synchronized(dataset) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84a0a629/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index f1837ae..858db1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -75,8 +75,10 @@ public class Mover {
private void add(Source source, StorageGroup target) {
sources.put(source);
- targets.put(target);
- getTargetStorages(target.getStorageType()).add(target);
+ if (target != null) {
+ targets.put(target);
+ getTargetStorages(target.getStorageType()).add(target);
+ }
}
private Source getSource(MLocation ml) {
@@ -126,12 +128,11 @@ public class Mover {
for(DatanodeStorageReport r : reports) {
final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo());
for(StorageType t : StorageType.asList()) {
+ final Source source = dn.addSource(t, Long.MAX_VALUE, dispatcher);
final long maxRemaining = getMaxRemaining(r, t);
- if (maxRemaining > 0L) {
- final Source source = dn.addSource(t, Long.MAX_VALUE, dispatcher);
- final StorageGroup target = dn.addTarget(t, maxRemaining);
- storages.add(source, target);
- }
+ final StorageGroup target = maxRemaining > 0L ? dn.addTarget(t,
+ maxRemaining) : null;
+ storages.add(source, target);
}
}
}
@@ -155,7 +156,10 @@ public class Mover {
DBlock newDBlock(Block block, List<MLocation> locations) {
final DBlock db = new DBlock(block);
for(MLocation ml : locations) {
- db.addLocation(storages.getTarget(ml));
+ StorageGroup source = storages.getSource(ml);
+ if (source != null) {
+ db.addLocation(source);
+ }
}
return db;
}
@@ -349,7 +353,7 @@ public class Mover {
for (final StorageType t : diff.existing) {
for (final MLocation ml : locations) {
final Source source = storages.getSource(ml);
- if (ml.storageType == t) {
+ if (ml.storageType == t && source != null) {
// try to schedule one replica move.
if (scheduleMoveReplica(db, source, diff.expected)) {
return true;
@@ -363,7 +367,9 @@ public class Mover {
@VisibleForTesting
boolean scheduleMoveReplica(DBlock db, MLocation ml,
List<StorageType> targetTypes) {
- return scheduleMoveReplica(db, storages.getSource(ml), targetTypes);
+ final Source source = storages.getSource(ml);
+ return source == null ? false : scheduleMoveReplica(db,
+ storages.getSource(ml), targetTypes);
}
boolean scheduleMoveReplica(DBlock db, Source source,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84a0a629/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index ad813cb..e40f142 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.mover;
+import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -26,10 +27,12 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Joiner;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.ReconfigurationException;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -55,6 +58,8 @@ import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
import org.apache.hadoop.io.IOUtils;
import org.apache.log4j.Level;
@@ -599,6 +604,18 @@ public class TestStorageMover {
}
}
+ private void setVolumeFull(DataNode dn, StorageType type) {
+ List<? extends FsVolumeSpi> volumes = dn.getFSDataset().getVolumes();
+ for (int j = 0; j < volumes.size(); ++j) {
+ FsVolumeImpl volume = (FsVolumeImpl) volumes.get(j);
+ if (volume.getStorageType() == type) {
+ LOG.info("setCapacity to 0 for [" + volume.getStorageType() + "]"
+ + volume.getStorageID());
+ volume.setCapacityForTesting(0);
+ }
+ }
+ }
+
/**
* Test DISK is running out of spaces.
*/
@@ -608,76 +625,51 @@ public class TestStorageMover {
final PathPolicyMap pathPolicyMap = new PathPolicyMap(0);
final NamespaceScheme nsScheme = pathPolicyMap.newNamespaceScheme();
- final long diskCapacity = (6 + HdfsConstants.MIN_BLOCKS_FOR_WRITE)
- * BLOCK_SIZE;
- final long archiveCapacity = 100 * BLOCK_SIZE;
- final long[][] capacities = genCapacities(NUM_DATANODES, 1, 1,
- diskCapacity, archiveCapacity);
Configuration conf = new Configuration(DEFAULT_CONF);
final ClusterScheme clusterScheme = new ClusterScheme(conf,
- NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES, 1, 1), capacities);
+ NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES), null);
final MigrationTest test = new MigrationTest(clusterScheme, nsScheme);
try {
test.runBasicTest(false);
- // create hot files with replication 3 until not more spaces.
+ // create 2 hot files with replication 3
final short replication = 3;
- {
- int hotFileCount = 0;
- try {
- for (; ; hotFileCount++) {
- final Path p = new Path(pathPolicyMap.hot, "file" + hotFileCount);
- DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
- waitForAllReplicas(replication, p, test.dfs);
- }
- } catch (IOException e) {
- LOG.info("Expected: hotFileCount=" + hotFileCount, e);
- }
- Assert.assertTrue(hotFileCount >= 1);
- }
-
- // create hot files with replication 1 to use up all remaining spaces.
- {
- int hotFileCount_r1 = 0;
- try {
- for (; ; hotFileCount_r1++) {
- final Path p = new Path(pathPolicyMap.hot, "file_r1_" + hotFileCount_r1);
- DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short) 1, 0L);
- waitForAllReplicas(1, p, test.dfs);
- }
- } catch (IOException e) {
- LOG.info("Expected: hotFileCount_r1=" + hotFileCount_r1, e);
- }
- }
-
- { // test increasing replication. Since DISK is full,
- // new replicas should be stored in ARCHIVE as a fallback storage.
- final Path file0 = new Path(pathPolicyMap.hot, "file0");
- final Replication r = test.getReplication(file0);
- final short newReplication = (short) 5;
- test.dfs.setReplication(file0, newReplication);
- Thread.sleep(10000);
- test.verifyReplication(file0, r.disk, newReplication - r.disk);
- }
-
- { // test creating a cold file and then increase replication
- final Path p = new Path(pathPolicyMap.cold, "foo");
+ for (int i = 0; i < 2; i++) {
+ final Path p = new Path(pathPolicyMap.hot, "file" + i);
DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
- test.verifyReplication(p, 0, replication);
-
- final short newReplication = 5;
- test.dfs.setReplication(p, newReplication);
- Thread.sleep(10000);
- test.verifyReplication(p, 0, newReplication);
+ waitForAllReplicas(replication, p, test.dfs);
}
- { //test move a hot file to warm
- final Path file1 = new Path(pathPolicyMap.hot, "file1");
- test.dfs.rename(file1, pathPolicyMap.warm);
- test.migrate();
- test.verifyFile(new Path(pathPolicyMap.warm, "file1"), WARM.getId());
+ // set all the DISK volume to full
+ for (DataNode dn : test.cluster.getDataNodes()) {
+ setVolumeFull(dn, StorageType.DISK);
+ DataNodeTestUtils.triggerHeartbeat(dn);
}
+
+ // test increasing replication. Since DISK is full,
+ // new replicas should be stored in ARCHIVE as a fallback storage.
+ final Path file0 = new Path(pathPolicyMap.hot, "file0");
+ final Replication r = test.getReplication(file0);
+ final short newReplication = (short) 5;
+ test.dfs.setReplication(file0, newReplication);
+ Thread.sleep(10000);
+ test.verifyReplication(file0, r.disk, newReplication - r.disk);
+
+ // test creating a cold file and then increase replication
+ final Path p = new Path(pathPolicyMap.cold, "foo");
+ DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
+ test.verifyReplication(p, 0, replication);
+
+ test.dfs.setReplication(p, newReplication);
+ Thread.sleep(10000);
+ test.verifyReplication(p, 0, newReplication);
+
+ //test move a hot file to warm
+ final Path file1 = new Path(pathPolicyMap.hot, "file1");
+ test.dfs.rename(file1, pathPolicyMap.warm);
+ test.migrate();
+ test.verifyFile(new Path(pathPolicyMap.warm, "file1"), WARM.getId());
} finally {
test.shutdownCluster();
}
@@ -692,53 +684,31 @@ public class TestStorageMover {
final PathPolicyMap pathPolicyMap = new PathPolicyMap(0);
final NamespaceScheme nsScheme = pathPolicyMap.newNamespaceScheme();
- final long diskCapacity = 100 * BLOCK_SIZE;
- final long archiveCapacity = (6 + HdfsConstants.MIN_BLOCKS_FOR_WRITE)
- * BLOCK_SIZE;
- final long[][] capacities = genCapacities(NUM_DATANODES, 1, 1,
- diskCapacity, archiveCapacity);
final ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF,
- NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES, 1, 1), capacities);
+ NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES), null);
final MigrationTest test = new MigrationTest(clusterScheme, nsScheme);
try {
test.runBasicTest(false);
- // create cold files with replication 3 until not more spaces.
+ // create 2 hot files with replication 3
final short replication = 3;
- {
- int coldFileCount = 0;
- try {
- for (; ; coldFileCount++) {
- final Path p = new Path(pathPolicyMap.cold, "file" + coldFileCount);
- DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
- waitForAllReplicas(replication, p, test.dfs);
- }
- } catch (IOException e) {
- LOG.info("Expected: coldFileCount=" + coldFileCount, e);
- }
- Assert.assertTrue(coldFileCount >= 1);
+ for (int i = 0; i < 2; i++) {
+ final Path p = new Path(pathPolicyMap.cold, "file" + i);
+ DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
+ waitForAllReplicas(replication, p, test.dfs);
}
- // create cold files with replication 1 to use up all remaining spaces.
- {
- int coldFileCount_r1 = 0;
- try {
- for (; ; coldFileCount_r1++) {
- final Path p = new Path(pathPolicyMap.cold, "file_r1_" + coldFileCount_r1);
- DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short) 1, 0L);
- waitForAllReplicas(1, p, test.dfs);
- }
- } catch (IOException e) {
- LOG.info("Expected: coldFileCount_r1=" + coldFileCount_r1, e);
- }
+ // set all the ARCHIVE volume to full
+ for (DataNode dn : test.cluster.getDataNodes()) {
+ setVolumeFull(dn, StorageType.ARCHIVE);
+ DataNodeTestUtils.triggerHeartbeat(dn);
}
{ // test increasing replication but new replicas cannot be created
// since no more ARCHIVE space.
final Path file0 = new Path(pathPolicyMap.cold, "file0");
final Replication r = test.getReplication(file0);
- LOG.info("XXX " + file0 + ": replication=" + r);
Assert.assertEquals(0, r.disk);
final short newReplication = (short) 5;
[25/50] [abbrv] git commit: HDFS-7109. TestDataStorage does not
release file locks between tests. Contributed by Chris Nauroth.
Posted by cu...@apache.org.
HDFS-7109. TestDataStorage does not release file locks between tests. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0795fe37
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0795fe37
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0795fe37
Branch: refs/heads/YARN-1051
Commit: 0795fe3721cf86ed89d5e9058c977a1113247d08
Parents: 47491f9
Author: cnauroth <cn...@apache.org>
Authored: Sat Sep 20 21:01:13 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Sat Sep 20 21:01:13 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java | 1 +
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0795fe37/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index dfdc21f..5a69d9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -908,6 +908,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7107. Avoid Findbugs warning for synchronization on
AbstractNNFailoverProxyProvider#fallbackToSimpleAuth. (cnauroth)
+ HDFS-7109. TestDataStorage does not release file locks between tests.
+ (cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0795fe37/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
index 1d700e3..ed32243 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
@@ -63,6 +63,7 @@ public class TestDataStorage {
@After
public void tearDown() throws IOException {
+ storage.unlockAll();
FileUtil.fullyDelete(TEST_DIR);
}
[39/50] [abbrv] git commit: YARN-1707. Introduce APIs to
add/remove/resize queues in the CapacityScheduler. Contributed by Carlo
Curino and Subru Krishnan
Posted by cu...@apache.org.
YARN-1707. Introduce APIs to add/remove/resize queues in the CapacityScheduler. Contributed by Carlo Curino and Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60d4f29d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60d4f29d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60d4f29d
Branch: refs/heads/YARN-1051
Commit: 60d4f29d96c39a7be0e7f0c2a708ad4c8519be89
Parents: 43efdd3
Author: carlo curino <Carlo Curino>
Authored: Fri Sep 12 16:43:38 2014 -0700
Committer: carlo curino <Carlo Curino>
Committed: Mon Sep 22 16:28:48 2014 -0700
----------------------------------------------------------------------
.../scheduler/AbstractYarnScheduler.java | 25 ++
.../SchedulerDynamicEditException.java | 13 +
.../scheduler/YarnScheduler.java | 42 +++
.../scheduler/capacity/CapacityScheduler.java | 209 ++++++++++++--
.../scheduler/capacity/LeafQueue.java | 28 +-
.../scheduler/capacity/ParentQueue.java | 8 +-
.../scheduler/capacity/PlanQueue.java | 193 +++++++++++++
.../scheduler/capacity/ReservationQueue.java | 98 +++++++
.../scheduler/common/QueueEntitlement.java | 28 ++
.../webapp/dao/CapacitySchedulerQueueInfo.java | 9 +
.../capacity/TestCapacityScheduler.java | 2 +-
.../TestCapacitySchedulerDynamicBehavior.java | 282 +++++++++++++++++++
.../capacity/TestReservationQueue.java | 103 +++++++
13 files changed, 1012 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 0b5447b..8e8d627 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
@@ -220,6 +221,24 @@ public abstract class AbstractYarnScheduler
+ " does not support moving apps between queues");
}
+ public void removeQueue(String queueName) throws YarnException {
+ throw new YarnException(getClass().getSimpleName()
+ + " does not support removing queues");
+ }
+
+ @Override
+ public void addQueue(Queue newQueue) throws YarnException {
+ throw new YarnException(getClass().getSimpleName()
+ + " does not support this operation");
+ }
+
+ @Override
+ public void setEntitlement(String queue, QueueEntitlement entitlement)
+ throws YarnException {
+ throw new YarnException(getClass().getSimpleName()
+ + " does not support this operation");
+ }
+
private void killOrphanContainerOnNode(RMNode node,
NMContainerStatus container) {
if (!container.getContainerState().equals(ContainerState.COMPLETE)) {
@@ -503,4 +522,10 @@ public abstract class AbstractYarnScheduler
public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
return EnumSet.of(SchedulerResourceTypes.MEMORY);
}
+
+ @Override
+ public Set<String> getPlanQueues() throws YarnException {
+ throw new YarnException(getClass().getSimpleName()
+ + " does not support reservations");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerDynamicEditException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerDynamicEditException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerDynamicEditException.java
new file mode 100644
index 0000000..42dc36a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerDynamicEditException.java
@@ -0,0 +1,13 @@
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+public class SchedulerDynamicEditException extends YarnException {
+
+ private static final long serialVersionUID = 7100374511387193257L;
+
+ public SchedulerDynamicEditException(String string) {
+ super(string);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index b6c1018..d1b5275 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.io.IOException;
import java.util.EnumSet;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
@@ -224,6 +226,46 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
void killAllAppsInQueue(String queueName) throws YarnException;
/**
+ * Remove an existing queue. Implementations might limit when a queue could be
+ * removed (e.g., must have zero entitlement, and no applications running, or
+ * must be a leaf, etc..).
+ *
+ * @param queueName name of the queue to remove
+ * @throws YarnException
+ */
+ void removeQueue(String queueName) throws YarnException;
+
+ /**
+ * Add to the scheduler a new Queue. Implementations might limit what type of
+ * queues can be dynamically added (e.g., Queue must be a leaf, must be
+ * attached to existing parent, must have zero entitlement).
+ *
+ * @param newQueue the queue being added.
+ * @throws YarnException
+ */
+ void addQueue(Queue newQueue) throws YarnException;
+
+ /**
+ * This method increase the entitlement for current queue (must respect
+ * invariants, e.g., no overcommit of parents, non negative, etc.).
+ * Entitlement is a general term for weights in FairScheduler, capacity for
+ * the CapacityScheduler, etc.
+ *
+ * @param queue the queue for which we change entitlement
+ * @param entitlement the new entitlement for the queue (capacity,
+ * maxCapacity, etc..)
+ * @throws YarnException
+ */
+ void setEntitlement(String queue, QueueEntitlement entitlement)
+ throws YarnException;
+
+ /**
+ * Gets the list of names for queues managed by the Reservation System
+ * @return the list of queues which support reservations
+ */
+ public Set<String> getPlanQueues() throws YarnException;
+
+ /**
* Return a collection of the resource types that are considered when
* scheduling
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index bdfc819..522b975 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -24,6 +24,8 @@ import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -90,6 +92,11 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+
@LimitedPrivate("yarn")
@Evolving
@SuppressWarnings("unchecked")
@@ -473,9 +480,12 @@ public class CapacityScheduler extends
private void validateExistingQueues(
Map<String, CSQueue> queues, Map<String, CSQueue> newQueues)
throws IOException {
- for (String queue : queues.keySet()) {
- if (!newQueues.containsKey(queue)) {
- throw new IOException(queue + " cannot be found during refresh!");
+ // check that all static queues are included in the newQueues list
+ for (Map.Entry<String, CSQueue> e : queues.entrySet()) {
+ if (!(e.getValue() instanceof ReservationQueue)) {
+ if (!newQueues.containsKey(e.getKey())) {
+ throw new IOException(e.getKey() + " cannot be found during refresh!");
+ }
}
}
}
@@ -507,26 +517,42 @@ public class CapacityScheduler extends
Map<String, CSQueue> oldQueues,
QueueHook hook) throws IOException {
CSQueue queue;
+ String fullQueueName =
+ (parent == null) ? queueName
+ : (parent.getQueuePath() + "." + queueName);
String[] childQueueNames =
- conf.getQueues((parent == null) ?
- queueName : (parent.getQueuePath()+"."+queueName));
+ conf.getQueues(fullQueueName);
+ boolean isReservableQueue = conf.isReservableQueue(fullQueueName);
if (childQueueNames == null || childQueueNames.length == 0) {
if (null == parent) {
throw new IllegalStateException(
"Queue configuration missing child queue names for " + queueName);
}
- queue =
- new LeafQueue(csContext, queueName, parent,oldQueues.get(queueName));
-
- // Used only for unit tests
- queue = hook.hook(queue);
+ // Check if the queue will be dynamically managed by the Reservation
+ // system
+ if (isReservableQueue) {
+ queue =
+ new PlanQueue(csContext, queueName, parent,
+ oldQueues.get(queueName));
+ } else {
+ queue =
+ new LeafQueue(csContext, queueName, parent,
+ oldQueues.get(queueName));
+
+ // Used only for unit tests
+ queue = hook.hook(queue);
+ }
} else {
+ if (isReservableQueue) {
+ throw new IllegalStateException(
+ "Only Leaf Queues can be reservable for " + queueName);
+ }
ParentQueue parentQueue =
new ParentQueue(csContext, queueName, parent,oldQueues.get(queueName));
// Used only for unit tests
queue = hook.hook(parentQueue);
-
+
List<CSQueue> childQueues = new ArrayList<CSQueue>();
for (String childQueueName : childQueueNames) {
CSQueue childQueue =
@@ -548,7 +574,7 @@ public class CapacityScheduler extends
return queue;
}
- synchronized CSQueue getQueue(String queueName) {
+ public synchronized CSQueue getQueue(String queueName) {
if (queueName == null) {
return null;
}
@@ -716,7 +742,7 @@ public class CapacityScheduler extends
ApplicationAttemptId applicationAttemptId,
RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) {
LOG.info("Application Attempt " + applicationAttemptId + " is done." +
- " finalState=" + rmAppAttemptFinalState);
+ " finalState=" + rmAppAttemptFinalState);
FiCaSchedulerApp attempt = getApplicationAttempt(applicationAttemptId);
SchedulerApplication<FiCaSchedulerApp> application =
@@ -995,9 +1021,16 @@ public class CapacityScheduler extends
case APP_ADDED:
{
AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event;
- addApplication(appAddedEvent.getApplicationId(),
- appAddedEvent.getQueue(),
- appAddedEvent.getUser(), appAddedEvent.getIsAppRecovering());
+ String queueName =
+ resolveReservationQueueName(appAddedEvent.getQueue(),
+ appAddedEvent.getApplicationId(),
+ appAddedEvent.getReservationID());
+ if (queueName != null) {
+ addApplication(appAddedEvent.getApplicationId(),
+ queueName,
+ appAddedEvent.getUser(),
+ appAddedEvent.getIsAppRecovering());
+ }
}
break;
case APP_REMOVED:
@@ -1230,6 +1263,123 @@ public class CapacityScheduler extends
}
}
+ private synchronized String resolveReservationQueueName(String queueName,
+ ApplicationId applicationId, ReservationId reservationID) {
+ CSQueue queue = getQueue(queueName);
+ // Check if the queue is a plan queue
+ if ((queue == null) || !(queue instanceof PlanQueue)) {
+ return queueName;
+ }
+ if (reservationID != null) {
+ String resQName = reservationID.toString();
+ queue = getQueue(resQName);
+ if (queue == null) {
+ String message =
+ "Application "
+ + applicationId
+ + " submitted to a reservation which is not yet currently active: "
+ + resQName;
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMAppRejectedEvent(applicationId, message));
+ return null;
+ }
+ // use the reservation queue to run the app
+ queueName = resQName;
+ } else {
+ // use the default child queue of the plan for unreserved apps
+ queueName = queueName + PlanQueue.DEFAULT_QUEUE_SUFFIX;
+ }
+ return queueName;
+ }
+
+ @Override
+ public synchronized void removeQueue(String queueName)
+ throws SchedulerDynamicEditException {
+ LOG.info("Removing queue: " + queueName);
+ CSQueue q = this.getQueue(queueName);
+ if (!(q instanceof ReservationQueue)) {
+ throw new SchedulerDynamicEditException("The queue that we are asked "
+ + "to remove (" + queueName + ") is not a ReservationQueue");
+ }
+ ReservationQueue disposableLeafQueue = (ReservationQueue) q;
+ // at this point we should have no more apps
+ if (disposableLeafQueue.getNumApplications() > 0) {
+ throw new SchedulerDynamicEditException("The queue " + queueName
+ + " is not empty " + disposableLeafQueue.getApplications().size()
+ + " active apps " + disposableLeafQueue.pendingApplications.size()
+ + " pending apps");
+ }
+
+ ((PlanQueue) disposableLeafQueue.getParent()).removeChildQueue(q);
+ this.queues.remove(queueName);
+ LOG.info("Removal of ReservationQueue " + queueName + " has succeeded");
+ }
+
+ @Override
+ public synchronized void addQueue(Queue queue)
+ throws SchedulerDynamicEditException {
+
+ if (!(queue instanceof ReservationQueue)) {
+ throw new SchedulerDynamicEditException("Queue " + queue.getQueueName()
+ + " is not a ReservationQueue");
+ }
+
+ ReservationQueue newQueue = (ReservationQueue) queue;
+
+ if (newQueue.getParent() == null
+ || !(newQueue.getParent() instanceof PlanQueue)) {
+ throw new SchedulerDynamicEditException("ParentQueue for "
+ + newQueue.getQueueName()
+ + " is not properly set (should be set and be a PlanQueue)");
+ }
+
+ PlanQueue parentPlan = (PlanQueue) newQueue.getParent();
+ String queuename = newQueue.getQueueName();
+ parentPlan.addChildQueue(newQueue);
+ this.queues.put(queuename, newQueue);
+ LOG.info("Creation of ReservationQueue " + newQueue + " succeeded");
+ }
+
+ @Override
+ public synchronized void setEntitlement(String inQueue,
+ QueueEntitlement entitlement) throws SchedulerDynamicEditException,
+ YarnException {
+ LeafQueue queue = getAndCheckLeafQueue(inQueue);
+ ParentQueue parent = (ParentQueue) queue.getParent();
+
+ if (!(queue instanceof ReservationQueue)) {
+ throw new SchedulerDynamicEditException("Entitlement can not be"
+ + " modified dynamically since queue " + inQueue
+ + " is not a ReservationQueue");
+ }
+
+ if (!(parent instanceof PlanQueue)) {
+ throw new SchedulerDynamicEditException("The parent of ReservationQueue "
+ + inQueue + " must be an PlanQueue");
+ }
+
+ ReservationQueue newQueue = (ReservationQueue) queue;
+
+ float sumChilds = ((PlanQueue) parent).sumOfChildCapacities();
+ float newChildCap = sumChilds - queue.getCapacity() + entitlement.getCapacity();
+
+ if (newChildCap >= 0 && newChildCap < 1.0f + CSQueueUtils.EPSILON) {
+ // note: epsilon checks here are not ok, as the epsilons might accumulate
+ // and become a problem in aggregate
+ if (Math.abs(entitlement.getCapacity() - queue.getCapacity()) == 0
+ && Math.abs(entitlement.getMaxCapacity() - queue.getMaximumCapacity()) == 0) {
+ return;
+ }
+ newQueue.setEntitlement(entitlement);
+ } else {
+ throw new SchedulerDynamicEditException(
+ "Sum of child queues would exceed 100% for PlanQueue: "
+ + parent.getQueueName());
+ }
+ LOG.info("Set entitlement for ReservationQueue " + inQueue + " to "
+ + queue.getCapacity() + " request was (" + entitlement.getCapacity() + ")");
+ }
+
@Override
public synchronized String moveApplication(ApplicationId appId,
String targetQueueName) throws YarnException {
@@ -1237,11 +1387,12 @@ public class CapacityScheduler extends
getApplicationAttempt(ApplicationAttemptId.newInstance(appId, 0));
String sourceQueueName = app.getQueue().getQueueName();
LeafQueue source = getAndCheckLeafQueue(sourceQueueName);
- LeafQueue dest = getAndCheckLeafQueue(targetQueueName);
+ String destQueueName = handleMoveToPlanQueue(targetQueueName);
+ LeafQueue dest = getAndCheckLeafQueue(destQueueName);
// Validation check - ACLs, submission limits for user & queue
String user = app.getUser();
try {
- dest.submitApplication(appId, user, targetQueueName);
+ dest.submitApplication(appId, user, destQueueName);
} catch (AccessControlException e) {
throw new YarnException(e);
}
@@ -1260,7 +1411,7 @@ public class CapacityScheduler extends
dest.submitApplicationAttempt(app, user);
applications.get(appId).setQueue(dest);
LOG.info("App: " + app.getApplicationId() + " successfully moved from "
- + sourceQueueName + " to: " + targetQueueName);
+ + sourceQueueName + " to: " + destQueueName);
return targetQueueName;
}
@@ -1295,4 +1446,24 @@ public class CapacityScheduler extends
return EnumSet
.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU);
}
+
+ private String handleMoveToPlanQueue(String targetQueueName) {
+ CSQueue dest = getQueue(targetQueueName);
+ if (dest != null && dest instanceof PlanQueue) {
+ // use the default child reservation queue of the plan
+ targetQueueName = targetQueueName + PlanQueue.DEFAULT_QUEUE_SUFFIX;
+ }
+ return targetQueueName;
+ }
+
+ @Override
+ public Set<String> getPlanQueues() {
+ Set<String> ret = new HashSet<String>();
+ for (Map.Entry<String, CSQueue> l : queues.entrySet()) {
+ if (l.getValue() instanceof PlanQueue) {
+ ret.add(l.getKey());
+ }
+ }
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 5c93c5f..b5268ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -85,8 +85,8 @@ public class LeafQueue implements CSQueue {
private int userLimit;
private float userLimitFactor;
- private int maxApplications;
- private int maxApplicationsPerUser;
+ protected int maxApplications;
+ protected int maxApplicationsPerUser;
private float maxAMResourcePerQueuePercent;
private int maxActiveApplications; // Based on absolute max capacity
@@ -150,8 +150,7 @@ public class LeafQueue implements CSQueue {
Resources.subtract(maximumAllocation, minimumAllocation),
maximumAllocation);
- float capacity =
- (float)cs.getConfiguration().getCapacity(getQueuePath()) / 100;
+ float capacity = getCapacityFromConf();
float absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
float maximumCapacity =
@@ -217,6 +216,11 @@ public class LeafQueue implements CSQueue {
this.activeApplications = new TreeSet<FiCaSchedulerApp>(applicationComparator);
}
+ // externalizing in method, to allow overriding
+ protected float getCapacityFromConf() {
+ return (float)scheduler.getConfiguration().getCapacity(getQueuePath()) / 100;
+ }
+
private synchronized void setupQueueConfigs(
Resource clusterResource,
float capacity, float absoluteCapacity,
@@ -475,7 +479,7 @@ public class LeafQueue implements CSQueue {
* Set user limit factor - used only for testing.
* @param userLimitFactor new user limit factor
*/
- synchronized void setUserLimitFactor(int userLimitFactor) {
+ synchronized void setUserLimitFactor(float userLimitFactor) {
this.userLimitFactor = userLimitFactor;
}
@@ -817,7 +821,7 @@ public class LeafQueue implements CSQueue {
getApplication(reservedContainer.getApplicationAttemptId());
synchronized (application) {
return assignReservedContainer(application, node, reservedContainer,
- clusterResource);
+ clusterResource);
}
}
@@ -1661,4 +1665,16 @@ public class LeafQueue implements CSQueue {
getParent().detachContainer(clusterResource, application, rmContainer);
}
}
+
+ public void setCapacity(float capacity) {
+ this.capacity = capacity;
+ }
+
+ public void setAbsoluteCapacity(float absoluteCapacity) {
+ this.absoluteCapacity = absoluteCapacity;
+ }
+
+ public void setMaxApplications(int maxApplications) {
+ this.maxApplications = maxApplications;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 8c654b7..7e36b5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -75,7 +75,7 @@ public class ParentQueue implements CSQueue {
private float usedCapacity = 0.0f;
- private final Set<CSQueue> childQueues;
+ protected final Set<CSQueue> childQueues;
private final Comparator<CSQueue> queueComparator;
private Resource usedResources = Resources.createResource(0, 0);
@@ -156,7 +156,7 @@ public class ParentQueue implements CSQueue {
", fullname=" + getQueuePath());
}
- private synchronized void setupQueueConfigs(
+ protected synchronized void setupQueueConfigs(
Resource clusterResource,
float capacity, float absoluteCapacity,
float maximumCapacity, float absoluteMaxCapacity,
@@ -824,4 +824,8 @@ public class ParentQueue implements CSQueue {
}
}
}
+
+ public Map<QueueACL, AccessControlList> getACLs() {
+ return acls;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
new file mode 100644
index 0000000..4ada778
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -0,0 +1,193 @@
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This represents a dynamic queue managed by the {@link ReservationSystem}.
+ * From the user perspective this is equivalent to a LeafQueue that respect
+ * reservations, but functionality wise is a sub-class of ParentQueue
+ *
+ */
+public class PlanQueue extends ParentQueue {
+
+ public static final String DEFAULT_QUEUE_SUFFIX = "-default";
+
+ private static final Logger LOG = LoggerFactory.getLogger(PlanQueue.class);
+
+ private int maxAppsForReservation;
+ private int maxAppsPerUserForReservation;
+ private int userLimit;
+ private float userLimitFactor;
+ protected CapacitySchedulerContext schedulerContext;
+ private boolean showReservationsAsQueues;
+
+ public PlanQueue(CapacitySchedulerContext cs, String queueName,
+ CSQueue parent, CSQueue old) {
+ super(cs, queueName, parent, old);
+
+ this.schedulerContext = cs;
+ // Set the reservation queue attributes for the Plan
+ CapacitySchedulerConfiguration conf = cs.getConfiguration();
+ String queuePath = super.getQueuePath();
+ int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePath);
+ showReservationsAsQueues = conf.getShowReservationAsQueues(queuePath);
+ if (maxAppsForReservation < 0) {
+ maxAppsForReservation =
+ (int) (CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * super
+ .getAbsoluteCapacity());
+ }
+ int userLimit = conf.getUserLimit(queuePath);
+ float userLimitFactor = conf.getUserLimitFactor(queuePath);
+ int maxAppsPerUserForReservation =
+ (int) (maxAppsForReservation * (userLimit / 100.0f) * userLimitFactor);
+ updateQuotas(userLimit, userLimitFactor, maxAppsForReservation,
+ maxAppsPerUserForReservation);
+
+ StringBuffer queueInfo = new StringBuffer();
+ queueInfo.append("Created Plan Queue: ").append(queueName)
+ .append("\nwith capacity: [").append(super.getCapacity())
+ .append("]\nwith max capacity: [").append(super.getMaximumCapacity())
+ .append("\nwith max reservation apps: [").append(maxAppsForReservation)
+ .append("]\nwith max reservation apps per user: [")
+ .append(maxAppsPerUserForReservation).append("]\nwith user limit: [")
+ .append(userLimit).append("]\nwith user limit factor: [")
+ .append(userLimitFactor).append("].");
+ LOG.info(queueInfo.toString());
+ }
+
+ @Override
+ public synchronized void reinitialize(CSQueue newlyParsedQueue,
+ Resource clusterResource) throws IOException {
+ // Sanity check
+ if (!(newlyParsedQueue instanceof PlanQueue)
+ || !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
+ throw new IOException("Trying to reinitialize " + getQueuePath()
+ + " from " + newlyParsedQueue.getQueuePath());
+ }
+
+ PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue;
+
+ if (newlyParsedParentQueue.getChildQueues().size() > 0) {
+ throw new IOException(
+ "Reservable Queue should not have sub-queues in the"
+ + "configuration");
+ }
+
+ // Set new configs
+ setupQueueConfigs(clusterResource, newlyParsedParentQueue.getCapacity(),
+ newlyParsedParentQueue.getAbsoluteCapacity(),
+ newlyParsedParentQueue.getMaximumCapacity(),
+ newlyParsedParentQueue.getAbsoluteMaximumCapacity(),
+ newlyParsedParentQueue.getState(), newlyParsedParentQueue.getACLs());
+
+ updateQuotas(newlyParsedParentQueue.userLimit,
+ newlyParsedParentQueue.userLimitFactor,
+ newlyParsedParentQueue.maxAppsForReservation,
+ newlyParsedParentQueue.maxAppsPerUserForReservation);
+
+ // run reinitialize on each existing queue, to trigger absolute cap
+ // recomputations
+ for (CSQueue res : this.getChildQueues()) {
+ res.reinitialize(res, clusterResource);
+ }
+ showReservationsAsQueues = newlyParsedParentQueue.showReservationsAsQueues;
+ }
+
+ synchronized void addChildQueue(CSQueue newQueue)
+ throws SchedulerDynamicEditException {
+ if (newQueue.getCapacity() > 0) {
+ throw new SchedulerDynamicEditException("Queue " + newQueue
+ + " being added has non zero capacity.");
+ }
+ boolean added = this.childQueues.add(newQueue);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("updateChildQueues (action: add queue): " + added + " "
+ + getChildQueuesToPrint());
+ }
+ }
+
+ synchronized void removeChildQueue(CSQueue remQueue)
+ throws SchedulerDynamicEditException {
+ if (remQueue.getCapacity() > 0) {
+ throw new SchedulerDynamicEditException("Queue " + remQueue
+ + " being removed has non zero capacity.");
+ }
+ Iterator<CSQueue> qiter = childQueues.iterator();
+ while (qiter.hasNext()) {
+ CSQueue cs = qiter.next();
+ if (cs.equals(remQueue)) {
+ qiter.remove();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removed child queue: {}", cs.getQueueName());
+ }
+ }
+ }
+ }
+
+ protected synchronized float sumOfChildCapacities() {
+ float ret = 0;
+ for (CSQueue l : childQueues) {
+ ret += l.getCapacity();
+ }
+ return ret;
+ }
+
+ private void updateQuotas(int userLimit, float userLimitFactor,
+ int maxAppsForReservation, int maxAppsPerUserForReservation) {
+ this.userLimit = userLimit;
+ this.userLimitFactor = userLimitFactor;
+ this.maxAppsForReservation = maxAppsForReservation;
+ this.maxAppsPerUserForReservation = maxAppsPerUserForReservation;
+ }
+
+ /**
+ * Number of maximum applications for each of the reservations in this Plan.
+ *
+ * @return maxAppsForreservation
+ */
+ public int getMaxApplicationsForReservations() {
+ return maxAppsForReservation;
+ }
+
+ /**
+ * Number of maximum applications per user for each of the reservations in
+ * this Plan.
+ *
+ * @return maxAppsPerUserForreservation
+ */
+ public int getMaxApplicationsPerUserForReservation() {
+ return maxAppsPerUserForReservation;
+ }
+
+ /**
+ * User limit value for each of the reservations in this Plan.
+ *
+ * @return userLimit
+ */
+ public int getUserLimitForReservation() {
+ return userLimit;
+ }
+
+ /**
+ * User limit factor value for each of the reservations in this Plan.
+ *
+ * @return userLimitFactor
+ */
+ public float getUserLimitFactor() {
+ return userLimitFactor;
+ }
+
+ /**
+ * Determine whether to hide/show the ReservationQueues
+ */
+ public boolean showReservationsAsQueues() {
+ return showReservationsAsQueues;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
new file mode 100644
index 0000000..48733fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
@@ -0,0 +1,98 @@
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This represents a dynamic {@link LeafQueue} managed by the
+ * {@link ReservationSystem}
+ *
+ */
+public class ReservationQueue extends LeafQueue {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(ReservationQueue.class);
+
+ private PlanQueue parent;
+
+ private int maxSystemApps;
+
+ public ReservationQueue(CapacitySchedulerContext cs, String queueName,
+ PlanQueue parent) {
+ super(cs, queueName, parent, null);
+ maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
+ // the following parameters are common to all reservation in the plan
+ updateQuotas(parent.getUserLimitForReservation(),
+ parent.getUserLimitFactor(),
+ parent.getMaxApplicationsForReservations(),
+ parent.getMaxApplicationsPerUserForReservation());
+ this.parent = parent;
+ }
+
+ @Override
+ public synchronized void reinitialize(CSQueue newlyParsedQueue,
+ Resource clusterResource) throws IOException {
+ // Sanity check
+ if (!(newlyParsedQueue instanceof ReservationQueue)
+ || !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
+ throw new IOException("Trying to reinitialize " + getQueuePath()
+ + " from " + newlyParsedQueue.getQueuePath());
+ }
+ CSQueueUtils.updateQueueStatistics(
+ parent.schedulerContext.getResourceCalculator(), newlyParsedQueue,
+ parent, parent.schedulerContext.getClusterResource(),
+ parent.schedulerContext.getMinimumResourceCapability());
+ super.reinitialize(newlyParsedQueue, clusterResource);
+ updateQuotas(parent.getUserLimitForReservation(),
+ parent.getUserLimitFactor(),
+ parent.getMaxApplicationsForReservations(),
+ parent.getMaxApplicationsPerUserForReservation());
+ }
+
+ /**
+ * This methods to change capacity for a queue and adjusts its
+ * absoluteCapacity
+ *
+ * @param entitlement the new entitlement for the queue (capacity,
+ * maxCapacity, etc..)
+ * @throws SchedulerDynamicEditException
+ */
+ public synchronized void setEntitlement(QueueEntitlement entitlement)
+ throws SchedulerDynamicEditException {
+ float capacity = entitlement.getCapacity();
+ if (capacity < 0 || capacity > 1.0f) {
+ throw new SchedulerDynamicEditException(
+ "Capacity demand is not in the [0,1] range: " + capacity);
+ }
+ setCapacity(capacity);
+ setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity());
+ setMaxApplications((int) (maxSystemApps * getAbsoluteCapacity()));
+ // note: we currently set maxCapacity to capacity
+ // this might be revised later
+ setMaxCapacity(entitlement.getMaxCapacity());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("successfully changed to " + capacity + " for queue "
+ + this.getQueueName());
+ }
+ }
+
+ private void updateQuotas(int userLimit, float userLimitFactor,
+ int maxAppsForReservation, int maxAppsPerUserForReservation) {
+ setUserLimit(userLimit);
+ setUserLimitFactor(userLimitFactor);
+ setMaxApplications(maxAppsForReservation);
+ maxApplicationsPerUser = maxAppsPerUserForReservation;
+ }
+
+ // used by the super constructor, we initialize to zero
+ protected float getCapacityFromConf() {
+ return 0f;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/QueueEntitlement.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/QueueEntitlement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/QueueEntitlement.java
new file mode 100644
index 0000000..a348e13
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/QueueEntitlement.java
@@ -0,0 +1,28 @@
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.common;
+
+public class QueueEntitlement {
+
+ private float capacity;
+ private float maxCapacity;
+
+ public QueueEntitlement(float capacity, float maxCapacity){
+ this.setCapacity(capacity);
+ this.maxCapacity = maxCapacity;
+ }
+
+ public float getMaxCapacity() {
+ return maxCapacity;
+ }
+
+ public void setMaxCapacity(float maxCapacity) {
+ this.maxCapacity = maxCapacity;
+ }
+
+ public float getCapacity() {
+ return capacity;
+ }
+
+ public void setCapacity(float capacity) {
+ this.capacity = capacity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index ac16ce0..0a8faad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@@ -48,6 +49,7 @@ public class CapacitySchedulerQueueInfo {
protected QueueState state;
protected CapacitySchedulerQueueInfoList queues;
protected ResourceInfo resourcesUsed;
+ private boolean hideReservationQueues = true;
CapacitySchedulerQueueInfo() {
};
@@ -69,6 +71,10 @@ public class CapacitySchedulerQueueInfo {
queueName = q.getQueueName();
state = q.getState();
resourcesUsed = new ResourceInfo(q.getUsedResources());
+ if(q instanceof PlanQueue &&
+ ((PlanQueue)q).showReservationsAsQueues()) {
+ hideReservationQueues = false;
+ }
}
public float getCapacity() {
@@ -112,6 +118,9 @@ public class CapacitySchedulerQueueInfo {
}
public CapacitySchedulerQueueInfoList getQueues() {
+ if(hideReservationQueues) {
+ return new CapacitySchedulerQueueInfoList();
+ }
return this.queues;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index e029749..05bb427 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -408,7 +408,7 @@ public class TestCapacityScheduler {
cs.stop();
}
- private void checkQueueCapacities(CapacityScheduler cs,
+ void checkQueueCapacities(CapacityScheduler cs,
float capacityA, float capacityB) {
CSQueue rootQueue = cs.getRootQueue();
CSQueue queueA = findQueue(rootQueue, A);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
new file mode 100644
index 0000000..aecbfa8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
@@ -0,0 +1,282 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCapacitySchedulerDynamicBehavior {
+ private static final Log LOG = LogFactory
+ .getLog(TestCapacitySchedulerDynamicBehavior.class);
+ private static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+ private static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+ private static final String B1 = B + ".b1";
+ private static final String B2 = B + ".b2";
+ private static final String B3 = B + ".b3";
+ private static float A_CAPACITY = 10.5f;
+ private static float B_CAPACITY = 89.5f;
+ private static float A1_CAPACITY = 30;
+ private static float A2_CAPACITY = 70;
+ private static float B1_CAPACITY = 79.2f;
+ private static float B2_CAPACITY = 0.8f;
+ private static float B3_CAPACITY = 20;
+
+ private final TestCapacityScheduler tcs = new TestCapacityScheduler();
+
+ private int GB = 1024;
+
+ private MockRM rm;
+
+ @Before
+ public void setUp() {
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ setupPlanQueueConfiguration(conf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.RM_RESERVATIONS_ENABLE, false);
+ rm = new MockRM(conf);
+ rm.start();
+ }
+
+ @Test
+ public void testRefreshQueuesWithReservations() throws Exception {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ // Test add one reservation dynamically and manually modify capacity
+ ReservationQueue a1 =
+ new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+ cs.addQueue(a1);
+ a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
+
+ // Test add another reservation queue and use setEntitlement to modify
+ // capacity
+ ReservationQueue a2 =
+ new ReservationQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
+ cs.addQueue(a2);
+ cs.setEntitlement("a2", new QueueEntitlement(A2_CAPACITY / 100, 1.0f));
+
+ // Verify all allocations match
+ tcs.checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
+
+ // Reinitialize and verify all dynamic queued survived
+ CapacitySchedulerConfiguration conf = cs.getConfiguration();
+ conf.setCapacity(A, 80f);
+ conf.setCapacity(B, 20f);
+ cs.reinitialize(conf, rm.getRMContext());
+
+ tcs.checkQueueCapacities(cs, 80f, 20f);
+ }
+
+ @Test
+ public void testAddQueueFailCases() throws Exception {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ try {
+ // Test invalid addition (adding non-zero size queue)
+ ReservationQueue a1 =
+ new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+ a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
+ cs.addQueue(a1);
+ fail();
+ } catch (Exception e) {
+ // expected
+ }
+
+ // Test add one reservation dynamically and manually modify capacity
+ ReservationQueue a1 =
+ new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+ cs.addQueue(a1);
+ a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
+
+ // Test add another reservation queue and use setEntitlement to modify
+ // capacity
+ ReservationQueue a2 =
+ new ReservationQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
+
+ cs.addQueue(a2);
+
+ try {
+ // Test invalid entitlement (sum of queues exceed 100%)
+ cs.setEntitlement("a2", new QueueEntitlement(A2_CAPACITY / 100 + 0.1f,
+ 1.0f));
+ fail();
+ } catch (Exception e) {
+ // expected
+ }
+
+ cs.setEntitlement("a2", new QueueEntitlement(A2_CAPACITY / 100, 1.0f));
+
+ // Verify all allocations match
+ tcs.checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
+
+ cs.stop();
+ }
+
+ @Test
+ public void testRemoveQueue() throws Exception {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ // Test add one reservation dynamically and manually modify capacity
+ ReservationQueue a1 =
+ new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+ cs.addQueue(a1);
+ a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
+
+ // submit an app
+ RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1");
+ // check preconditions
+ List<ApplicationAttemptId> appsInA1 = cs.getAppsInQueue("a1");
+ assertEquals(1, appsInA1.size());
+ try {
+ cs.removeQueue("a1");
+ fail();
+ } catch (SchedulerDynamicEditException s) {
+ // expected a1 contains applications
+ }
+ // clear queue by killling all apps
+ cs.killAllAppsInQueue("a1");
+ // wait for events of move to propagate
+ rm.waitForState(app.getApplicationId(), RMAppState.KILLED);
+
+ try {
+ cs.removeQueue("a1");
+ fail();
+ } catch (SchedulerDynamicEditException s) {
+ // expected a1 is not zero capacity
+ }
+ // set capacity to zero
+ cs.setEntitlement("a1", new QueueEntitlement(0f, 0f));
+ cs.removeQueue("a1");
+
+ assertTrue(cs.getQueue("a1") == null);
+
+ rm.stop();
+ }
+
+ @Test
+ public void testMoveAppToPlanQueue() throws Exception {
+ CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
+
+ // submit an app
+ RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "b1");
+ ApplicationAttemptId appAttemptId =
+ rm.getApplicationReport(app.getApplicationId())
+ .getCurrentApplicationAttemptId();
+
+ // check preconditions
+ List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
+ assertEquals(1, appsInB1.size());
+
+ List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
+ assertEquals(1, appsInB.size());
+ assertTrue(appsInB.contains(appAttemptId));
+
+ List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
+ assertTrue(appsInA.isEmpty());
+
+ String queue =
+ scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue()
+ .getQueueName();
+ Assert.assertTrue(queue.equals("b1"));
+
+ List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
+ assertTrue(appsInRoot.contains(appAttemptId));
+ assertEquals(1, appsInRoot.size());
+
+ // create the default reservation queue
+ String defQName = "a" + PlanQueue.DEFAULT_QUEUE_SUFFIX;
+ ReservationQueue defQ =
+ new ReservationQueue(scheduler, defQName,
+ (PlanQueue) scheduler.getQueue("a"));
+ scheduler.addQueue(defQ);
+ defQ.setEntitlement(new QueueEntitlement(1f, 1f));
+
+ List<ApplicationAttemptId> appsInDefQ = scheduler.getAppsInQueue(defQName);
+ assertTrue(appsInDefQ.isEmpty());
+
+ // now move the app to plan queue
+ scheduler.moveApplication(app.getApplicationId(), "a");
+
+ // check postconditions
+ appsInDefQ = scheduler.getAppsInQueue(defQName);
+ assertEquals(1, appsInDefQ.size());
+ queue =
+ scheduler.getApplicationAttempt(appsInDefQ.get(0)).getQueue()
+ .getQueueName();
+ Assert.assertTrue(queue.equals(defQName));
+
+ appsInA = scheduler.getAppsInQueue("a");
+ assertTrue(appsInA.contains(appAttemptId));
+ assertEquals(1, appsInA.size());
+
+ appsInRoot = scheduler.getAppsInQueue("root");
+ assertTrue(appsInRoot.contains(appAttemptId));
+ assertEquals(1, appsInRoot.size());
+
+ appsInB1 = scheduler.getAppsInQueue("b1");
+ assertTrue(appsInB1.isEmpty());
+
+ appsInB = scheduler.getAppsInQueue("b");
+ assertTrue(appsInB.isEmpty());
+
+ rm.stop();
+ }
+
+ private void setupPlanQueueConfiguration(CapacitySchedulerConfiguration conf) {
+
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[] { "a", "b" });
+
+ conf.setCapacity(A, A_CAPACITY);
+ conf.setCapacity(B, B_CAPACITY);
+
+ // Define 2nd-level queues
+ conf.setQueues(B, new String[] { "b1", "b2", "b3" });
+ conf.setCapacity(B1, B1_CAPACITY);
+ conf.setUserLimitFactor(B1, 100.0f);
+ conf.setCapacity(B2, B2_CAPACITY);
+ conf.setUserLimitFactor(B2, 100.0f);
+ conf.setCapacity(B3, B3_CAPACITY);
+ conf.setUserLimitFactor(B3, 100.0f);
+
+ conf.setReservableQueue(A, true);
+ conf.setReservationWindow(A, 86400 * 1000);
+ conf.setAverageCapacity(A, 1.0f);
+
+ LOG.info("Setup a as a plan queue");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d4f29d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
new file mode 100644
index 0000000..c53b7a9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestReservationQueue {
+
+ CapacitySchedulerConfiguration csConf;
+ CapacitySchedulerContext csContext;
+ final static int GB = 1024;
+ private final ResourceCalculator resourceCalculator =
+ new DefaultResourceCalculator();
+ ReservationQueue reservationQueue;
+
+ @Before
+ public void setup() {
+
+ // setup a context / conf
+ csConf = new CapacitySchedulerConfiguration();
+ YarnConfiguration conf = new YarnConfiguration();
+ csContext = mock(CapacitySchedulerContext.class);
+ when(csContext.getConfiguration()).thenReturn(csConf);
+ when(csContext.getConf()).thenReturn(conf);
+ when(csContext.getMinimumResourceCapability()).thenReturn(
+ Resources.createResource(GB, 1));
+ when(csContext.getMaximumResourceCapability()).thenReturn(
+ Resources.createResource(16 * GB, 32));
+ when(csContext.getClusterResource()).thenReturn(
+ Resources.createResource(100 * 16 * GB, 100 * 32));
+ when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
+
+ // create a queue
+ PlanQueue pq = new PlanQueue(csContext, "root", null, null);
+ reservationQueue = new ReservationQueue(csContext, "a", pq);
+
+ }
+
+ @Test
+ public void testAddSubtractCapacity() throws Exception {
+
+ // verify that setting, adding, subtracting capacity works
+ reservationQueue.setCapacity(1.0F);
+ assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
+ reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
+ reservationQueue.setEntitlement(new QueueEntitlement(0.9f, 1f));
+ assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
+ reservationQueue.getCapacity() - 0.9 < CSQueueUtils.EPSILON);
+ reservationQueue.setEntitlement(new QueueEntitlement(1f, 1f));
+ assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
+ reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
+ reservationQueue.setEntitlement(new QueueEntitlement(0f, 1f));
+ assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
+ reservationQueue.getCapacity() < CSQueueUtils.EPSILON);
+
+ try {
+ reservationQueue.setEntitlement(new QueueEntitlement(1.1f, 1f));
+ fail();
+ } catch (SchedulerDynamicEditException iae) {
+ // expected
+ assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
+ reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
+ }
+
+ try {
+ reservationQueue.setEntitlement(new QueueEntitlement(-0.1f, 1f));
+ fail();
+ } catch (SchedulerDynamicEditException iae) {
+ // expected
+ assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
+ reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
+ }
+
+ }
+}
[45/50] [abbrv] git commit: YARN-1710. Logic to find allocations
within a Plan that satisfy user ReservationRequest(s). Contributed by Carlo
Curino and Subru Krishnan.
Posted by cu...@apache.org.
YARN-1710. Logic to find allocations within a Plan that satisfy user ReservationRequest(s). Contributed by Carlo Curino and Subru Krishnan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d297491d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d297491d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d297491d
Branch: refs/heads/YARN-1051
Commit: d297491d8213ca223ebfef171b54189dd82e25a3
Parents: 36a8326
Author: carlo curino <Carlo Curino>
Authored: Mon Sep 15 16:56:28 2014 -0700
Committer: carlo curino <Carlo Curino>
Committed: Mon Sep 22 16:32:11 2014 -0700
----------------------------------------------------------------------
YARN-1051-CHANGES.txt | 12 +-
.../reservation/GreedyReservationAgent.java | 367 ++++++++++++
.../reservation/ReservationAgent.java | 55 ++
.../exceptions/ContractValidationException.java | 12 +
.../reservation/TestGreedyReservationAgent.java | 588 +++++++++++++++++++
5 files changed, 1031 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d297491d/YARN-1051-CHANGES.txt
----------------------------------------------------------------------
diff --git a/YARN-1051-CHANGES.txt b/YARN-1051-CHANGES.txt
index 410d974..deece7c 100644
--- a/YARN-1051-CHANGES.txt
+++ b/YARN-1051-CHANGES.txt
@@ -4,7 +4,13 @@ CapacityScheduler. (Carlo Curino and Subru Krishnan via curino)
YARN-2475. Logic for responding to capacity drops for the
ReservationSystem. (Carlo Curino and Subru Krishnan via curino)
-YARN-1708. Public YARN APIs for creating/updating/deleting reservations. (subru)
+YARN-1708. Public YARN APIs for creating/updating/deleting
+reservations. (Carlo Curino and Subru Krishnan via subru)
-YARN-1709. In-memory data structures used to track resources over time to
-enable reservations. (subru)
+YARN-1709. In-memory data structures used to track resources over
+time to enable reservations. (Carlo Curino and Subru Krishnan via
+subru)
+
+YARN-1710. Logic to find allocations within a Plan that satisfy
+user ReservationRequest(s). (Carlo Curino and Subru Krishnan via
+curino)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d297491d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java
new file mode 100644
index 0000000..3214f93
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java
@@ -0,0 +1,367 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ContractValidationException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This Agent employs a simple greedy placement strategy, placing the various
+ * stages of a {@link ReservationRequest} from the deadline moving backward
+ * towards the arrival. This allows jobs with earlier deadline to be scheduled
+ * greedily as well. Combined with an opportunistic anticipation of work if the
+ * cluster is not fully utilized also seems to provide good latency for
+ * best-effort jobs (i.e., jobs running without a reservation).
+ *
+ * This agent does not account for locality and only consider container
+ * granularity for validation purposes (i.e., you can't exceed max-container
+ * size).
+ */
+public class GreedyReservationAgent implements ReservationAgent {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(GreedyReservationAgent.class);
+
+ @Override
+ public boolean createReservation(ReservationId reservationId, String user,
+ Plan plan, ReservationDefinition contract) throws PlanningException {
+ return computeAllocation(reservationId, user, plan, contract, null);
+ }
+
+ @Override
+ public boolean updateReservation(ReservationId reservationId, String user,
+ Plan plan, ReservationDefinition contract) throws PlanningException {
+ return computeAllocation(reservationId, user, plan, contract,
+ plan.getReservationById(reservationId));
+ }
+
+ @Override
+ public boolean deleteReservation(ReservationId reservationId, String user,
+ Plan plan) throws PlanningException {
+ return plan.deleteReservation(reservationId);
+ }
+
+ private boolean computeAllocation(ReservationId reservationId, String user,
+ Plan plan, ReservationDefinition contract,
+ ReservationAllocation oldReservation) throws PlanningException,
+ ContractValidationException {
+ LOG.info("placing the following ReservationRequest: " + contract);
+
+ Resource totalCapacity = plan.getTotalCapacity();
+
+ // Here we can addd logic to adjust the ResourceDefinition to account for
+ // system "imperfections" (e.g., scheduling delays for large containers).
+
+ // Align with plan step conservatively (i.e., ceil arrival, and floor
+ // deadline)
+ long earliestStart = contract.getArrival();
+ long step = plan.getStep();
+ if (earliestStart % step != 0) {
+ earliestStart = earliestStart + (step - (earliestStart % step));
+ }
+ long deadline =
+ contract.getDeadline() - contract.getDeadline() % plan.getStep();
+
+ // setup temporary variables to handle time-relations between stages and
+ // intermediate answers
+ long curDeadline = deadline;
+ long oldDeadline = -1;
+
+ Map<ReservationInterval, ReservationRequest> allocations =
+ new HashMap<ReservationInterval, ReservationRequest>();
+ RLESparseResourceAllocation tempAssigned =
+ new RLESparseResourceAllocation(plan.getResourceCalculator(),
+ plan.getMinimumAllocation());
+
+ List<ReservationRequest> stages = contract.getReservationRequests()
+ .getReservationResources();
+ ReservationRequestInterpreter type = contract.getReservationRequests()
+ .getInterpreter();
+
+ // Iterate the stages in backward from deadline
+ for (ListIterator<ReservationRequest> li =
+ stages.listIterator(stages.size()); li.hasPrevious();) {
+
+ ReservationRequest currentReservationStage = li.previous();
+
+ // validate the RR respect basic constraints
+ validateInput(plan, currentReservationStage, totalCapacity);
+
+ // run allocation for a single stage
+ Map<ReservationInterval, ReservationRequest> curAlloc =
+ placeSingleStage(plan, tempAssigned, currentReservationStage,
+ earliestStart, curDeadline, oldReservation, totalCapacity);
+
+ if (curAlloc == null) {
+ // if we did not find an allocation for the currentReservationStage
+ // return null, unless the ReservationDefinition we are placing is of
+ // type ANY
+ if (type != ReservationRequestInterpreter.R_ANY) {
+ throw new PlanningException("The GreedyAgent"
+ + " couldn't find a valid allocation for your request");
+ } else {
+ continue;
+ }
+ } else {
+
+ // if we did find an allocation add it to the set of allocations
+ allocations.putAll(curAlloc);
+
+ // if this request is of type ANY we are done searching (greedy)
+ // and can return the current allocation (break-out of the search)
+ if (type == ReservationRequestInterpreter.R_ANY) {
+ break;
+ }
+
+ // if the request is of ORDER or ORDER_NO_GAP we constraint the next
+ // round of allocation to precede the current allocation, by setting
+ // curDeadline
+ if (type == ReservationRequestInterpreter.R_ORDER
+ || type == ReservationRequestInterpreter.R_ORDER_NO_GAP) {
+ curDeadline = findEarliestTime(curAlloc.keySet());
+
+ // for ORDER_NO_GAP verify that the allocation found so far has no
+ // gap, return null otherwise (the greedy procedure failed to find a
+ // no-gap
+ // allocation)
+ if (type == ReservationRequestInterpreter.R_ORDER_NO_GAP
+ && oldDeadline > 0) {
+ if (oldDeadline - findLatestTime(curAlloc.keySet()) > plan
+ .getStep()) {
+ throw new PlanningException("The GreedyAgent"
+ + " couldn't find a valid allocation for your request");
+ }
+ }
+ // keep the variable oldDeadline pointing to the last deadline we
+ // found
+ oldDeadline = curDeadline;
+ }
+ }
+ }
+
+ // / If we got here is because we failed to find an allocation for the
+ // ReservationDefinition give-up and report failure to the user
+ if (allocations.isEmpty()) {
+ throw new PlanningException("The GreedyAgent"
+ + " couldn't find a valid allocation for your request");
+ }
+
+ // create reservation with above allocations if not null/empty
+
+ ReservationRequest ZERO_RES =
+ ReservationRequest.newInstance(Resource.newInstance(0, 0), 0);
+
+ long firstStartTime = findEarliestTime(allocations.keySet());
+
+ // add zero-padding from arrival up to the first non-null allocation
+ // to guarantee that the reservation exists starting at arrival
+ if (firstStartTime > earliestStart) {
+ allocations.put(new ReservationInterval(earliestStart,
+ firstStartTime), ZERO_RES);
+ firstStartTime = earliestStart;
+ // consider to add trailing zeros at the end for simmetry
+ }
+
+ // Actually add/update the reservation in the plan.
+ // This is subject to validation as other agents might be placing
+ // in parallel and there might be sharing policies the agent is not
+ // aware off.
+ ReservationAllocation capReservation =
+ new InMemoryReservationAllocation(reservationId, contract, user,
+ plan.getQueueName(), firstStartTime,
+ findLatestTime(allocations.keySet()), allocations,
+ plan.getResourceCalculator(), plan.getMinimumAllocation());
+ if (oldReservation != null) {
+ return plan.updateReservation(capReservation);
+ } else {
+ return plan.addReservation(capReservation);
+ }
+ }
+
+ private void validateInput(Plan plan, ReservationRequest rr,
+ Resource totalCapacity) throws ContractValidationException {
+
+ if (rr.getConcurrency() < 1) {
+ throw new ContractValidationException("Gang Size should be >= 1");
+ }
+
+ if (rr.getNumContainers() <= 0) {
+ throw new ContractValidationException("Num containers should be >= 0");
+ }
+
+ // check that gangSize and numContainers are compatible
+ if (rr.getNumContainers() % rr.getConcurrency() != 0) {
+ throw new ContractValidationException(
+ "Parallelism must be an exact multiple of gang size");
+ }
+
+ // check that the largest container request does not exceed
+ // the cluster-wide limit for container sizes
+ if (Resources.greaterThan(plan.getResourceCalculator(), totalCapacity,
+ rr.getCapability(), plan.getMaximumAllocation())) {
+ throw new ContractValidationException("Individual"
+ + " capability requests should not exceed cluster's maxAlloc");
+ }
+ }
+
+ /**
+ * This method actually perform the placement of an atomic stage of the
+ * reservation. The key idea is to traverse the plan backward for a
+ * "lease-duration" worth of time, and compute what is the maximum multiple of
+ * our concurrency (gang) parameter we can fit. We do this and move towards
+ * previous instant in time until the time-window is exhausted or we placed
+ * all the user request.
+ */
+ private Map<ReservationInterval, ReservationRequest> placeSingleStage(
+ Plan plan, RLESparseResourceAllocation tempAssigned,
+ ReservationRequest rr, long earliestStart, long curDeadline,
+ ReservationAllocation oldResAllocation, final Resource totalCapacity) {
+
+ Map<ReservationInterval, ReservationRequest> allocationRequests =
+ new HashMap<ReservationInterval, ReservationRequest>();
+
+ // compute the gang as a resource and get the duration
+ Resource gang = Resources.multiply(rr.getCapability(), rr.getConcurrency());
+ long dur = rr.getDuration();
+ long step = plan.getStep();
+
+ // ceil the duration to the next multiple of the plan step
+ if (dur % step != 0) {
+ dur += (step - (dur % step));
+ }
+
+ // we know for sure that this division has no remainder (part of contract
+ // with user, validate before
+ int gangsToPlace = rr.getNumContainers() / rr.getConcurrency();
+
+ int maxGang = 0;
+
+ // loop trying to place until we are done, or we are considering
+ // an invalid range of times
+ while (gangsToPlace > 0 && curDeadline - dur >= earliestStart) {
+
+ // as we run along we remember how many gangs we can fit, and what
+ // was the most constraining moment in time (we will restart just
+ // after that to place the next batch)
+ maxGang = gangsToPlace;
+ long minPoint = curDeadline;
+ int curMaxGang = maxGang;
+
+ // start placing at deadline (excluded due to [,) interval semantics and
+ // move backward
+ for (long t = curDeadline - plan.getStep(); t >= curDeadline - dur
+ && maxGang > 0; t = t - plan.getStep()) {
+
+ // As we run along we will logically remove the previous allocation for
+ // this reservation
+ // if one existed
+ Resource oldResCap = Resource.newInstance(0, 0);
+ if (oldResAllocation != null) {
+ oldResCap = oldResAllocation.getResourcesAtTime(t);
+ }
+
+ // compute net available resources
+ Resource netAvailableRes = Resources.clone(totalCapacity);
+ Resources.addTo(netAvailableRes, oldResCap);
+ Resources.subtractFrom(netAvailableRes,
+ plan.getTotalCommittedResources(t));
+ Resources.subtractFrom(netAvailableRes,
+ tempAssigned.getCapacityAtTime(t));
+
+ // compute maximum number of gangs we could fit
+ curMaxGang =
+ (int) Math.floor(Resources.divide(plan.getResourceCalculator(),
+ totalCapacity, netAvailableRes, gang));
+
+ // pick the minimum between available resources in this instant, and how
+ // many gangs we have to place
+ curMaxGang = Math.min(gangsToPlace, curMaxGang);
+
+ // compare with previous max, and set it. also remember *where* we found
+ // the minimum (useful for next attempts)
+ if (curMaxGang <= maxGang) {
+ maxGang = curMaxGang;
+ minPoint = t;
+ }
+ }
+
+ // if we were able to place any gang, record this, and decrement
+ // gangsToPlace
+ if (maxGang > 0) {
+ gangsToPlace -= maxGang;
+
+ ReservationInterval reservationInt =
+ new ReservationInterval(curDeadline - dur, curDeadline);
+ ReservationRequest reservationRes =
+ ReservationRequest.newInstance(rr.getCapability(),
+ rr.getConcurrency() * maxGang, rr.getConcurrency(),
+ rr.getDuration());
+ // remember occupied space (plan is read-only till we find a plausible
+ // allocation for the entire request). This is needed since we might be
+ // placing other ReservationRequest within the same
+ // ReservationDefinition,
+ // and we must avoid double-counting the available resources
+ tempAssigned.addInterval(reservationInt, reservationRes);
+ allocationRequests.put(reservationInt, reservationRes);
+
+ }
+
+ // reset our new starting point (curDeadline) to the most constraining
+ // point so far, we will look "left" of that to find more places where
+ // to schedule gangs (for sure nothing on the "right" of this point can
+ // fit a full gang.
+ curDeadline = minPoint;
+ }
+
+ // if no gangs are left to place we succeed and return the allocation
+ if (gangsToPlace == 0) {
+ return allocationRequests;
+ } else {
+ // If we are here is becasue we did not manage to satisfy this request.
+ // So we need to remove unwanted side-effect from tempAssigned (needed
+ // for ANY).
+ for (Map.Entry<ReservationInterval, ReservationRequest> tempAllocation :
+ allocationRequests.entrySet()) {
+ tempAssigned.removeInterval(tempAllocation.getKey(),
+ tempAllocation.getValue());
+ }
+ // and return null to signal failure in this allocation
+ return null;
+ }
+ }
+
+ // finds the leftmost point of this set of ReservationInterval
+ private long findEarliestTime(Set<ReservationInterval> resInt) {
+ long ret = Long.MAX_VALUE;
+ for (ReservationInterval s : resInt) {
+ if (s.getStartTime() < ret) {
+ ret = s.getStartTime();
+ }
+ }
+ return ret;
+ }
+
+ // finds the rightmost point of this set of ReservationIntervals
+ private long findLatestTime(Set<ReservationInterval> resInt) {
+ long ret = Long.MIN_VALUE;
+ for (ReservationInterval s : resInt) {
+ if (s.getEndTime() > ret) {
+ ret = s.getEndTime();
+ }
+ }
+ return ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d297491d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAgent.java
new file mode 100644
index 0000000..fe1941d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAgent.java
@@ -0,0 +1,55 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+
+/**
+ * An entity that seeks to acquire resources to satisfy an user's contract
+ */
+public interface ReservationAgent {
+
+ /**
+ * Create a reservation for the user that abides by the specified contract
+ *
+ * @param reservationId the identifier of the reservation to be created.
+ * @param user the user who wants to create the reservation
+ * @param plan the Plan to which the reservation must be fitted
+ * @param contract encapsulates the resources the user requires for his
+ * session
+ *
+ * @return whether the create operation was successful or not
+ * @throws PlanningException if the session cannot be fitted into the plan
+ */
+ public boolean createReservation(ReservationId reservationId, String user,
+ Plan plan, ReservationDefinition contract) throws PlanningException;
+
+ /**
+ * Update a reservation for the user that abides by the specified contract
+ *
+ * @param reservationId the identifier of the reservation to be updated
+ * @param user the user who wants to create the session
+ * @param plan the Plan to which the reservation must be fitted
+ * @param contract encapsulates the resources the user requires for his
+ * reservation
+ *
+ * @return whether the update operation was successful or not
+ * @throws PlanningException if the reservation cannot be fitted into the plan
+ */
+ public boolean updateReservation(ReservationId reservationId, String user,
+ Plan plan, ReservationDefinition contract) throws PlanningException;
+
+ /**
+ * Delete an user reservation
+ *
+ * @param reservationId the identifier of the reservation to be deleted
+ * @param user the user who wants to create the reservation
+ * @param plan the Plan to which the session must be fitted
+ *
+ * @return whether the delete operation was successful or not
+ * @throws PlanningException if the reservation cannot be fitted into the plan
+ */
+ public boolean deleteReservation(ReservationId reservationId, String user,
+ Plan plan) throws PlanningException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d297491d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ContractValidationException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ContractValidationException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ContractValidationException.java
new file mode 100644
index 0000000..7ee5a76
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ContractValidationException.java
@@ -0,0 +1,12 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions;
+
+public class ContractValidationException extends PlanningException {
+
+ private static final long serialVersionUID = 1L;
+
+ public ContractValidationException(String message) {
+ super(message);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d297491d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
new file mode 100644
index 0000000..0b0201d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
@@ -0,0 +1,588 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+import org.mortbay.log.Log;
+
+public class TestGreedyReservationAgent {
+
+ ReservationAgent agent;
+ InMemoryPlan plan;
+ Resource minAlloc = Resource.newInstance(1024, 1);
+ ResourceCalculator res = new DefaultResourceCalculator();
+ Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
+ Random rand = new Random();
+ long step;
+
+ @Before
+ public void setup() throws Exception {
+
+ long seed = rand.nextLong();
+ rand.setSeed(seed);
+ Log.info("Running with seed: " + seed);
+
+ // setting completely loose quotas
+ long timeWindow = 1000000L;
+ Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
+ step = 1000L;
+ ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+ CapacityScheduler scheduler = testUtil.mockCapacityScheduler(125);
+ String reservationQ = testUtil.getFullReservationQueueName();
+ CapacitySchedulerConfiguration capConf = scheduler.getConfiguration();
+ capConf.setReservationWindow(reservationQ, timeWindow);
+ capConf.setMaximumCapacity(reservationQ, 100);
+ capConf.setAverageCapacity(reservationQ, 100);
+ CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
+ policy.init(reservationQ, capConf, new HashSet<String>());
+ agent = new GreedyReservationAgent();
+
+ QueueMetrics queueMetrics = QueueMetrics.forQueue("dedicated",
+ mock(ParentQueue.class), false, capConf);
+
+ plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step,
+ res, minAlloc, maxAlloc, "dedicated", null, true);
+ }
+
+ @SuppressWarnings("javadoc")
+ @Test
+ public void testSimple() throws PlanningException {
+
+ prepareBasicPlan();
+
+ // create a request with a single atomic ask
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(5 * step);
+ rr.setDeadline(20 * step);
+ ReservationRequest r = ReservationRequest.newInstance(
+ Resource.newInstance(2048, 2), 10, 5, 10 * step);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setReservationResources(Collections.singletonList(r));
+ rr.setReservationRequests(reqs);
+
+ ReservationId reservationID = ReservationSystemTestUtil
+ .getNewReservationId();
+ agent.createReservation(reservationID, "u1", plan, rr);
+
+ assertTrue("Agent-based allocation failed", reservationID != null);
+ assertTrue("Agent-based allocation failed", plan.getAllReservations()
+ .size() == 3);
+
+ ReservationAllocation cs = plan.getReservationById(reservationID);
+
+ System.out.println("--------AFTER SIMPLE ALLOCATION (queue: "
+ + reservationID + ")----------");
+ System.out.println(plan.toString());
+ System.out.println(plan.toCumulativeString());
+
+ for (long i = 10 * step; i < 20 * step; i++) {
+ assertTrue(
+ "Agent-based allocation unexpected",
+ Resources.equals(cs.getResourcesAtTime(i),
+ Resource.newInstance(2048 * 10, 2 * 10)));
+ }
+
+ }
+
+ @Test
+ public void testOrder() throws PlanningException {
+ prepareBasicPlan();
+
+ // create a completely utilized segment around time 30
+ int[] f = { 100, 100 };
+
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", 30 * step, 30 * step + f.length * step,
+ ReservationSystemTestUtil.generateAllocation(30 * step, step, f),
+ res, minAlloc)));
+
+ // create a chain of 4 RR, mixing gang and non-gang
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(0 * step);
+ rr.setDeadline(70 * step);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ORDER);
+ ReservationRequest r = ReservationRequest.newInstance(
+ Resource.newInstance(2048, 2), 10, 1, 10 * step);
+ ReservationRequest r2 = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), 10, 10, 20 * step);
+ List<ReservationRequest> list = new ArrayList<ReservationRequest>();
+ list.add(r);
+ list.add(r2);
+ list.add(r);
+ list.add(r2);
+ reqs.setReservationResources(list);
+ rr.setReservationRequests(reqs);
+
+ // submit to agent
+ ReservationId reservationID = ReservationSystemTestUtil
+ .getNewReservationId();
+ agent.createReservation(reservationID, "u1", plan, rr);
+
+ // validate
+ assertTrue("Agent-based allocation failed", reservationID != null);
+ assertTrue("Agent-based allocation failed", plan.getAllReservations()
+ .size() == 4);
+
+ ReservationAllocation cs = plan.getReservationById(reservationID);
+
+ assertTrue(cs.toString(), check(cs, 0 * step, 10 * step, 20, 1024, 1));
+ assertTrue(cs.toString(), check(cs, 10 * step, 30 * step, 10, 1024, 1));
+ assertTrue(cs.toString(), check(cs, 40 * step, 50 * step, 20, 1024, 1));
+ assertTrue(cs.toString(), check(cs, 50 * step, 70 * step, 10, 1024, 1));
+
+ System.out.println("--------AFTER ORDER ALLOCATION (queue: "
+ + reservationID + ")----------");
+ System.out.println(plan.toString());
+ System.out.println(plan.toCumulativeString());
+
+ }
+
+ @Test
+ public void testOrderNoGapImpossible() throws PlanningException {
+ prepareBasicPlan();
+ // create a completely utilized segment at time 30
+ int[] f = { 100, 100 };
+
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", 30 * step, 30 * step + f.length * step,
+ ReservationSystemTestUtil.generateAllocation(30 * step, step, f),
+ res, minAlloc)));
+
+ // create a chain of 4 RR, mixing gang and non-gang
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(0L);
+
+ rr.setDeadline(70L);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ORDER_NO_GAP);
+ ReservationRequest r = ReservationRequest.newInstance(
+ Resource.newInstance(2048, 2), 10, 1, 10);
+ ReservationRequest r2 = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), 10, 10, 20);
+ List<ReservationRequest> list = new ArrayList<ReservationRequest>();
+ list.add(r);
+ list.add(r2);
+ list.add(r);
+ list.add(r2);
+ reqs.setReservationResources(list);
+ rr.setReservationRequests(reqs);
+
+ ReservationId reservationID = ReservationSystemTestUtil
+ .getNewReservationId();
+ boolean result = false;
+ try {
+ // submit to agent
+ result = agent.createReservation(reservationID, "u1", plan, rr);
+ fail();
+ } catch (PlanningException p) {
+ // expected
+ }
+
+ // validate
+ assertFalse("Agent-based allocation should have failed", result);
+ assertTrue("Agent-based allocation should have failed", plan
+ .getAllReservations().size() == 3);
+
+ System.out
+ .println("--------AFTER ORDER_NO_GAP IMPOSSIBLE ALLOCATION (queue: "
+ + reservationID + ")----------");
+ System.out.println(plan.toString());
+ System.out.println(plan.toCumulativeString());
+
+ }
+
+ @Test
+ public void testOrderNoGap() throws PlanningException {
+ prepareBasicPlan();
+ // create a chain of 4 RR, mixing gang and non-gang
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(0 * step);
+ rr.setDeadline(60 * step);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ORDER_NO_GAP);
+ ReservationRequest r = ReservationRequest.newInstance(
+ Resource.newInstance(2048, 2), 10, 1, 10 * step);
+ ReservationRequest r2 = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), 10, 10, 20 * step);
+ List<ReservationRequest> list = new ArrayList<ReservationRequest>();
+ list.add(r);
+ list.add(r2);
+ list.add(r);
+ list.add(r2);
+ reqs.setReservationResources(list);
+ rr.setReservationRequests(reqs);
+ rr.setReservationRequests(reqs);
+
+ // submit to agent
+ ReservationId reservationID = ReservationSystemTestUtil
+ .getNewReservationId();
+ agent.createReservation(reservationID, "u1", plan, rr);
+
+ System.out.println("--------AFTER ORDER ALLOCATION (queue: "
+ + reservationID + ")----------");
+ System.out.println(plan.toString());
+ System.out.println(plan.toCumulativeString());
+
+ // validate
+ assertTrue("Agent-based allocation failed", reservationID != null);
+ assertTrue("Agent-based allocation failed", plan.getAllReservations()
+ .size() == 3);
+
+ ReservationAllocation cs = plan.getReservationById(reservationID);
+
+ assertTrue(cs.toString(), check(cs, 0 * step, 10 * step, 20, 1024, 1));
+ assertTrue(cs.toString(), check(cs, 10 * step, 30 * step, 10, 1024, 1));
+ assertTrue(cs.toString(), check(cs, 30 * step, 40 * step, 20, 1024, 1));
+ assertTrue(cs.toString(), check(cs, 40 * step, 60 * step, 10, 1024, 1));
+
+ }
+
+ @Test
+ public void testSingleSliding() throws PlanningException {
+ prepareBasicPlan();
+
+ // create a single request for which we need subsequent (tight) packing.
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(100 * step);
+ rr.setDeadline(120 * step);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
+ ReservationRequest r = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), 200, 10, 10 * step);
+
+ List<ReservationRequest> list = new ArrayList<ReservationRequest>();
+ list.add(r);
+ reqs.setReservationResources(list);
+ rr.setReservationRequests(reqs);
+
+ // submit to agent
+ ReservationId reservationID = ReservationSystemTestUtil
+ .getNewReservationId();
+ agent.createReservation(reservationID, "u1", plan, rr);
+
+ // validate results, we expect the second one to be accepted
+ assertTrue("Agent-based allocation failed", reservationID != null);
+ assertTrue("Agent-based allocation failed", plan.getAllReservations()
+ .size() == 3);
+
+ ReservationAllocation cs = plan.getReservationById(reservationID);
+
+ assertTrue(cs.toString(), check(cs, 100 * step, 120 * step, 100, 1024, 1));
+
+ System.out.println("--------AFTER packed ALLOCATION (queue: "
+ + reservationID + ")----------");
+ System.out.println(plan.toString());
+ System.out.println(plan.toCumulativeString());
+
+ }
+
+ @Test
+ public void testAny() throws PlanningException {
+ prepareBasicPlan();
+ // create an ANY request, with an impossible step (last in list, first
+ // considered),
+ // and two satisfiable ones. We expect the second one to be returned.
+
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(100 * step);
+ rr.setDeadline(120 * step);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ANY);
+ ReservationRequest r = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), 5, 5, 10 * step);
+ ReservationRequest r2 = ReservationRequest.newInstance(
+ Resource.newInstance(2048, 2), 10, 5, 10 * step);
+ ReservationRequest r3 = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), 110, 110, 10 * step);
+
+ List<ReservationRequest> list = new ArrayList<ReservationRequest>();
+ list.add(r);
+ list.add(r2);
+ list.add(r3);
+ reqs.setReservationResources(list);
+ rr.setReservationRequests(reqs);
+
+ // submit to agent
+ ReservationId reservationID = ReservationSystemTestUtil
+ .getNewReservationId();
+ boolean res = agent.createReservation(reservationID, "u1", plan, rr);
+
+ // validate results, we expect the second one to be accepted
+ assertTrue("Agent-based allocation failed", res);
+ assertTrue("Agent-based allocation failed", plan.getAllReservations()
+ .size() == 3);
+
+ ReservationAllocation cs = plan.getReservationById(reservationID);
+
+ assertTrue(cs.toString(), check(cs, 110 * step, 120 * step, 20, 1024, 1));
+
+ System.out.println("--------AFTER ANY ALLOCATION (queue: " + reservationID
+ + ")----------");
+ System.out.println(plan.toString());
+ System.out.println(plan.toCumulativeString());
+
+ }
+
+ @Test
+ public void testAnyImpossible() throws PlanningException {
+ prepareBasicPlan();
+ // create an ANY request, with all impossible alternatives
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(100L);
+ rr.setDeadline(120L);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ANY);
+
+ // longer than arrival-deadline
+ ReservationRequest r1 = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), 35, 5, 30);
+ // above max cluster size
+ ReservationRequest r2 = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), 110, 110, 10);
+
+ List<ReservationRequest> list = new ArrayList<ReservationRequest>();
+ list.add(r1);
+ list.add(r2);
+ reqs.setReservationResources(list);
+ rr.setReservationRequests(reqs);
+
+ ReservationId reservationID = ReservationSystemTestUtil
+ .getNewReservationId();
+ boolean result = false;
+ try {
+ // submit to agent
+ result = agent.createReservation(reservationID, "u1", plan, rr);
+ fail();
+ } catch (PlanningException p) {
+ // expected
+ }
+ // validate results, we expect the second one to be accepted
+ assertFalse("Agent-based allocation should have failed", result);
+ assertTrue("Agent-based allocation should have failed", plan
+ .getAllReservations().size() == 2);
+
+ System.out.println("--------AFTER ANY IMPOSSIBLE ALLOCATION (queue: "
+ + reservationID + ")----------");
+ System.out.println(plan.toString());
+ System.out.println(plan.toCumulativeString());
+
+ }
+
+ @Test
+ public void testAll() throws PlanningException {
+ prepareBasicPlan();
+ // create an ALL request
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(100 * step);
+ rr.setDeadline(120 * step);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
+ ReservationRequest r = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), 5, 5, 10 * step);
+ ReservationRequest r2 = ReservationRequest.newInstance(
+ Resource.newInstance(2048, 2), 10, 10, 20 * step);
+
+ List<ReservationRequest> list = new ArrayList<ReservationRequest>();
+ list.add(r);
+ list.add(r2);
+ reqs.setReservationResources(list);
+ rr.setReservationRequests(reqs);
+
+ // submit to agent
+ ReservationId reservationID = ReservationSystemTestUtil
+ .getNewReservationId();
+ agent.createReservation(reservationID, "u1", plan, rr);
+
+ // validate results, we expect the second one to be accepted
+ assertTrue("Agent-based allocation failed", reservationID != null);
+ assertTrue("Agent-based allocation failed", plan.getAllReservations()
+ .size() == 3);
+
+ ReservationAllocation cs = plan.getReservationById(reservationID);
+
+ assertTrue(cs.toString(), check(cs, 100 * step, 110 * step, 20, 1024, 1));
+ assertTrue(cs.toString(), check(cs, 110 * step, 120 * step, 25, 1024, 1));
+
+ System.out.println("--------AFTER ALL ALLOCATION (queue: " + reservationID
+ + ")----------");
+ System.out.println(plan.toString());
+ System.out.println(plan.toCumulativeString());
+
+ }
+
+ @Test
+ public void testAllImpossible() throws PlanningException {
+ prepareBasicPlan();
+ // create an ALL request, with an impossible combination, it should be
+ // rejected, and allocation remain unchanged
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(100L);
+ rr.setDeadline(120L);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
+ ReservationRequest r = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), 55, 5, 10);
+ ReservationRequest r2 = ReservationRequest.newInstance(
+ Resource.newInstance(2048, 2), 55, 5, 20);
+
+ List<ReservationRequest> list = new ArrayList<ReservationRequest>();
+ list.add(r);
+ list.add(r2);
+ reqs.setReservationResources(list);
+ rr.setReservationRequests(reqs);
+
+ ReservationId reservationID = ReservationSystemTestUtil
+ .getNewReservationId();
+ boolean result = false;
+ try {
+ // submit to agent
+ result = agent.createReservation(reservationID, "u1", plan, rr);
+ fail();
+ } catch (PlanningException p) {
+ // expected
+ }
+
+ // validate results, we expect the second one to be accepted
+ assertFalse("Agent-based allocation failed", result);
+ assertTrue("Agent-based allocation failed", plan.getAllReservations()
+ .size() == 2);
+
+ System.out.println("--------AFTER ALL IMPOSSIBLE ALLOCATION (queue: "
+ + reservationID + ")----------");
+ System.out.println(plan.toString());
+ System.out.println(plan.toCumulativeString());
+
+ }
+
+ private void prepareBasicPlan() throws PlanningException {
+
+ // insert in the reservation a couple of controlled reservations, to create
+ // conditions for assignment that are non-empty
+
+ int[] f = { 10, 10, 20, 20, 20, 10, 10 };
+
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", 0L, 0L + f.length * step, ReservationSystemTestUtil
+ .generateAllocation(0, step, f), res, minAlloc)));
+
+ int[] f2 = { 5, 5, 5, 5, 5, 5, 5 };
+ Map<ReservationInterval, ReservationRequest> alloc =
+ ReservationSystemTestUtil.generateAllocation(5000, step, f2);
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", 5000, 5000 + f2.length * step, alloc, res, minAlloc)));
+
+ System.out.println("--------BEFORE AGENT----------");
+ System.out.println(plan.toString());
+ System.out.println(plan.toCumulativeString());
+ }
+
+ private boolean check(ReservationAllocation cs, long start, long end,
+ int containers, int mem, int cores) {
+
+ boolean res = true;
+ for (long i = start; i < end; i++) {
+ res = res
+ && Resources.equals(cs.getResourcesAtTime(i),
+ Resource.newInstance(mem * containers, cores * containers));
+ }
+ return res;
+ }
+
+ public void testStress(int numJobs) throws PlanningException, IOException {
+
+ long timeWindow = 1000000L;
+ Resource clusterCapacity = Resource.newInstance(500 * 100 * 1024, 500 * 32);
+ step = 1000L;
+ ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+ CapacityScheduler scheduler = testUtil.mockCapacityScheduler(500 * 100);
+ String reservationQ = testUtil.getFullReservationQueueName();
+ CapacitySchedulerConfiguration capConf = scheduler.getConfiguration();
+ capConf.setReservationWindow(reservationQ, timeWindow);
+ capConf.setMaximumCapacity(reservationQ, 100);
+ capConf.setAverageCapacity(reservationQ, 100);
+ CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
+ policy.init(reservationQ, capConf, new HashSet<String>());
+
+ plan = new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, agent,
+ clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true);
+
+ int acc = 0;
+ List<ReservationDefinition> list = new ArrayList<ReservationDefinition>();
+ for (long i = 0; i < numJobs; i++) {
+ list.add(ReservationSystemTestUtil.generateRandomRR(rand, i));
+ }
+
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < numJobs; i++) {
+
+ try {
+ if (agent.createReservation(
+ ReservationSystemTestUtil.getNewReservationId(), "u" + i % 100,
+ plan, list.get(i))) {
+ acc++;
+ }
+ } catch (PlanningException p) {
+ // ignore exceptions
+ }
+ }
+
+ long end = System.currentTimeMillis();
+ System.out.println("Submitted " + numJobs + " jobs " + " accepted " + acc
+ + " in " + (end - start) + "ms");
+ }
+
+ public static void main(String[] arg) {
+
+ // run a stress test with by default 1000 random jobs
+ int numJobs = 1000;
+ if (arg.length > 0) {
+ numJobs = Integer.parseInt(arg[0]);
+ }
+
+ try {
+ TestGreedyReservationAgent test = new TestGreedyReservationAgent();
+ test.setup();
+ test.testStress(numJobs);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+}
[38/50] [abbrv] git commit: YARN-2539. FairScheduler: Set the default
value for maxAMShare to 0.5. (Wei Yan via kasha)
Posted by cu...@apache.org.
YARN-2539. FairScheduler: Set the default value for maxAMShare to 0.5. (Wei Yan via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43efdd30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43efdd30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43efdd30
Branch: refs/heads/YARN-1051
Commit: 43efdd30b52f4ee8967a03dac4d548095d6cafdb
Parents: 23e17ce
Author: Karthik Kambatla <ka...@apache.org>
Authored: Mon Sep 22 16:09:52 2014 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Mon Sep 22 16:09:52 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../scheduler/fair/AllocationConfiguration.java | 2 +-
.../fair/AllocationFileLoaderService.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 30 +++++++++++---------
.../src/site/apt/FairScheduler.apt.vm | 4 +--
5 files changed, 23 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43efdd30/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e5e397e..2bc118d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -241,6 +241,9 @@ Release 2.6.0 - UNRELEASED
YARN-1372. Ensure all completed containers are reported to the AMs across
RM restart. (Anubhav Dhoot via jianhe)
+ YARN-2539. FairScheduler: Set the default value for maxAMShare to 0.5.
+ (Wei Yan via kasha)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43efdd30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index de5a999..70a6496 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -130,7 +130,7 @@ public class AllocationConfiguration {
queueMaxAMShares = new HashMap<String, Float>();
userMaxAppsDefault = Integer.MAX_VALUE;
queueMaxAppsDefault = Integer.MAX_VALUE;
- queueMaxAMShareDefault = -1.0f;
+ queueMaxAMShareDefault = 0.5f;
queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>();
minSharePreemptionTimeouts = new HashMap<String, Long>();
fairSharePreemptionTimeouts = new HashMap<String, Long>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43efdd30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index c2dfc84..2022510 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -224,7 +224,7 @@ public class AllocationFileLoaderService extends AbstractService {
new HashMap<String, Map<QueueACL, AccessControlList>>();
int userMaxAppsDefault = Integer.MAX_VALUE;
int queueMaxAppsDefault = Integer.MAX_VALUE;
- float queueMaxAMShareDefault = -1.0f;
+ float queueMaxAMShareDefault = 0.5f;
long defaultFairSharePreemptionTimeout = Long.MAX_VALUE;
long defaultMinSharePreemptionTimeout = Long.MAX_VALUE;
float defaultFairSharePreemptionThreshold = 0.5f;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43efdd30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 1c9b289..a6e928a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3128,7 +3128,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
out.println("<queue name=\"queue1\">");
out.println("</queue>");
out.println("<queue name=\"queue2\">");
- out.println("<maxAMShare>1.0</maxAMShare>");
+ out.println("<maxAMShare>0.4</maxAMShare>");
out.println("</queue>");
out.println("<queue name=\"queue3\">");
out.println("</queue>");
@@ -3172,40 +3172,42 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Queue queue5's fair share should be 0", 0, queue5
.getFairShare().getMemory());
- List<String> queues = Arrays.asList("root.default", "root.queue3",
- "root.queue4", "root.queue5");
+ List<String> queues = Arrays.asList("root.queue3", "root.queue4",
+ "root.queue5");
for (String queue : queues) {
createSchedulingRequest(1 * 1024, queue, "user1");
scheduler.update();
scheduler.handle(updateEvent);
}
- Resource amResource1 = Resource.newInstance(2048, 1);
+ Resource amResource1 = Resource.newInstance(1024, 1);
int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority();
- // Exceeds queue limit, but default maxAMShare is -1.0 so it doesn't matter
+ // The fair share is 2048 MB, and the default maxAMShare is 0.5f,
+ // so the AM is accepted.
ApplicationAttemptId attId1 = createAppAttemptId(1, 1);
createApplicationWithAMResource(attId1, "queue1", "test1", amResource1);
- createSchedulingRequestExistingApplication(2048, 1, amPriority, attId1);
+ createSchedulingRequestExistingApplication(1024, 1, amPriority, attId1);
FSAppAttempt app1 = scheduler.getSchedulerApp(attId1);
scheduler.update();
scheduler.handle(updateEvent);
- assertEquals("Application1's AM requests 2048 MB memory",
- 2048, app1.getAMResource().getMemory());
+ assertEquals("Application1's AM requests 1024 MB memory",
+ 1024, app1.getAMResource().getMemory());
assertEquals("Application1's AM should be running",
1, app1.getLiveContainers().size());
- assertEquals("Queue1's AM resource usage should be 2048 MB memory",
- 2048, queue1.getAmResourceUsage().getMemory());
+ assertEquals("Queue1's AM resource usage should be 1024 MB memory",
+ 1024, queue1.getAmResourceUsage().getMemory());
- // Exceeds queue limit, and maxAMShare is 1.0
+ // Now the fair share is 1639 MB, and the maxAMShare is 0.4f,
+ // so the AM is not accepted.
ApplicationAttemptId attId2 = createAppAttemptId(2, 1);
createApplicationWithAMResource(attId2, "queue2", "test1", amResource1);
- createSchedulingRequestExistingApplication(2048, 1, amPriority, attId2);
+ createSchedulingRequestExistingApplication(1024, 1, amPriority, attId2);
FSAppAttempt app2 = scheduler.getSchedulerApp(attId2);
scheduler.update();
scheduler.handle(updateEvent);
- assertEquals("Application2's AM requests 2048 MB memory",
- 2048, app2.getAMResource().getMemory());
+ assertEquals("Application2's AM requests 1024 MB memory",
+ 1024, app2.getAMResource().getMemory());
assertEquals("Application2's AM should not be running",
0, app2.getLiveContainers().size());
assertEquals("Queue2's AM resource usage should be 0 MB memory",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43efdd30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
index df61422..dcf9486 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
@@ -246,8 +246,8 @@ Allocation file format
* maxAMShare: limit the fraction of the queue's fair share that can be used
to run application masters. This property can only be used for leaf queues.
For example, if set to 1.0f, then AMs in the leaf queue can take up to 100%
- of both the memory and CPU fair share. The default value is -1.0f, which
- means that this check is disabled.
+ of both the memory and CPU fair share. The value of -1.0f will disable
+ this feature and the amShare will not be checked. The default value is 0.5f.
* weight: to share the cluster non-proportionally with other queues. Weights
default to 1, and a queue with weight 2 should receive approximately twice
[04/50] [abbrv] git commit: Revise the HDFS-6584 entry CHANGES.txt.
Posted by cu...@apache.org.
Revise the HDFS-6584 entry CHANGES.txt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d01a684
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d01a684
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d01a684
Branch: refs/heads/YARN-1051
Commit: 5d01a684a38a765eabec53ce88687f1808b6c956
Parents: fd3cddf
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Fri Sep 19 07:46:18 2014 +0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Fri Sep 19 07:46:18 2014 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d01a684/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index df3d3f1..fc22282 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -18,8 +18,8 @@ Trunk (Unreleased)
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
- HDFS-6584. Support archival storage; see the detail subtasks in the
- breakdown section. (szetszwo)
+ HDFS-6584. Support archival storage. (See breakdown of tasks below for
+ features and contributors)
IMPROVEMENTS
[37/50] [abbrv] git commit: YARN-2540. FairScheduler: Queue filters
not working on scheduler page in RM UI. (Ashwin Shankar via kasha)
Posted by cu...@apache.org.
YARN-2540. FairScheduler: Queue filters not working on scheduler page in RM UI. (Ashwin Shankar via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23e17ce0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23e17ce0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23e17ce0
Branch: refs/heads/YARN-1051
Commit: 23e17ce0ffba11a08c033b753573a756532cd1c2
Parents: 912ad32
Author: Karthik Kambatla <ka...@apache.org>
Authored: Mon Sep 22 15:40:47 2014 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Mon Sep 22 15:40:47 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/webapp/FairSchedulerPage.java | 6 +++---
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e17ce0/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7e5d0f9..e5e397e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -414,6 +414,9 @@ Release 2.6.0 - UNRELEASED
YARN-2453. TestProportionalCapacityPreemptionPolicy fails with
FairScheduler. (Zhihai Xu via kasha)
+
+ YARN-2540. FairScheduler: Queue filters not working on scheduler page in
+ RM UI. (Ashwin Shankar via kasha)
Release 2.5.1 - 2014-09-05
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e17ce0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
index bcf7781..8c54f4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
@@ -220,9 +220,9 @@ public class FairSchedulerPage extends RmView {
" }",
" });",
" $('#cs').bind('select_node.jstree', function(e, data) {",
- " var q = $('.q', data.rslt.obj).first().text();",
- " if (q == 'root') q = '';",
- " else q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';",
+ " var queues = $('.q', data.rslt.obj);",
+ " var q = '^' + queues.first().text();",
+ " q += queues.length == 1 ? '$' : '\\\\.';",
" $('#apps').dataTable().fnFilter(q, 4, true);",
" });",
" $('#cs').show();",
[24/50] [abbrv] git commit: HDFS-7107. Avoid Findbugs warning for
synchronization on AbstractNNFailoverProxyProvider#fallbackToSimpleAuth.
Contributed by Chris Nauroth.
Posted by cu...@apache.org.
HDFS-7107. Avoid Findbugs warning for synchronization on AbstractNNFailoverProxyProvider#fallbackToSimpleAuth. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47491f9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47491f9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47491f9a
Branch: refs/heads/YARN-1051
Commit: 47491f9a119faf75abaa56f676ae93460583bd3f
Parents: 84a0a62
Author: cnauroth <cn...@apache.org>
Authored: Sat Sep 20 20:53:09 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Sat Sep 20 20:53:09 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../server/namenode/ha/AbstractNNFailoverProxyProvider.java | 6 ++++--
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47491f9a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9fa06d4..dfdc21f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -905,6 +905,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7105. Allow falling back to a non-SASL connection on
DataTransferProtocol in several edge cases. (cnauroth)
+ HDFS-7107. Avoid Findbugs warning for synchronization on
+ AbstractNNFailoverProxyProvider#fallbackToSimpleAuth. (cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47491f9a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
index 08e82be..a0aa10b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
@@ -38,12 +38,14 @@ public abstract class AbstractNNFailoverProxyProvider<T> implements
public abstract boolean useLogicalURI();
/**
- * Set for tracking if a secure client falls back to simple auth.
+ * Set for tracking if a secure client falls back to simple auth. This method
+ * is synchronized only to stifle a Findbugs warning.
*
* @param fallbackToSimpleAuth - set to true or false during this method to
* indicate if a secure client falls back to simple auth
*/
- public void setFallbackToSimpleAuth(AtomicBoolean fallbackToSimpleAuth) {
+ public synchronized void setFallbackToSimpleAuth(
+ AtomicBoolean fallbackToSimpleAuth) {
this.fallbackToSimpleAuth = fallbackToSimpleAuth;
}
}
[28/50] [abbrv] git commit: HDFS-6987. Move CipherSuite xattr
information up to the encryption zone root. Contributed by Zhe Zhang.
Posted by cu...@apache.org.
HDFS-6987. Move CipherSuite xattr information up to the encryption zone root. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1737950d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1737950d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1737950d
Branch: refs/heads/YARN-1051
Commit: 1737950d0fc83c68f386881b843c41b0b1e342de
Parents: c50fc92
Author: Andrew Wang <wa...@apache.org>
Authored: Sun Sep 21 21:28:14 2014 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Sun Sep 21 21:29:46 2014 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/FileEncryptionInfo.java | 12 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 1 -
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +-
.../hadoop/hdfs/protocol/EncryptionZone.java | 35 +++--
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 48 +++++-
.../server/namenode/EncryptionZoneManager.java | 51 ++++---
.../hdfs/server/namenode/FSDirectory.java | 152 ++++++++++++++-----
.../hdfs/server/namenode/FSEditLogLoader.java | 8 +-
.../hdfs/server/namenode/FSNamesystem.java | 28 ++--
.../hdfs/server/namenode/INodesInPath.java | 3 +-
.../hadoop-hdfs/src/main/proto/encryption.proto | 7 +-
.../hadoop-hdfs/src/main/proto/hdfs.proto | 22 ++-
13 files changed, 271 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
index f960233..641709d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
@@ -34,6 +34,7 @@ public class FileEncryptionInfo {
private final CipherSuite cipherSuite;
private final byte[] edek;
private final byte[] iv;
+ private final String keyName;
private final String ezKeyVersionName;
/**
@@ -42,14 +43,16 @@ public class FileEncryptionInfo {
* @param suite CipherSuite used to encrypt the file
* @param edek encrypted data encryption key (EDEK) of the file
* @param iv initialization vector (IV) used to encrypt the file
+ * @param keyName name of the key used for the encryption zone
* @param ezKeyVersionName name of the KeyVersion used to encrypt the
* encrypted data encryption key.
*/
public FileEncryptionInfo(final CipherSuite suite, final byte[] edek,
- final byte[] iv, final String ezKeyVersionName) {
+ final byte[] iv, final String keyName, final String ezKeyVersionName) {
checkNotNull(suite);
checkNotNull(edek);
checkNotNull(iv);
+ checkNotNull(keyName);
checkNotNull(ezKeyVersionName);
checkArgument(edek.length == suite.getAlgorithmBlockSize(),
"Unexpected key length");
@@ -58,6 +61,7 @@ public class FileEncryptionInfo {
this.cipherSuite = suite;
this.edek = edek;
this.iv = iv;
+ this.keyName = keyName;
this.ezKeyVersionName = ezKeyVersionName;
}
@@ -84,6 +88,11 @@ public class FileEncryptionInfo {
}
/**
+ * @return name of the encryption zone key.
+ */
+ public String getKeyName() { return keyName; }
+
+ /**
* @return name of the encryption zone KeyVersion used to encrypt the
* encrypted data encryption key (EDEK).
*/
@@ -95,6 +104,7 @@ public class FileEncryptionInfo {
builder.append("cipherSuite: " + cipherSuite);
builder.append(", edek: " + Hex.encodeHexString(edek));
builder.append(", iv: " + Hex.encodeHexString(iv));
+ builder.append(", keyName: " + keyName);
builder.append(", ezKeyVersionName: " + ezKeyVersionName);
builder.append("}");
return builder.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
index 16e78ce..697d7ec 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -169,7 +169,6 @@ public class MiniKMS {
kms.set(KMSConfiguration.KEY_PROVIDER_URI,
"jceks://file@" + new Path(kmsConfDir, "kms.keystore").toUri());
kms.set("hadoop.kms.authentication.type", "simple");
- kms.setBoolean(KMSConfiguration.KEY_AUTHORIZATION_ENABLE, false);
Writer writer = new FileWriter(kmsFile);
kms.writeXml(writer);
writer.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f29f7c5..d95e228 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -133,6 +133,9 @@ Trunk (Unreleased)
HDFS-6609. Use DirectorySnapshottableFeature to represent a snapshottable
directory. (Jing Zhao via wheat9)
+ HDFS-6987. Move CipherSuite xattr information up to the encryption zone
+ root. (Zhe Zhang via wang)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index ed08be0..6f94370 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1319,8 +1319,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
" an encrypted file");
}
EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
- //TODO: here we have to put the keyName to be provided by HDFS-6987
- null, feInfo.getEzKeyVersionName(), feInfo.getIV(),
+ feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
feInfo.getEncryptedDataEncryptionKey());
try {
return provider.decryptEncryptedKey(ekv);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java
index a20513a..58e9eba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java
@@ -21,6 +21,7 @@ import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.crypto.CipherSuite;
/**
* A simple class for representing an encryption zone. Presently an encryption
@@ -31,32 +32,40 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Evolving
public class EncryptionZone {
+ private final long id;
private final String path;
+ private final CipherSuite suite;
private final String keyName;
- private final long id;
- public EncryptionZone(String path, String keyName, long id) {
+ public EncryptionZone(long id, String path,
+ CipherSuite suite, String keyName) {
+ this.id = id;
this.path = path;
+ this.suite = suite;
this.keyName = keyName;
- this.id = id;
+ }
+
+ public long getId() {
+ return id;
}
public String getPath() {
return path;
}
- public String getKeyName() {
- return keyName;
+ public CipherSuite getSuite() {
+ return suite;
}
- public long getId() {
- return id;
+ public String getKeyName() {
+ return keyName;
}
@Override
public int hashCode() {
return new HashCodeBuilder(13, 31).
- append(path).append(keyName).append(id).
+ append(id).append(path).
+ append(suite).append(keyName).
toHashCode();
}
@@ -74,16 +83,18 @@ public class EncryptionZone {
EncryptionZone rhs = (EncryptionZone) obj;
return new EqualsBuilder().
+ append(id, rhs.id).
append(path, rhs.path).
+ append(suite, rhs.suite).
append(keyName, rhs.keyName).
- append(id, rhs.id).
isEquals();
}
@Override
public String toString() {
- return "EncryptionZone [path=" + path +
- ", keyName=" + keyName +
- ", id=" + id + "]";
+ return "EncryptionZone [id=" + id +
+ ", path=" + path +
+ ", suite=" + suite +
+ ", keyName=" + keyName + "]";
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 862a803..c0b71eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -177,7 +177,6 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
-import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
@@ -2332,12 +2331,14 @@ public class PBHelper {
return EncryptionZoneProto.newBuilder()
.setId(zone.getId())
.setKeyName(zone.getKeyName())
- .setPath(zone.getPath()).build();
+ .setPath(zone.getPath())
+ .setSuite(convert(zone.getSuite()))
+ .build();
}
public static EncryptionZone convert(EncryptionZoneProto proto) {
- return new EncryptionZone(proto.getPath(), proto.getKeyName(),
- proto.getId());
+ return new EncryptionZone(proto.getId(), proto.getPath(),
+ convert(proto.getSuite()), proto.getKeyName());
}
public static ShortCircuitShmSlotProto convert(SlotId slotId) {
@@ -2662,6 +2663,30 @@ public class PBHelper {
.setKey(getByteString(info.getEncryptedDataEncryptionKey()))
.setIv(getByteString(info.getIV()))
.setEzKeyVersionName(info.getEzKeyVersionName())
+ .setKeyName(info.getKeyName())
+ .build();
+ }
+
+ public static HdfsProtos.PerFileEncryptionInfoProto convertPerFileEncInfo(
+ FileEncryptionInfo info) {
+ if (info == null) {
+ return null;
+ }
+ return HdfsProtos.PerFileEncryptionInfoProto.newBuilder()
+ .setKey(getByteString(info.getEncryptedDataEncryptionKey()))
+ .setIv(getByteString(info.getIV()))
+ .setEzKeyVersionName(info.getEzKeyVersionName())
+ .build();
+ }
+
+ public static HdfsProtos.ZoneEncryptionInfoProto convert(
+ CipherSuite suite, String keyName) {
+ if (suite == null || keyName == null) {
+ return null;
+ }
+ return HdfsProtos.ZoneEncryptionInfoProto.newBuilder()
+ .setSuite(convert(suite))
+ .setKeyName(keyName)
.build();
}
@@ -2674,7 +2699,20 @@ public class PBHelper {
byte[] key = proto.getKey().toByteArray();
byte[] iv = proto.getIv().toByteArray();
String ezKeyVersionName = proto.getEzKeyVersionName();
- return new FileEncryptionInfo(suite, key, iv, ezKeyVersionName);
+ String keyName = proto.getKeyName();
+ return new FileEncryptionInfo(suite, key, iv, keyName, ezKeyVersionName);
+ }
+
+ public static FileEncryptionInfo convert(
+ HdfsProtos.PerFileEncryptionInfoProto fileProto,
+ CipherSuite suite, String keyName) {
+ if (fileProto == null || suite == null || keyName == null) {
+ return null;
+ }
+ byte[] key = fileProto.getKey().toByteArray();
+ byte[] iv = fileProto.getIv().toByteArray();
+ String ezKeyVersionName = fileProto.getEzKeyVersionName();
+ return new FileEncryptionInfo(suite, key, iv, keyName, ezKeyVersionName);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index c428690..e22f8b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -26,6 +26,7 @@ import java.util.TreeMap;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
@@ -33,6 +34,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,8 +56,8 @@ public class EncryptionZoneManager {
public static Logger LOG = LoggerFactory.getLogger(EncryptionZoneManager
.class);
- private static final EncryptionZone NULL_EZ =
- new EncryptionZone("", "", -1);
+ public static final EncryptionZone NULL_EZ =
+ new EncryptionZone(-1, "", CipherSuite.UNKNOWN, "");
/**
* EncryptionZoneInt is the internal representation of an encryption zone. The
@@ -62,21 +65,27 @@ public class EncryptionZoneManager {
* contains the EZ's pathname.
*/
private static class EncryptionZoneInt {
- private final String keyName;
private final long inodeId;
+ private final CipherSuite suite;
+ private final String keyName;
- EncryptionZoneInt(long inodeId, String keyName) {
- this.keyName = keyName;
+ EncryptionZoneInt(long inodeId, CipherSuite suite, String keyName) {
this.inodeId = inodeId;
- }
-
- String getKeyName() {
- return keyName;
+ this.suite = suite;
+ this.keyName = keyName;
}
long getINodeId() {
return inodeId;
}
+
+ CipherSuite getSuite() {
+ return suite;
+ }
+
+ String getKeyName() {
+ return keyName;
+ }
}
private final TreeMap<Long, EncryptionZoneInt> encryptionZones;
@@ -109,9 +118,9 @@ public class EncryptionZoneManager {
* @param inodeId of the encryption zone
* @param keyName encryption zone key name
*/
- void addEncryptionZone(Long inodeId, String keyName) {
+ void addEncryptionZone(Long inodeId, CipherSuite suite, String keyName) {
assert dir.hasWriteLock();
- unprotectedAddEncryptionZone(inodeId, keyName);
+ unprotectedAddEncryptionZone(inodeId, suite, keyName);
}
/**
@@ -122,8 +131,10 @@ public class EncryptionZoneManager {
* @param inodeId of the encryption zone
* @param keyName encryption zone key name
*/
- void unprotectedAddEncryptionZone(Long inodeId, String keyName) {
- final EncryptionZoneInt ez = new EncryptionZoneInt(inodeId, keyName);
+ void unprotectedAddEncryptionZone(Long inodeId,
+ CipherSuite suite, String keyName) {
+ final EncryptionZoneInt ez = new EncryptionZoneInt(
+ inodeId, suite, keyName);
encryptionZones.put(inodeId, ez);
}
@@ -207,8 +218,8 @@ public class EncryptionZoneManager {
if (ezi == null) {
return NULL_EZ;
} else {
- return new EncryptionZone(getFullPathName(ezi), ezi.getKeyName(),
- ezi.getINodeId());
+ return new EncryptionZone(ezi.getINodeId(), getFullPathName(ezi),
+ ezi.getSuite(), ezi.getKeyName());
}
}
@@ -264,7 +275,7 @@ public class EncryptionZoneManager {
* <p/>
* Called while holding the FSDirectory lock.
*/
- XAttr createEncryptionZone(String src, String keyName)
+ XAttr createEncryptionZone(String src, CipherSuite suite, String keyName)
throws IOException {
assert dir.hasWriteLock();
if (dir.isNonEmptyDirectory(src)) {
@@ -284,8 +295,10 @@ public class EncryptionZoneManager {
"encryption zone. (" + getFullPathName(ezi) + ")");
}
+ final HdfsProtos.ZoneEncryptionInfoProto proto =
+ PBHelper.convert(suite, keyName);
final XAttr ezXAttr = XAttrHelper
- .buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, keyName.getBytes());
+ .buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, proto.toByteArray());
final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
xattrs.add(ezXAttr);
@@ -327,8 +340,8 @@ public class EncryptionZoneManager {
continue;
}
// Add the EZ to the result list
- zones.add(new EncryptionZone(pathName,
- ezi.getKeyName(), ezi.getINodeId()));
+ zones.add(new EncryptionZone(ezi.getINodeId(), pathName,
+ ezi.getSuite(), ezi.getKeyName()));
count++;
if (count >= numResponses) {
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 9346ea5..961808e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -37,6 +37,7 @@ import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileEncryptionInfo;
@@ -1402,9 +1403,10 @@ public class FSDirectory implements Closeable {
if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
return getSnapshotsListing(srcs, startAfter);
}
- final INodesInPath inodesInPath = getLastINodeInPath(srcs);
+ final INodesInPath inodesInPath = getINodesInPath(srcs, true);
+ final INode[] inodes = inodesInPath.getINodes();
final int snapshot = inodesInPath.getPathSnapshotId();
- final INode targetNode = inodesInPath.getLastINode();
+ final INode targetNode = inodes[inodes.length - 1];
if (targetNode == null)
return null;
byte parentStoragePolicy = isSuperUser ?
@@ -1414,7 +1416,7 @@ public class FSDirectory implements Closeable {
return new DirectoryListing(
new HdfsFileStatus[]{createFileStatus(HdfsFileStatus.EMPTY_NAME,
targetNode, needLocation, parentStoragePolicy, snapshot,
- isRawPath)}, 0);
+ isRawPath, inodesInPath)}, 0);
}
final INodeDirectory dirInode = targetNode.asDirectory();
@@ -1431,7 +1433,7 @@ public class FSDirectory implements Closeable {
cur.getLocalStoragePolicyID(): BlockStoragePolicy.ID_UNSPECIFIED;
listing[i] = createFileStatus(cur.getLocalNameBytes(), cur, needLocation,
getStoragePolicyID(curPolicy, parentStoragePolicy), snapshot,
- isRawPath);
+ isRawPath, inodesInPath);
listingCnt++;
if (needLocation) {
// Once we hit lsLimit locations, stop.
@@ -1482,7 +1484,8 @@ public class FSDirectory implements Closeable {
for (int i = 0; i < numOfListing; i++) {
Root sRoot = snapshots.get(i + skipSize).getRoot();
listing[i] = createFileStatus(sRoot.getLocalNameBytes(), sRoot,
- BlockStoragePolicy.ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, false);
+ BlockStoragePolicy.ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID,
+ false, null);
}
return new DirectoryListing(
listing, snapshots.size() - skipSize - numOfListing);
@@ -1505,12 +1508,14 @@ public class FSDirectory implements Closeable {
if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
return getFileInfo4DotSnapshot(srcs);
}
- final INodesInPath inodesInPath = getLastINodeInPath(srcs, resolveLink);
- final INode i = inodesInPath.getINode(0);
+ final INodesInPath inodesInPath = getINodesInPath(srcs, resolveLink);
+ final INode[] inodes = inodesInPath.getINodes();
+ final INode i = inodes[inodes.length - 1];
byte policyId = includeStoragePolicy && i != null && !i.isSymlink() ?
i.getStoragePolicyID() : BlockStoragePolicy.ID_UNSPECIFIED;
return i == null ? null : createFileStatus(HdfsFileStatus.EMPTY_NAME, i,
- policyId, inodesInPath.getPathSnapshotId(), isRawPath);
+ policyId, inodesInPath.getPathSnapshotId(), isRawPath,
+ inodesInPath);
} finally {
readUnlock();
}
@@ -2162,8 +2167,17 @@ public class FSDirectory implements Closeable {
for (XAttr xattr : xattrs) {
final String xaName = XAttrHelper.getPrefixName(xattr);
if (CRYPTO_XATTR_ENCRYPTION_ZONE.equals(xaName)) {
- ezManager.unprotectedAddEncryptionZone(inode.getId(),
- new String(xattr.getValue()));
+ try {
+ final HdfsProtos.ZoneEncryptionInfoProto ezProto =
+ HdfsProtos.ZoneEncryptionInfoProto.parseFrom(
+ xattr.getValue());
+ ezManager.unprotectedAddEncryptionZone(inode.getId(),
+ PBHelper.convert(ezProto.getSuite()),
+ ezProto.getKeyName());
+ } catch (InvalidProtocolBufferException e) {
+ NameNode.LOG.warn("Error parsing protocol buffer of " +
+ "EZ XAttr " + xattr.getName());
+ }
}
}
}
@@ -2355,12 +2369,15 @@ public class FSDirectory implements Closeable {
* @throws IOException if any error occurs
*/
private HdfsFileStatus createFileStatus(byte[] path, INode node,
- boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath)
+ boolean needLocation, byte storagePolicy, int snapshot,
+ boolean isRawPath, INodesInPath iip)
throws IOException {
if (needLocation) {
- return createLocatedFileStatus(path, node, storagePolicy, snapshot, isRawPath);
+ return createLocatedFileStatus(path, node, storagePolicy, snapshot,
+ isRawPath, iip);
} else {
- return createFileStatus(path, node, storagePolicy, snapshot, isRawPath);
+ return createFileStatus(path, node, storagePolicy, snapshot,
+ isRawPath, iip);
}
}
@@ -2368,14 +2385,14 @@ public class FSDirectory implements Closeable {
* Create FileStatus by file INode
*/
HdfsFileStatus createFileStatus(byte[] path, INode node, byte storagePolicy,
- int snapshot, boolean isRawPath) throws IOException {
+ int snapshot, boolean isRawPath, INodesInPath iip) throws IOException {
long size = 0; // length is zero for directories
short replication = 0;
long blocksize = 0;
final boolean isEncrypted;
final FileEncryptionInfo feInfo = isRawPath ? null :
- getFileEncryptionInfo(node, snapshot);
+ getFileEncryptionInfo(node, snapshot, iip);
if (node.isFile()) {
final INodeFile fileNode = node.asFile();
@@ -2413,7 +2430,8 @@ public class FSDirectory implements Closeable {
* Create FileStatus with location info by file INode
*/
private HdfsLocatedFileStatus createLocatedFileStatus(byte[] path, INode node,
- byte storagePolicy, int snapshot, boolean isRawPath) throws IOException {
+ byte storagePolicy, int snapshot, boolean isRawPath,
+ INodesInPath iip) throws IOException {
assert hasReadLock();
long size = 0; // length is zero for directories
short replication = 0;
@@ -2421,7 +2439,7 @@ public class FSDirectory implements Closeable {
LocatedBlocks loc = null;
final boolean isEncrypted;
final FileEncryptionInfo feInfo = isRawPath ? null :
- getFileEncryptionInfo(node, snapshot);
+ getFileEncryptionInfo(node, snapshot, iip);
if (node.isFile()) {
final INodeFile fileNode = node.asFile();
size = fileNode.computeFileSize(snapshot);
@@ -2746,11 +2764,11 @@ public class FSDirectory implements Closeable {
}
}
- XAttr createEncryptionZone(String src, String keyName)
+ XAttr createEncryptionZone(String src, CipherSuite suite, String keyName)
throws IOException {
writeLock();
try {
- return ezManager.createEncryptionZone(src, keyName);
+ return ezManager.createEncryptionZone(src, suite, keyName);
} finally {
writeUnlock();
}
@@ -2781,7 +2799,8 @@ public class FSDirectory implements Closeable {
void setFileEncryptionInfo(String src, FileEncryptionInfo info)
throws IOException {
// Make the PB for the xattr
- final HdfsProtos.FileEncryptionInfoProto proto = PBHelper.convert(info);
+ final HdfsProtos.PerFileEncryptionInfoProto proto =
+ PBHelper.convertPerFileEncInfo(info);
final byte[] protoBytes = proto.toByteArray();
final XAttr fileEncryptionAttr =
XAttrHelper.buildXAttr(CRYPTO_XATTR_FILE_ENCRYPTION_INFO, protoBytes);
@@ -2797,35 +2816,64 @@ public class FSDirectory implements Closeable {
}
/**
- * Return the FileEncryptionInfo for an INode, or null if the INode is not
- * an encrypted file.
- */
- FileEncryptionInfo getFileEncryptionInfo(INode inode, int snapshotId)
- throws IOException {
+ * This function combines the per-file encryption info (obtained
+ * from the inode's XAttrs), and the encryption info from its zone, and
+ * returns a consolidated FileEncryptionInfo instance. Null is returned
+ * for non-encrypted files.
+ *
+ * @param inode inode of the file
+ * @param snapshotId ID of the snapshot that
+ * we want to get encryption info from
+ * @param iip inodes in the path containing the file, passed in to
+ * avoid obtaining the list of inodes again; if iip is
+ * null then the list of inodes will be obtained again
+ * @return consolidated file encryption info; null for non-encrypted files
+ */
+ FileEncryptionInfo getFileEncryptionInfo(INode inode, int snapshotId,
+ INodesInPath iip) throws IOException {
if (!inode.isFile()) {
return null;
}
readLock();
try {
- List<XAttr> xAttrs = XAttrStorage.readINodeXAttrs(inode, snapshotId);
- if (xAttrs == null) {
- return null;
+ if (iip == null) {
+ iip = getINodesInPath(inode.getFullPathName(), true);
}
- for (XAttr x : xAttrs) {
- if (XAttrHelper.getPrefixName(x)
- .equals(CRYPTO_XATTR_FILE_ENCRYPTION_INFO)) {
- try {
- HdfsProtos.FileEncryptionInfoProto proto =
- HdfsProtos.FileEncryptionInfoProto.parseFrom(x.getValue());
- FileEncryptionInfo feInfo = PBHelper.convert(proto);
- return feInfo;
- } catch (InvalidProtocolBufferException e) {
- throw new IOException("Could not parse file encryption info for " +
- "inode " + inode, e);
- }
+ EncryptionZone encryptionZone = getEZForPath(iip);
+ if (encryptionZone == null ||
+ encryptionZone.equals(EncryptionZoneManager.NULL_EZ)) {
+ // not an encrypted file
+ return null;
+ } else if(encryptionZone.getPath() == null
+ || encryptionZone.getPath().isEmpty()) {
+ if (NameNode.LOG.isDebugEnabled()) {
+ NameNode.LOG.debug("Encryption zone " +
+ encryptionZone.getPath() + " does not have a valid path.");
}
}
- return null;
+
+ CipherSuite suite = encryptionZone.getSuite();
+ String keyName = encryptionZone.getKeyName();
+
+ XAttr fileXAttr = unprotectedGetXAttrByName(inode, snapshotId,
+ CRYPTO_XATTR_FILE_ENCRYPTION_INFO);
+
+ if (fileXAttr == null) {
+ NameNode.LOG.warn("Could not find encryption XAttr for file " +
+ inode.getFullPathName() + " in encryption zone " +
+ encryptionZone.getPath());
+ return null;
+ }
+
+ try {
+ HdfsProtos.PerFileEncryptionInfoProto fileProto =
+ HdfsProtos.PerFileEncryptionInfoProto.parseFrom(
+ fileXAttr.getValue());
+ return PBHelper.convert(fileProto, suite, keyName);
+ } catch (InvalidProtocolBufferException e) {
+ throw new IOException("Could not parse file encryption info for " +
+ "inode " + inode, e);
+ }
} finally {
readUnlock();
}
@@ -2860,7 +2908,11 @@ public class FSDirectory implements Closeable {
* of encryption zones.
*/
if (CRYPTO_XATTR_ENCRYPTION_ZONE.equals(xaName)) {
- ezManager.addEncryptionZone(inode.getId(), new String(xattr.getValue()));
+ final HdfsProtos.ZoneEncryptionInfoProto ezProto =
+ HdfsProtos.ZoneEncryptionInfoProto.parseFrom(xattr.getValue());
+ ezManager.addEncryptionZone(inode.getId(),
+ PBHelper.convert(ezProto.getSuite()),
+ ezProto.getKeyName());
}
if (!isFile && SECURITY_XATTR_UNREADABLE_BY_SUPERUSER.equals(xaName)) {
@@ -2977,6 +3029,22 @@ public class FSDirectory implements Closeable {
return XAttrStorage.readINodeXAttrs(inode, snapshotId);
}
+ private XAttr unprotectedGetXAttrByName(INode inode, int snapshotId,
+ String xAttrName)
+ throws IOException {
+ List<XAttr> xAttrs = XAttrStorage.readINodeXAttrs(inode, snapshotId);
+ if (xAttrs == null) {
+ return null;
+ }
+ for (XAttr x : xAttrs) {
+ if (XAttrHelper.getPrefixName(x)
+ .equals(xAttrName)) {
+ return x;
+ }
+ }
+ return null;
+ }
+
private static INode resolveLastINode(String src, INodesInPath iip)
throws FileNotFoundException {
INode inode = iip.getLastINode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 46edf54..cc0572e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -341,8 +341,10 @@ public class FSEditLogLoader {
// 3. OP_ADD to open file for append
// See if the file already exists (persistBlocks call)
- final INodesInPath iip = fsDir.getLastINodeInPath(path);
- INodeFile oldFile = INodeFile.valueOf(iip.getINode(0), path, true);
+ final INodesInPath iip = fsDir.getINodesInPath(path, true);
+ final INode[] inodes = iip.getINodes();
+ INodeFile oldFile = INodeFile.valueOf(
+ inodes[inodes.length - 1], path, true);
if (oldFile != null && addCloseOp.overwrite) {
// This is OP_ADD with overwrite
fsDir.unprotectedDelete(path, addCloseOp.mtime);
@@ -372,7 +374,7 @@ public class FSEditLogLoader {
HdfsFileStatus stat = fsNamesys.dir.createFileStatus(
HdfsFileStatus.EMPTY_NAME, newFile,
BlockStoragePolicy.ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID,
- false);
+ false, iip);
fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId,
addCloseOp.rpcCallId, stat);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 4dc2786..876cf49 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import static org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import static org.apache.hadoop.crypto.key.KeyProviderCryptoExtension
.EncryptedKeyVersion;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
@@ -134,6 +133,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CipherSuite;
+import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import org.apache.hadoop.fs.CacheFlag;
@@ -165,7 +165,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.UnknownCipherSuiteException;
import org.apache.hadoop.hdfs.protocol.AclException;
@@ -1833,8 +1832,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
doAccessTime = false;
}
- final INodesInPath iip = dir.getLastINodeInPath(src);
- final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src);
+ final INodesInPath iip = dir.getINodesInPath(src, true);
+ final INode[] inodes = iip.getINodes();
+ final INodeFile inode = INodeFile.valueOf(
+ inodes[inodes.length - 1], src);
if (isPermissionEnabled) {
checkUnreadableBySuperuser(pc, inode, iip.getPathSnapshotId());
}
@@ -1867,7 +1868,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
final FileEncryptionInfo feInfo =
FSDirectory.isReservedRawName(srcArg) ?
- null : dir.getFileEncryptionInfo(inode, iip.getPathSnapshotId());
+ null : dir.getFileEncryptionInfo(inode, iip.getPathSnapshotId(),
+ iip);
final LocatedBlocks blocks =
blockManager.createLocatedBlocks(inode.getBlocks(), fileSize,
@@ -2599,7 +2601,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
feInfo = new FileEncryptionInfo(suite,
edek.getEncryptedKeyVersion().getMaterial(),
edek.getEncryptedKeyIv(),
- edek.getEncryptionKeyVersionName());
+ ezKeyName, edek.getEncryptionKeyVersionName());
Preconditions.checkNotNull(feInfo);
}
@@ -8619,8 +8621,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
throw new IOException("Must specify a key name when creating an " +
"encryption zone");
}
- KeyVersion keyVersion = provider.getCurrentKey(keyName);
- if (keyVersion == null) {
+ KeyProvider.Metadata metadata = provider.getMetadata(keyName);
+ if (metadata == null) {
/*
* It would be nice if we threw something more specific than
* IOException when the key is not found, but the KeyProvider API
@@ -8631,7 +8633,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
*/
throw new IOException("Key " + keyName + " doesn't exist.");
}
- createEncryptionZoneInt(src, keyName, cacheEntry != null);
+ createEncryptionZoneInt(src, metadata.getCipher(),
+ keyName, cacheEntry != null);
success = true;
} catch (AccessControlException e) {
logAuditEvent(false, "createEncryptionZone", src);
@@ -8641,8 +8644,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
}
- private void createEncryptionZoneInt(final String srcArg, String keyName,
- final boolean logRetryCache) throws IOException {
+ private void createEncryptionZoneInt(final String srcArg, String cipher,
+ String keyName, final boolean logRetryCache) throws IOException {
String src = srcArg;
HdfsFileStatus resultingStat = null;
checkSuperuserPrivilege();
@@ -8656,7 +8659,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
checkNameNodeSafeMode("Cannot create encryption zone on " + src);
src = resolvePath(src, pathComponents);
- final XAttr ezXAttr = dir.createEncryptionZone(src, keyName);
+ final CipherSuite suite = CipherSuite.convert(cipher);
+ final XAttr ezXAttr = dir.createEncryptionZone(src, suite, keyName);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(ezXAttr);
getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index 34a3268..ba052a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -133,6 +133,7 @@ public class INodesInPath {
* be thrown when the path refers to a symbolic link.
* @return the specified number of existing INodes in the path
*/
+ // TODO: Eliminate null elements from inodes (to be provided by HDFS-7104)
static INodesInPath resolve(final INodeDirectory startingDir,
final byte[][] components, final int numOfINodes,
final boolean resolveLink) throws UnresolvedLinkException {
@@ -311,7 +312,7 @@ public class INodesInPath {
}
/**
- * @return the whole inodes array including the null elements.
+ * @return the inodes array excluding the null elements.
*/
INode[] getINodes() {
if (capacity < inodes.length) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto
index 1a33cdc..c4b7009 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto
@@ -46,9 +46,10 @@ message ListEncryptionZonesRequestProto {
}
message EncryptionZoneProto {
- required string path = 1;
- required string keyName = 2;
- required int64 id = 3;
+ required int64 id = 1;
+ required string path = 2;
+ required CipherSuite suite = 3;
+ required string keyName = 4;
}
message ListEncryptionZonesResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1737950d/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
index f12055c..d1ba68f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
@@ -216,7 +216,27 @@ message FileEncryptionInfoProto {
required CipherSuite suite = 1;
required bytes key = 2;
required bytes iv = 3;
- required string ezKeyVersionName = 4;
+ required string keyName = 4;
+ required string ezKeyVersionName = 5;
+}
+
+/**
+ * Encryption information for an individual
+ * file within an encryption zone
+ */
+message PerFileEncryptionInfoProto {
+ required bytes key = 1;
+ required bytes iv = 2;
+ required string ezKeyVersionName = 3;
+}
+
+/**
+ * Encryption information for an encryption
+ * zone
+ */
+message ZoneEncryptionInfoProto {
+ required CipherSuite suite = 1;
+ required string keyName = 2;
}
/**
[12/50] [abbrv] git commit: HDFS-7096. Fix TestRpcProgramNfs3 to use
DFS_ENCRYPTION_KEY_PROVIDER_URI (clamb via cmccabe)
Posted by cu...@apache.org.
HDFS-7096. Fix TestRpcProgramNfs3 to use DFS_ENCRYPTION_KEY_PROVIDER_URI (clamb via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25fd69a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25fd69a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25fd69a8
Branch: refs/heads/YARN-1051
Commit: 25fd69a8f9defb21a5c0e3b7243657d9e8080319
Parents: bf27b9c
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Fri Sep 19 09:16:05 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Fri Sep 19 09:20:25 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 3 ++-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25fd69a8/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
index 68efac2..acd47fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -131,7 +132,7 @@ public class TestRpcProgramNfs3 {
String testRoot = fsHelper.getTestRootDir();
testRootDir = new File(testRoot).getAbsoluteFile();
final Path jksPath = new Path(testRootDir.toString(), "test.jks");
- config.set(KeyProviderFactory.KEY_PROVIDER_PATH,
+ config.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
ProxyUsers.refreshSuperUserGroupsConfiguration(config);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25fd69a8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ba1a272..9d4d8ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -779,6 +779,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7065. Pipeline close recovery race can cause block corruption (kihwal)
+ HDFS-7096. Fix TestRpcProgramNfs3 to use DFS_ENCRYPTION_KEY_PROVIDER_URI
+ (clamb via cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
[34/50] [abbrv] YARN-1372. Ensure all completed containers are
reported to the AMs across RM restart. Contributed by Anubhav Dhoot
Posted by cu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index b8e6f43..15028f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -28,6 +28,7 @@ import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -35,6 +36,7 @@ import static org.mockito.Mockito.when;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -91,6 +93,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent;
+
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -151,6 +158,7 @@ public class TestRMAppAttemptTransitions {
private NMTokenSecretManagerInRM nmTokenManager =
spy(new NMTokenSecretManagerInRM(conf));
private boolean transferStateFromPreviousAttempt = false;
+ private EventHandler<RMNodeEvent> rmnodeEventHandler;
private final class TestApplicationAttemptEventDispatcher implements
EventHandler<RMAppAttemptEvent> {
@@ -203,7 +211,7 @@ public class TestRMAppAttemptTransitions {
applicationMasterLauncher.handle(event);
}
}
-
+
private static int appId = 1;
private ApplicationSubmissionContext submissionContext = null;
@@ -268,6 +276,9 @@ public class TestRMAppAttemptTransitions {
rmDispatcher.register(AMLauncherEventType.class,
new TestAMLauncherEventDispatcher());
+ rmnodeEventHandler = mock(RMNodeImpl.class);
+ rmDispatcher.register(RMNodeEventType.class, rmnodeEventHandler);
+
rmDispatcher.init(conf);
rmDispatcher.start();
@@ -575,6 +586,8 @@ public class TestRMAppAttemptTransitions {
}
assertEquals(finishedContainerCount, applicationAttempt
.getJustFinishedContainers().size());
+ Assert.assertEquals(0, getFinishedContainersSentToAM(applicationAttempt)
+ .size());
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(finalStatus, applicationAttempt.getFinalApplicationStatus());
verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
@@ -704,7 +717,8 @@ public class TestRMAppAttemptTransitions {
application.handle(new RMAppRunningOnNodeEvent(application.getApplicationId(),
container.getNodeId()));
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
- applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class)));
+ applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class),
+ container.getNodeId()));
// complete AM
String diagnostics = "Successful";
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
@@ -752,10 +766,11 @@ public class TestRMAppAttemptTransitions {
when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
sendAttemptUpdateSavedEvent(applicationAttempt);
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
attemptId,
ContainerStatus.newInstance(
- amContainer.getId(), ContainerState.COMPLETE, "", 0)));
+ amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);
@@ -857,8 +872,9 @@ public class TestRMAppAttemptTransitions {
SchedulerUtils.LOST_CONTAINER);
// send CONTAINER_FINISHED event at SCHEDULED state,
// The state should be FINAL_SAVING with previous state SCHEDULED
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
- applicationAttempt.getAppAttemptId(), cs));
+ applicationAttempt.getAppAttemptId(), cs, anyNodeId));
// createApplicationAttemptState will return previous state (SCHEDULED),
// if the current state is FINAL_SAVING.
assertEquals(YarnApplicationAttemptState.SCHEDULED,
@@ -904,8 +920,9 @@ public class TestRMAppAttemptTransitions {
ContainerStatus cs =
BuilderUtils.newContainerStatus(amContainer.getId(),
ContainerState.COMPLETE, containerDiagMsg, exitCode);
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
- applicationAttempt.getAppAttemptId(), cs));
+ applicationAttempt.getAppAttemptId(), cs, anyNodeId));
assertEquals(YarnApplicationAttemptState.ALLOCATED,
applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
@@ -928,16 +945,17 @@ public class TestRMAppAttemptTransitions {
ContainerStatus cs = BuilderUtils.newContainerStatus(amContainer.getId(),
ContainerState.COMPLETE, containerDiagMsg, exitCode);
ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
- appAttemptId, cs));
+ appAttemptId, cs, anyNodeId));
// ignored ContainerFinished and Expire at FinalSaving if we were supposed
// to Failed state.
assertEquals(RMAppAttemptState.FINAL_SAVING,
- applicationAttempt.getAppAttemptState());
+ applicationAttempt.getAppAttemptState());
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
- amContainer.getId(), ContainerState.COMPLETE, "", 0)));
+ amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
applicationAttempt.handle(new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
assertEquals(RMAppAttemptState.FINAL_SAVING,
@@ -947,7 +965,7 @@ public class TestRMAppAttemptTransitions {
sendAttemptUpdateSavedEvent(applicationAttempt);
assertEquals(RMAppAttemptState.FAILED,
applicationAttempt.getAppAttemptState());
- assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
+ assertEquals(2, applicationAttempt.getJustFinishedContainers().size());
assertEquals(amContainer, applicationAttempt.getMasterContainer());
assertEquals(0, application.getRanNodes().size());
String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app",
@@ -971,10 +989,11 @@ public class TestRMAppAttemptTransitions {
// ignored ContainerFinished and Expire at FinalSaving if we were supposed
// to Killed state.
assertEquals(RMAppAttemptState.FINAL_SAVING,
- applicationAttempt.getAppAttemptState());
+ applicationAttempt.getAppAttemptState());
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
- amContainer.getId(), ContainerState.COMPLETE, "", 0)));
+ amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
applicationAttempt.handle(new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
assertEquals(RMAppAttemptState.FINAL_SAVING,
@@ -984,7 +1003,7 @@ public class TestRMAppAttemptTransitions {
sendAttemptUpdateSavedEvent(applicationAttempt);
assertEquals(RMAppAttemptState.KILLED,
applicationAttempt.getAppAttemptState());
- assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
+ assertEquals(1,applicationAttempt.getJustFinishedContainers().size());
assertEquals(amContainer, applicationAttempt.getMasterContainer());
assertEquals(0, application.getRanNodes().size());
String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app",
@@ -1144,13 +1163,14 @@ public class TestRMAppAttemptTransitions {
unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl,
diagnostics);
// container must be AM container to move from FINISHING to FINISHED
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(
new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(),
BuilderUtils.newContainerStatus(
BuilderUtils.newContainerId(
applicationAttempt.getAppAttemptId(), 42),
- ContainerState.COMPLETE, "", 0)));
+ ContainerState.COMPLETE, "", 0), anyNodeId));
testAppAttemptFinishingState(amContainer, finalStatus, trackingUrl,
diagnostics);
}
@@ -1165,13 +1185,14 @@ public class TestRMAppAttemptTransitions {
String diagnostics = "Successful";
unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl,
diagnostics);
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(
new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(),
BuilderUtils.newContainerStatus(amContainer.getId(),
- ContainerState.COMPLETE, "", 0)));
+ ContainerState.COMPLETE, "", 0), anyNodeId));
testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
- diagnostics, 0, false);
+ diagnostics, 1, false);
}
// While attempt is at FINAL_SAVING, Contaienr_Finished event may come before
@@ -1195,15 +1216,16 @@ public class TestRMAppAttemptTransitions {
assertEquals(YarnApplicationAttemptState.RUNNING,
applicationAttempt.createApplicationAttemptState());
// Container_finished event comes before Attempt_Saved event.
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
- amContainer.getId(), ContainerState.COMPLETE, "", 0)));
+ amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
// send attempt_saved
sendAttemptUpdateSavedEvent(applicationAttempt);
testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
- diagnostics, 0, false);
+ diagnostics, 1, false);
}
// While attempt is at FINAL_SAVING, Expire event may come before
@@ -1235,6 +1257,71 @@ public class TestRMAppAttemptTransitions {
diagnostics, 0, false);
}
+ @Test
+ public void testFinishedContainer() {
+ Container amContainer = allocateApplicationAttempt();
+ launchApplicationAttempt(amContainer);
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
+
+ // Complete one container
+ ContainerId containerId1 = BuilderUtils.newContainerId(applicationAttempt
+ .getAppAttemptId(), 2);
+ Container container1 = mock(Container.class);
+ ContainerStatus containerStatus1 = mock(ContainerStatus.class);
+ when(container1.getId()).thenReturn(
+ containerId1);
+ when(containerStatus1.getContainerId()).thenReturn(containerId1);
+ when(container1.getNodeId()).thenReturn(NodeId.newInstance("host", 1234));
+
+ application.handle(new RMAppRunningOnNodeEvent(application
+ .getApplicationId(),
+ container1.getNodeId()));
+ applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
+ applicationAttempt.getAppAttemptId(), containerStatus1,
+ container1.getNodeId()));
+
+ ArgumentCaptor<RMNodeFinishedContainersPulledByAMEvent> captor =
+ ArgumentCaptor.forClass(RMNodeFinishedContainersPulledByAMEvent.class);
+
+ // Verify justFinishedContainers
+ Assert.assertEquals(1, applicationAttempt.getJustFinishedContainers()
+ .size());
+ Assert.assertEquals(container1.getId(), applicationAttempt
+ .getJustFinishedContainers().get(0).getContainerId());
+ Assert.assertEquals(0, getFinishedContainersSentToAM(applicationAttempt)
+ .size());
+
+ // Verify finishedContainersSentToAM gets container after pull
+ List<ContainerStatus> containerStatuses = applicationAttempt
+ .pullJustFinishedContainers();
+ Assert.assertEquals(1, containerStatuses.size());
+ Mockito.verify(rmnodeEventHandler, never()).handle(Mockito
+ .any(RMNodeEvent.class));
+ Assert.assertTrue(applicationAttempt.getJustFinishedContainers().isEmpty());
+ Assert.assertEquals(1, getFinishedContainersSentToAM(applicationAttempt)
+ .size());
+
+ // Verify container is acked to NM via the RMNodeEvent after second pull
+ containerStatuses = applicationAttempt.pullJustFinishedContainers();
+ Assert.assertEquals(0, containerStatuses.size());
+ Mockito.verify(rmnodeEventHandler).handle(captor.capture());
+ Assert.assertEquals(container1.getId(), captor.getValue().getContainers()
+ .get(0));
+ Assert.assertTrue(applicationAttempt.getJustFinishedContainers().isEmpty());
+ Assert.assertEquals(0, getFinishedContainersSentToAM(applicationAttempt)
+ .size());
+ }
+
+ private static List<ContainerStatus> getFinishedContainersSentToAM(
+ RMAppAttempt applicationAttempt) {
+ List<ContainerStatus> containers = new ArrayList<ContainerStatus>();
+ for (List<ContainerStatus> containerStatuses: applicationAttempt
+ .getFinishedContainersSentToAMReference().values()) {
+ containers.addAll(containerStatuses);
+ }
+ return containers;
+ }
+
// this is to test user can get client tokens only after the client token
// master key is saved in the state store and also registered in
// ClientTokenSecretManager
@@ -1281,8 +1368,9 @@ public class TestRMAppAttemptTransitions {
ContainerStatus.newInstance(amContainer.getId(),
ContainerState.COMPLETE, "some error", 123);
ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
- appAttemptId, cs1));
+ appAttemptId, cs1, anyNodeId));
assertEquals(YarnApplicationAttemptState.RUNNING,
applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
@@ -1293,15 +1381,21 @@ public class TestRMAppAttemptTransitions {
verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
// failed attempt captured the container finished event.
- assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
+ assertEquals(1, applicationAttempt.getJustFinishedContainers().size());
ContainerStatus cs2 =
ContainerStatus.newInstance(ContainerId.newInstance(appAttemptId, 2),
ContainerState.COMPLETE, "", 0);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
- appAttemptId, cs2));
- assertEquals(1, applicationAttempt.getJustFinishedContainers().size());
- assertEquals(cs2.getContainerId(), applicationAttempt
- .getJustFinishedContainers().get(0).getContainerId());
+ appAttemptId, cs2, anyNodeId));
+ assertEquals(2, applicationAttempt.getJustFinishedContainers().size());
+ boolean found = false;
+ for (ContainerStatus containerStatus:applicationAttempt
+ .getJustFinishedContainers()) {
+ if (cs2.getContainerId().equals(containerStatus.getContainerId())) {
+ found = true;
+ }
+ }
+ assertTrue(found);
}
@@ -1322,8 +1416,9 @@ public class TestRMAppAttemptTransitions {
ContainerStatus.newInstance(amContainer.getId(),
ContainerState.COMPLETE, "some error", 123);
ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
- appAttemptId, cs1));
+ appAttemptId, cs1, anyNodeId));
assertEquals(YarnApplicationAttemptState.RUNNING,
applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a641496/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
index b3dc35f..a0d5d84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
@@ -161,7 +161,7 @@ public class TestAMRMTokens {
.getEventHandler()
.handle(
new RMAppAttemptContainerFinishedEvent(applicationAttemptId,
- containerStatus));
+ containerStatus, nm1.getNodeId()));
// Make sure the RMAppAttempt is at Finished State.
// Both AMRMToken and ClientToAMToken have been removed.
[50/50] [abbrv] git commit: Rebase against trunk
Posted by cu...@apache.org.
Rebase against trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e82461be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e82461be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e82461be
Branch: refs/heads/YARN-1051
Commit: e82461be56fab0c3b5188219c8fc09946b0826ba
Parents: e53fdea f00e7af
Author: carlo curino <Carlo Curino>
Authored: Mon Sep 22 16:46:56 2014 -0700
Committer: carlo curino <Carlo Curino>
Committed: Mon Sep 22 16:46:56 2014 -0700
----------------------------------------------------------------------
.../yarn/client/api/impl/TestYarnClient.java | 97 ++++++++++++++++++++
1 file changed, 97 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e82461be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --cc hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index d7bea7a,5607b6e..c1f452c
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@@ -808,66 -800,52 +808,163 @@@ public class TestYarnClient
public boolean isSecurityEnabled() {
return true;
}
- };
+ });
client.init(conf);
client.start();
- ApplicationSubmissionContext context =
- mock(ApplicationSubmissionContext.class);
- ApplicationId applicationId = ApplicationId.newInstance(0, 1);
- when(context.getApplicationId()).thenReturn(applicationId);
- DataOutputBuffer dob = new DataOutputBuffer();
- Credentials credentials = new Credentials();
- credentials.writeTokenStorageToStream(dob);
- ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
- null, null, null, null, tokens, null);
- when(context.getAMContainerSpec()).thenReturn(clc);
- client.submitApplication(context);
- // Check whether token is added or not
- credentials = new Credentials();
- DataInputByteBuffer dibb = new DataInputByteBuffer();
- tokens = clc.getTokens();
- if (tokens != null) {
- dibb.reset(tokens);
- credentials.readTokenStorageStream(dibb);
- tokens.rewind();
+ try {
+ // when i == 0, timeline DT already exists, no need to get one more
+ // when i == 1, timeline DT doesn't exist, need to get one more
+ for (int i = 0; i < 2; ++i) {
+ ApplicationSubmissionContext context =
+ mock(ApplicationSubmissionContext.class);
+ ApplicationId applicationId = ApplicationId.newInstance(0, i + 1);
+ when(context.getApplicationId()).thenReturn(applicationId);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ Credentials credentials = new Credentials();
+ if (i == 0) {
+ credentials.addToken(client.timelineService, dToken);
+ }
+ credentials.writeTokenStorageToStream(dob);
+ ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
+ null, null, null, null, tokens, null);
+ when(context.getAMContainerSpec()).thenReturn(clc);
+ client.submitApplication(context);
+ if (i == 0) {
+ // GetTimelineDelegationToken shouldn't be called
+ verify(client, never()).getTimelineDelegationToken();
+ }
+ // In either way, token should be there
+ credentials = new Credentials();
+ DataInputByteBuffer dibb = new DataInputByteBuffer();
+ tokens = clc.getTokens();
+ if (tokens != null) {
+ dibb.reset(tokens);
+ credentials.readTokenStorageStream(dibb);
+ tokens.rewind();
+ }
+ Collection<Token<? extends TokenIdentifier>> dTokens =
+ credentials.getAllTokens();
+ Assert.assertEquals(1, dTokens.size());
+ Assert.assertEquals(dToken, dTokens.iterator().next());
+ }
+ } finally {
+ client.stop();
}
- Collection<Token<? extends TokenIdentifier>> dTokens =
- credentials.getAllTokens();
- Assert.assertEquals(1, dTokens.size());
- Assert.assertEquals(dToken, dTokens.iterator().next());
- client.stop();
+ }
+
+ @Test
+ public void testReservationAPIs() {
+ // initialize
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ ReservationSystemTestUtil.setupQueueConfiguration(conf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
+ MiniYARNCluster cluster =
+ new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
+ YarnClient client = null;
+ try {
+ cluster.init(conf);
+ cluster.start();
+ final Configuration yarnConf = cluster.getConfig();
++ client = YarnClient.createYarnClient();
++ client.init(yarnConf);
++ client.start();
++
++ // create a reservation
++ Clock clock = new UTCClock();
++ long arrival = clock.getTime();
++ long duration = 60000;
++ long deadline = (long) (arrival + 1.05 * duration);
++ ReservationSubmissionRequest sRequest =
++ createSimpleReservationRequest(4, arrival, deadline, duration);
++ ReservationSubmissionResponse sResponse = null;
++ try {
++ sResponse = client.submitReservation(sRequest);
++ } catch (Exception e) {
++ Assert.fail(e.getMessage());
++ }
++ Assert.assertNotNull(sResponse);
++ ReservationId reservationID = sResponse.getReservationId();
++ Assert.assertNotNull(reservationID);
++ System.out.println("Submit reservation response: " + reservationID);
++
++ // Update the reservation
++ ReservationDefinition rDef = sRequest.getReservationDefinition();
++ ReservationRequest rr =
++ rDef.getReservationRequests().getReservationResources().get(0);
++ rr.setNumContainers(5);
++ arrival = clock.getTime();
++ duration = 30000;
++ deadline = (long) (arrival + 1.05 * duration);
++ rr.setDuration(duration);
++ rDef.setArrival(arrival);
++ rDef.setDeadline(deadline);
++ ReservationUpdateRequest uRequest =
++ ReservationUpdateRequest.newInstance(rDef, reservationID);
++ ReservationUpdateResponse uResponse = null;
++ try {
++ uResponse = client.updateReservation(uRequest);
++ } catch (Exception e) {
++ Assert.fail(e.getMessage());
++ }
++ Assert.assertNotNull(sResponse);
++ System.out.println("Update reservation response: " + uResponse);
++
++ // Delete the reservation
++ ReservationDeleteRequest dRequest =
++ ReservationDeleteRequest.newInstance(reservationID);
++ ReservationDeleteResponse dResponse = null;
++ try {
++ dResponse = client.deleteReservation(dRequest);
++ } catch (Exception e) {
++ Assert.fail(e.getMessage());
++ }
++ Assert.assertNotNull(sResponse);
++ System.out.println("Delete reservation response: " + dResponse);
++ } finally {
++ // clean-up
++ if (client != null) {
++ client.stop();
++ }
++ cluster.stop();
++ }
++ }
++
++ private ReservationSubmissionRequest createSimpleReservationRequest(
++ int numContainers, long arrival, long deadline, long duration) {
++ // create a request with a single atomic ask
++ ReservationRequest r =
++ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
++ numContainers, 1, duration);
++ ReservationRequests reqs =
++ ReservationRequests.newInstance(Collections.singletonList(r),
++ ReservationRequestInterpreter.R_ALL);
++ ReservationDefinition rDef =
++ ReservationDefinition.newInstance(arrival, deadline, reqs,
++ "testYarnClient#reservation");
++ ReservationSubmissionRequest request =
++ ReservationSubmissionRequest.newInstance(rDef,
++ ReservationSystemTestUtil.reservationQ);
++ return request;
+ }
+
+ @Test
+ public void testReservationAPIs() {
+ // initialize
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ ReservationSystemTestUtil.setupQueueConfiguration(conf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
+ MiniYARNCluster cluster =
+ new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
+ YarnClient client = null;
+ try {
+ cluster.init(conf);
+ cluster.start();
+ final Configuration yarnConf = cluster.getConfig();
client = YarnClient.createYarnClient();
client.init(yarnConf);
client.start();
[03/50] [abbrv] git commit: Fix
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt for HDFS-6584 after merge.
Posted by cu...@apache.org.
Fix hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt for HDFS-6584 after
merge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd3cddf3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd3cddf3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd3cddf3
Branch: refs/heads/YARN-1051
Commit: fd3cddf3640a0dbd14556368ae4c6e803083bcfc
Parents: 98f7bfe
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Fri Sep 19 07:33:43 2014 +0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Fri Sep 19 07:33:43 2014 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 155 ++++++++++++-----------
1 file changed, 79 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd3cddf3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 56d7cbc..df3d3f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1,81 +1,5 @@
Hadoop HDFS Change Log
-HDFS-6584: Archival Storage
-
- HDFS-6677. Change INodeFile and FSImage to support storage policy ID.
- (szetszwo)
-
- HDFS-6670. Add block storage policy support with default HOT, WARM and COLD
- policies. (szetszwo)
-
- HDFS-6671. Change BlockPlacementPolicy to consider block storage policy
- in replicaiton. (szetszwo)
-
- HDFS-6710. Change BlockPlacementPolicy to consider block storage policy
- in replica deletion. (szetszwo)
-
- HDFS-6679. Bump NameNodeLayoutVersion and update editsStored test files.
- (vinayakumarb via szetszwo)
-
- HDFS-6686. Change BlockPlacementPolicy to use fallback when some storage
- types are unavailable. (szetszwo)
-
- HDFS-6835. Add a new API to set storage policy. (jing9)
-
- HDFS-6847. Support storage policy on directories and include storage policy
- in HdfsFileStatus. (Jing Zhao via szetszwo)
-
- HDFS-6801. Add a new data migration tool, Mover, for archiving data.
- (szetszwo via jing9)
-
- HDFS-6863. Support migration for snapshot paths. (jing9)
-
- HDFS-6906. Add more tests for BlockStoragePolicy. (szetszwo via jing9)
-
- HDFS-6911. check if a block is already scheduled in Mover.
- (szetszwo via jing9)
-
- HDFS-6920. Check the storage type of delNodeHintStorage when deleting
- a replica. (szetszwo via jing9)
-
- HDFS-6944. Add retry and termination logic for Mover. (jing9)
-
- HDFS-6969. INode#getStoragePolicyID should always return the latest
- storage policy. (jing9)
-
- HDFS-6961. BlockPlacementPolicy#chooseTarget should check each valid
- storage type in each choosing round. (jing9)
-
- HDFS-6876. support set/get storage policy in DFSAdmin. (jing9)
-
- HDFS-6997. Add more tests for data migration and replicaion. (szetszwo)
-
- HDFS-6875. Support migration for a list of specified paths. (jing9)
-
- HDFS-7027. Mover does not terminate when some storage type is out of space.
- (szetszwo via jing9)
-
- HDFS-7029. Fix TestDFSInotifyEventInputStream and TestDistributedFileSystem.
- (szetszwo via jing9)
-
- HDFS-7028. FSDirectory should not get storage policy id from symlinks.
- (szetszwo)
-
- HDFS-7034. Fix TestBlockPlacement and TestStorageMover. (jing9)
-
- HDFS-7039. Fix Balancer tests. (szetszwo via jing9)
-
- HDFS-7062. Skip under construction block for migration. (jing9)
-
- HDFS-7052. Add Mover into hdfs script. (jing9)
-
- HDFS-7072. Fix TestBlockManager and TestStorageMover. (jing9 via szetszwo)
-
- HDFS-6864. Archival Storage: add user documentation. (szetszwo via jing9)
-
- HDFS-7088. Archival Storage: fix TestBalancer and
- TestBalancerWithMultipleNameNodes. (szetszwo via jing9)
-
Trunk (Unreleased)
INCOMPATIBLE CHANGES
@@ -94,6 +18,9 @@ Trunk (Unreleased)
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
+ HDFS-6584. Support archival storage; see the detail subtasks in the
+ breakdown section. (szetszwo)
+
IMPROVEMENTS
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
@@ -337,6 +264,82 @@ Trunk (Unreleased)
HDFS-6981. Fix DN upgrade with layout version change. (Arpit Agarwal)
+ BREAKDOWN OF HDFS-6584 ARCHIVAL STORAGE
+
+ HDFS-6677. Change INodeFile and FSImage to support storage policy ID.
+ (szetszwo)
+
+ HDFS-6670. Add block storage policy support with default HOT, WARM and COLD
+ policies. (szetszwo)
+
+ HDFS-6671. Change BlockPlacementPolicy to consider block storage policy
+ in replicaiton. (szetszwo)
+
+ HDFS-6710. Change BlockPlacementPolicy to consider block storage policy
+ in replica deletion. (szetszwo)
+
+ HDFS-6679. Bump NameNodeLayoutVersion and update editsStored test files.
+ (vinayakumarb via szetszwo)
+
+ HDFS-6686. Change BlockPlacementPolicy to use fallback when some storage
+ types are unavailable. (szetszwo)
+
+ HDFS-6835. Add a new API to set storage policy. (jing9)
+
+ HDFS-6847. Support storage policy on directories and include storage policy
+ in HdfsFileStatus. (Jing Zhao via szetszwo)
+
+ HDFS-6801. Add a new data migration tool, Mover, for archiving data.
+ (szetszwo via jing9)
+
+ HDFS-6863. Support migration for snapshot paths. (jing9)
+
+ HDFS-6906. Add more tests for BlockStoragePolicy. (szetszwo via jing9)
+
+ HDFS-6911. check if a block is already scheduled in Mover.
+ (szetszwo via jing9)
+
+ HDFS-6920. Check the storage type of delNodeHintStorage when deleting
+ a replica. (szetszwo via jing9)
+
+ HDFS-6944. Add retry and termination logic for Mover. (jing9)
+
+ HDFS-6969. INode#getStoragePolicyID should always return the latest
+ storage policy. (jing9)
+
+ HDFS-6961. BlockPlacementPolicy#chooseTarget should check each valid
+ storage type in each choosing round. (jing9)
+
+ HDFS-6876. support set/get storage policy in DFSAdmin. (jing9)
+
+ HDFS-6997. Add more tests for data migration and replicaion. (szetszwo)
+
+ HDFS-6875. Support migration for a list of specified paths. (jing9)
+
+ HDFS-7027. Mover does not terminate when some storage type is out of space.
+ (szetszwo via jing9)
+
+ HDFS-7029. Fix TestDFSInotifyEventInputStream and TestDistributedFileSystem.
+ (szetszwo via jing9)
+
+ HDFS-7028. FSDirectory should not get storage policy id from symlinks.
+ (szetszwo)
+
+ HDFS-7034. Fix TestBlockPlacement and TestStorageMover. (jing9)
+
+ HDFS-7039. Fix Balancer tests. (szetszwo via jing9)
+
+ HDFS-7062. Skip under construction block for migration. (jing9)
+
+ HDFS-7052. Add Mover into hdfs script. (jing9)
+
+ HDFS-7072. Fix TestBlockManager and TestStorageMover. (jing9 via szetszwo)
+
+ HDFS-6864. Archival Storage: add user documentation. (szetszwo via jing9)
+
+ HDFS-7088. Archival Storage: fix TestBalancer and
+ TestBalancerWithMultipleNameNodes. (szetszwo via jing9)
+
Release 2.6.0 - UNRELEASED
INCOMPATIBLE CHANGES
[20/50] [abbrv] git commit: HDFS-7105. Fix
TestJournalNode#testFailToStartWithBadConfig to match log output change.
Contributed by Ray Chiang.
Posted by cu...@apache.org.
HDFS-7105. Fix TestJournalNode#testFailToStartWithBadConfig to match log output change. Contributed by Ray Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/332e2e23
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/332e2e23
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/332e2e23
Branch: refs/heads/YARN-1051
Commit: 332e2e23ba6f0748a46c0bda76f426d9cad73edd
Parents: b6ceef9
Author: cnauroth <cn...@apache.org>
Authored: Fri Sep 19 21:04:55 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Fri Sep 19 21:04:55 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/332e2e23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8c45d69..fa9bbcd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -897,6 +897,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7051. TestDataNodeRollingUpgrade#isBlockFileInPrevious assumes Unix file
path separator. (cnauroth)
+ HDFS-7105. Fix TestJournalNode#testFailToStartWithBadConfig to match log
+ output change. (Ray Chiang via cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/332e2e23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
index a7ee927..1bf3add 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
@@ -287,7 +287,7 @@ public class TestJournalNode {
// Directory which cannot be created
conf.set(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY,
Shell.WINDOWS ? "\\\\cannotBeCreated" : "/proc/does-not-exist");
- assertJNFailsToStart(conf, "Can not create directory");
+ assertJNFailsToStart(conf, "Cannot create directory");
}
private static void assertJNFailsToStart(Configuration conf,
[41/50] [abbrv] YARN-1708. Public YARN APIs for
creating/updating/deleting reservations.
Posted by cu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionRequestPBImpl.java
new file mode 100644
index 0000000..9f10f4c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionRequestPBImpl.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationDefinitionProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionRequestProtoOrBuilder;
+
+import com.google.protobuf.TextFormat;
+
+public class ReservationSubmissionRequestPBImpl extends
+ ReservationSubmissionRequest {
+
+ ReservationSubmissionRequestProto proto = ReservationSubmissionRequestProto
+ .getDefaultInstance();
+ ReservationSubmissionRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ReservationDefinition reservationDefinition;
+
+ public ReservationSubmissionRequestPBImpl() {
+ builder = ReservationSubmissionRequestProto.newBuilder();
+ }
+
+ public ReservationSubmissionRequestPBImpl(
+ ReservationSubmissionRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReservationSubmissionRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.reservationDefinition != null) {
+ builder
+ .setReservationDefinition(convertToProtoFormat(reservationDefinition));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ReservationSubmissionRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public ReservationDefinition getReservationDefinition() {
+ ReservationSubmissionRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (reservationDefinition != null) {
+ return reservationDefinition;
+ }
+ if (!p.hasReservationDefinition()) {
+ return null;
+ }
+ reservationDefinition =
+ convertFromProtoFormat(p.getReservationDefinition());
+ return reservationDefinition;
+ }
+
+ @Override
+ public void setReservationDefinition(
+ ReservationDefinition reservationDefinition) {
+ maybeInitBuilder();
+ if (reservationDefinition == null) {
+ builder.clearReservationDefinition();
+ }
+ this.reservationDefinition = reservationDefinition;
+ }
+
+ @Override
+ public String getQueue() {
+ ReservationSubmissionRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasQueue()) {
+ return null;
+ }
+ return (p.getQueue());
+ }
+
+ @Override
+ public void setQueue(String planName) {
+ maybeInitBuilder();
+ if (planName == null) {
+ builder.clearQueue();
+ return;
+ }
+ builder.setQueue(planName);
+ }
+
+ private ReservationDefinitionProto convertToProtoFormat(
+ ReservationDefinition r) {
+ return ((ReservationDefinitionPBImpl) r).getProto();
+ }
+
+ private ReservationDefinitionPBImpl convertFromProtoFormat(
+ ReservationDefinitionProto r) {
+ return new ReservationDefinitionPBImpl(r);
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionResponsePBImpl.java
new file mode 100644
index 0000000..82151f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionResponsePBImpl.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionResponseProtoOrBuilder;
+
+import com.google.protobuf.TextFormat;
+
+public class ReservationSubmissionResponsePBImpl extends
+ ReservationSubmissionResponse {
+
+ ReservationSubmissionResponseProto proto = ReservationSubmissionResponseProto
+ .getDefaultInstance();
+ ReservationSubmissionResponseProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ReservationId reservationId;
+
+ public ReservationSubmissionResponsePBImpl() {
+ builder = ReservationSubmissionResponseProto.newBuilder();
+ }
+
+ public ReservationSubmissionResponsePBImpl(
+ ReservationSubmissionResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReservationSubmissionResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.reservationId != null) {
+ builder.setReservationId(convertToProtoFormat(this.reservationId));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ReservationSubmissionResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public ReservationId getReservationId() {
+ ReservationSubmissionResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (reservationId != null) {
+ return reservationId;
+ }
+ if (!p.hasReservationId()) {
+ return null;
+ }
+ reservationId = convertFromProtoFormat(p.getReservationId());
+ return reservationId;
+ }
+
+ @Override
+ public void setReservationId(ReservationId reservationId) {
+ maybeInitBuilder();
+ if (reservationId == null) {
+ builder.clearReservationId();
+ return;
+ }
+ this.reservationId = reservationId;
+ }
+
+ private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
+ return new ReservationIdPBImpl(p);
+ }
+
+ private ReservationIdProto convertToProtoFormat(ReservationId t) {
+ return ((ReservationIdPBImpl) t).getProto();
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationUpdateRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationUpdateRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationUpdateRequestPBImpl.java
new file mode 100644
index 0000000..89999f2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationUpdateRequestPBImpl.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationDefinitionProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateRequestProtoOrBuilder;
+
+import com.google.protobuf.TextFormat;
+
+public class ReservationUpdateRequestPBImpl extends ReservationUpdateRequest {
+
+ ReservationUpdateRequestProto proto = ReservationUpdateRequestProto
+ .getDefaultInstance();
+ ReservationUpdateRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ReservationDefinition reservationDefinition;
+ private ReservationId reservationId;
+
+ public ReservationUpdateRequestPBImpl() {
+ builder = ReservationUpdateRequestProto.newBuilder();
+ }
+
+ public ReservationUpdateRequestPBImpl(ReservationUpdateRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReservationUpdateRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.reservationId != null) {
+ builder.setReservationId(convertToProtoFormat(this.reservationId));
+ }
+ if (this.reservationDefinition != null) {
+ builder
+ .setReservationDefinition(convertToProtoFormat(reservationDefinition));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ReservationUpdateRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public ReservationDefinition getReservationDefinition() {
+ ReservationUpdateRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (reservationDefinition != null) {
+ return reservationDefinition;
+ }
+ if (!p.hasReservationDefinition()) {
+ return null;
+ }
+ reservationDefinition =
+ convertFromProtoFormat(p.getReservationDefinition());
+ return reservationDefinition;
+ }
+
+ @Override
+ public void setReservationDefinition(
+ ReservationDefinition reservationDefinition) {
+ maybeInitBuilder();
+ if (reservationDefinition == null) {
+ builder.clearReservationDefinition();
+ }
+ this.reservationDefinition = reservationDefinition;
+ }
+
+ @Override
+ public ReservationId getReservationId() {
+ ReservationUpdateRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (reservationId != null) {
+ return reservationId;
+ }
+ if (!p.hasReservationId()) {
+ return null;
+ }
+ reservationId = convertFromProtoFormat(p.getReservationId());
+ return reservationId;
+ }
+
+ @Override
+ public void setReservationId(ReservationId reservationId) {
+ maybeInitBuilder();
+ if (reservationId == null) {
+ builder.clearReservationId();
+ return;
+ }
+ this.reservationId = reservationId;
+ }
+
+ private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
+ return new ReservationIdPBImpl(p);
+ }
+
+ private ReservationIdProto convertToProtoFormat(ReservationId t) {
+ return ((ReservationIdPBImpl) t).getProto();
+ }
+
+ private ReservationDefinitionProto convertToProtoFormat(
+ ReservationDefinition r) {
+ return ((ReservationDefinitionPBImpl) r).getProto();
+ }
+
+ private ReservationDefinitionPBImpl convertFromProtoFormat(
+ ReservationDefinitionProto r) {
+ return new ReservationDefinitionPBImpl(r);
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationUpdateResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationUpdateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationUpdateResponsePBImpl.java
new file mode 100644
index 0000000..57c7b1b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationUpdateResponsePBImpl.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateResponseProto;
+
+import com.google.protobuf.TextFormat;
+
+public class ReservationUpdateResponsePBImpl extends ReservationUpdateResponse {
+
+ ReservationUpdateResponseProto proto = ReservationUpdateResponseProto
+ .getDefaultInstance();
+ ReservationUpdateResponseProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public ReservationUpdateResponsePBImpl() {
+ builder = ReservationUpdateResponseProto.newBuilder();
+ }
+
+ public ReservationUpdateResponsePBImpl(ReservationUpdateResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReservationUpdateResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
index 7b49a16..624ec87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
@@ -19,12 +19,14 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
import com.google.common.base.CharMatcher;
+
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
@@ -32,6 +34,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import com.google.protobuf.TextFormat;
@@ -53,6 +56,7 @@ extends ApplicationSubmissionContext {
private ContainerLaunchContext amContainer = null;
private Resource resource = null;
private Set<String> applicationTags = null;
+ private ReservationId reservationId;
public ApplicationSubmissionContextPBImpl() {
builder = ApplicationSubmissionContextProto.newBuilder();
@@ -110,6 +114,9 @@ extends ApplicationSubmissionContext {
builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
+ if (this.reservationId != null) {
+ builder.setReservationId(convertToProtoFormat(this.reservationId));
+ }
}
private void mergeLocalToProto() {
@@ -358,6 +365,29 @@ extends ApplicationSubmissionContext {
}
@Override
+ public ReservationId getReservationID() {
+ ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
+ if (reservationId != null) {
+ return reservationId;
+ }
+ if (!p.hasReservationId()) {
+ return null;
+ }
+ reservationId = convertFromProtoFormat(p.getReservationId());
+ return reservationId;
+ }
+
+ @Override
+ public void setReservationID(ReservationId reservationID) {
+ maybeInitBuilder();
+ if (reservationID == null) {
+ builder.clearReservationId();
+ return;
+ }
+ this.reservationId = reservationID;
+ }
+
+ @Override
public void
setKeepContainersAcrossApplicationAttempts(boolean keepContainers) {
maybeInitBuilder();
@@ -415,4 +445,12 @@ extends ApplicationSubmissionContext {
maybeInitBuilder();
builder.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
}
+
+ private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
+ return new ReservationIdPBImpl(p);
+ }
+
+ private ReservationIdProto convertToProtoFormat(ReservationId t) {
+ return ((ReservationIdPBImpl) t).getProto();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 4a3c137..f18a77c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto;
@@ -47,10 +48,12 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestInterpreterProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import com.google.protobuf.ByteString;
+
import org.apache.hadoop.yarn.proto.YarnServiceProtos;
@Private
@@ -237,4 +240,18 @@ public class ProtoUtils {
return ApplicationAccessType.valueOf(e.name().replace(
APP_ACCESS_TYPE_PREFIX, ""));
}
+
+ /*
+ * Reservation Request interpreter type
+ */
+ public static ReservationRequestInterpreterProto convertToProtoFormat(
+ ReservationRequestInterpreter e) {
+ return ReservationRequestInterpreterProto.valueOf(e.name());
+ }
+
+ public static ReservationRequestInterpreter convertFromProtoFormat(
+ ReservationRequestInterpreterProto e) {
+ return ReservationRequestInterpreter.valueOf(e.name());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
new file mode 100644
index 0000000..9f87108
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationDefinitionProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationDefinitionProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestsProto;
+
+public class ReservationDefinitionPBImpl extends ReservationDefinition {
+
+ ReservationDefinitionProto proto = ReservationDefinitionProto
+ .getDefaultInstance();
+ ReservationDefinitionProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ReservationRequests reservationReqs;
+
+ public ReservationDefinitionPBImpl() {
+ builder = ReservationDefinitionProto.newBuilder();
+ }
+
+ public ReservationDefinitionPBImpl(ReservationDefinitionProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReservationDefinitionProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.reservationReqs != null) {
+ builder
+ .setReservationRequests(convertToProtoFormat(this.reservationReqs));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ReservationDefinitionProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public long getArrival() {
+ ReservationDefinitionProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasArrival()) {
+ return 0;
+ }
+ return (p.getArrival());
+ }
+
+ @Override
+ public void setArrival(long earliestStartTime) {
+ maybeInitBuilder();
+ if (earliestStartTime <= 0) {
+ builder.clearArrival();
+ return;
+ }
+ builder.setArrival(earliestStartTime);
+ }
+
+ @Override
+ public long getDeadline() {
+ ReservationDefinitionProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasDeadline()) {
+ return 0;
+ }
+ return (p.getDeadline());
+ }
+
+ @Override
+ public void setDeadline(long latestEndTime) {
+ maybeInitBuilder();
+ if (latestEndTime <= 0) {
+ builder.clearDeadline();
+ return;
+ }
+ builder.setDeadline(latestEndTime);
+ }
+
+ @Override
+ public ReservationRequests getReservationRequests() {
+ ReservationDefinitionProtoOrBuilder p = viaProto ? proto : builder;
+ if (reservationReqs != null) {
+ return reservationReqs;
+ }
+ if (!p.hasReservationRequests()) {
+ return null;
+ }
+ reservationReqs = convertFromProtoFormat(p.getReservationRequests());
+ return reservationReqs;
+ }
+
+ @Override
+ public void setReservationRequests(ReservationRequests reservationRequests) {
+ if (reservationRequests == null) {
+ builder.clearReservationRequests();
+ return;
+ }
+ this.reservationReqs = reservationRequests;
+ }
+
+ @Override
+ public String getReservationName() {
+ ReservationDefinitionProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasReservationName()) {
+ return null;
+ }
+ return (p.getReservationName());
+ }
+
+ @Override
+ public void setReservationName(String name) {
+ maybeInitBuilder();
+ if (name == null) {
+ builder.clearReservationName();
+ return;
+ }
+ builder.setReservationName(name);
+ }
+
+ private ReservationRequestsPBImpl convertFromProtoFormat(
+ ReservationRequestsProto p) {
+ return new ReservationRequestsPBImpl(p);
+ }
+
+ private ReservationRequestsProto convertToProtoFormat(ReservationRequests t) {
+ return ((ReservationRequestsPBImpl) t).getProto();
+ }
+
+ @Override
+ public String toString() {
+ return "{Arrival: " + getArrival() + ", Deadline: " + getDeadline()
+ + ", Reservation Name: " + getReservationName() + ", Resources: "
+ + getReservationRequests() + "}";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationIdPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationIdPBImpl.java
new file mode 100644
index 0000000..609d4ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationIdPBImpl.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
+
+import com.google.common.base.Preconditions;
+
+@Private
+@Unstable
+public class ReservationIdPBImpl extends ReservationId {
+ ReservationIdProto proto = null;
+ ReservationIdProto.Builder builder = null;
+
+ public ReservationIdPBImpl() {
+ builder = ReservationIdProto.newBuilder();
+ }
+
+ public ReservationIdPBImpl(ReservationIdProto proto) {
+ this.proto = proto;
+ }
+
+ public ReservationIdProto getProto() {
+ return proto;
+ }
+
+ @Override
+ public long getId() {
+ Preconditions.checkNotNull(proto);
+ return proto.getId();
+ }
+
+ @Override
+ protected void setId(long id) {
+ Preconditions.checkNotNull(builder);
+ builder.setId(id);
+ }
+
+ @Override
+ public long getClusterTimestamp() {
+ Preconditions.checkNotNull(proto);
+ return proto.getClusterTimestamp();
+ }
+
+ @Override
+ protected void setClusterTimestamp(long clusterTimestamp) {
+ Preconditions.checkNotNull(builder);
+ builder.setClusterTimestamp((clusterTimestamp));
+ }
+
+ @Override
+ protected void build() {
+ proto = builder.build();
+ builder = null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestPBImpl.java
new file mode 100644
index 0000000..e36b9fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestPBImpl.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+
+@Private
+@Unstable
+public class ReservationRequestPBImpl extends ReservationRequest {
+ ReservationRequestProto proto = ReservationRequestProto.getDefaultInstance();
+ ReservationRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private Resource capability = null;
+
+ public ReservationRequestPBImpl() {
+ builder = ReservationRequestProto.newBuilder();
+ }
+
+ public ReservationRequestPBImpl(ReservationRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReservationRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.capability != null) {
+ builder.setCapability(convertToProtoFormat(this.capability));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ReservationRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public Resource getCapability() {
+ ReservationRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.capability != null) {
+ return this.capability;
+ }
+ if (!p.hasCapability()) {
+ return null;
+ }
+ this.capability = convertFromProtoFormat(p.getCapability());
+ return this.capability;
+ }
+
+ @Override
+ public void setCapability(Resource capability) {
+ maybeInitBuilder();
+ if (capability == null)
+ builder.clearCapability();
+ this.capability = capability;
+ }
+
+ @Override
+ public int getNumContainers() {
+ ReservationRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getNumContainers());
+ }
+
+ @Override
+ public void setNumContainers(int numContainers) {
+ maybeInitBuilder();
+ builder.setNumContainers((numContainers));
+ }
+
+ @Override
+ public int getConcurrency() {
+ ReservationRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasConcurrency()) {
+ return 1;
+ }
+ return (p.getConcurrency());
+ }
+
+ @Override
+ public void setConcurrency(int numContainers) {
+ maybeInitBuilder();
+ builder.setConcurrency(numContainers);
+ }
+
+ @Override
+ public long getDuration() {
+ ReservationRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasDuration()) {
+ return 0;
+ }
+ return (p.getDuration());
+ }
+
+ @Override
+ public void setDuration(long duration) {
+ maybeInitBuilder();
+ builder.setDuration(duration);
+ }
+
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource t) {
+ return ((ResourcePBImpl) t).getProto();
+ }
+
+ @Override
+ public String toString() {
+ return "{Capability: " + getCapability() + ", # Containers: "
+ + getNumContainers() + ", Concurrency: " + getConcurrency()
+ + ", Lease Duration: " + getDuration() + "}";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestsPBImpl.java
new file mode 100644
index 0000000..4b39836
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestsPBImpl.java
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestInterpreterProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestsProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestsProtoOrBuilder;
+
+public class ReservationRequestsPBImpl extends ReservationRequests {
+
+ ReservationRequestsProto proto = ReservationRequestsProto
+ .getDefaultInstance();
+ ReservationRequestsProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public List<ReservationRequest> reservationRequests;
+
+ public ReservationRequestsPBImpl() {
+ builder = ReservationRequestsProto.newBuilder();
+ }
+
+ public ReservationRequestsPBImpl(ReservationRequestsProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReservationRequestsProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.reservationRequests != null) {
+ addReservationResourcesToProto();
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ReservationRequestsProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public List<ReservationRequest> getReservationResources() {
+ initReservationRequestsList();
+ return reservationRequests;
+ }
+
+ @Override
+ public void setReservationResources(List<ReservationRequest> resources) {
+ if (resources == null) {
+ builder.clearReservationResources();
+ return;
+ }
+ this.reservationRequests = resources;
+ }
+
+ @Override
+ public ReservationRequestInterpreter getInterpreter() {
+ ReservationRequestsProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasInterpreter()) {
+ return null;
+ }
+ return (convertFromProtoFormat(p.getInterpreter()));
+ }
+
+ @Override
+ public void setInterpreter(ReservationRequestInterpreter interpreter) {
+ maybeInitBuilder();
+ if (interpreter == null) {
+ builder.clearInterpreter();
+ return;
+ }
+ builder.setInterpreter(convertToProtoFormat(interpreter));
+ }
+
+ private void initReservationRequestsList() {
+ if (this.reservationRequests != null) {
+ return;
+ }
+ ReservationRequestsProtoOrBuilder p = viaProto ? proto : builder;
+ List<ReservationRequestProto> resourceProtos =
+ p.getReservationResourcesList();
+ reservationRequests = new ArrayList<ReservationRequest>();
+
+ for (ReservationRequestProto r : resourceProtos) {
+ reservationRequests.add(convertFromProtoFormat(r));
+ }
+ }
+
+ private void addReservationResourcesToProto() {
+ maybeInitBuilder();
+ builder.clearReservationResources();
+ if (reservationRequests == null)
+ return;
+ Iterable<ReservationRequestProto> iterable =
+ new Iterable<ReservationRequestProto>() {
+ @Override
+ public Iterator<ReservationRequestProto> iterator() {
+ return new Iterator<ReservationRequestProto>() {
+
+ Iterator<ReservationRequest> iter = reservationRequests
+ .iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public ReservationRequestProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+
+ }
+
+ };
+ builder.addAllReservationResources(iterable);
+ }
+
+ private ReservationRequestProto convertToProtoFormat(ReservationRequest r) {
+ return ((ReservationRequestPBImpl) r).getProto();
+ }
+
+ private ReservationRequestPBImpl convertFromProtoFormat(
+ ReservationRequestProto r) {
+ return new ReservationRequestPBImpl(r);
+ }
+
+ private ReservationRequestInterpreterProto convertToProtoFormat(
+ ReservationRequestInterpreter r) {
+ return ProtoUtils.convertToProtoFormat(r);
+ }
+
+ private ReservationRequestInterpreter convertFromProtoFormat(
+ ReservationRequestInterpreterProto r) {
+ return ProtoUtils.convertFromProtoFormat(r);
+ }
+
+ @Override
+ public String toString() {
+ return "{Reservation Resources: " + getReservationResources()
+ + ", Reservation Type: " + getInterpreter() + "}";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f1d205a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/UTCClock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/UTCClock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/UTCClock.java
new file mode 100644
index 0000000..ecef3dd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/UTCClock.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.util;
+
+import java.util.Calendar;
+import java.util.TimeZone;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+/**
+ * Implementation of {@link Clock} that gives the current UTC time in
+ * milliseconds.
+ */
+@Public
+@Evolving
+public class UTCClock implements Clock {
+
+ private final TimeZone utcZone = TimeZone.getTimeZone("UTC");
+
+ public long getTime() {
+ return Calendar.getInstance(utcZone).getTimeInMillis();
+ }
+}
\ No newline at end of file
[33/50] [abbrv] git commit: MAPREDUCE-5279. Made MR headroom
calculation honor cpu dimension when YARN scheduler resource type is memory
plus cpu. Contributed by Peng Zhang and Varun Vasudev.
Posted by cu...@apache.org.
MAPREDUCE-5279. Made MR headroom calculation honor cpu dimension when YARN scheduler resource type is memory plus cpu. Contributed by Peng Zhang and Varun Vasudev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/376233cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/376233cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/376233cd
Branch: refs/heads/YARN-1051
Commit: 376233cdd4a4ddbde5a92a0627f78338cb4c38b7
Parents: 26cba7f
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Sep 22 09:28:47 2014 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Sep 22 09:28:47 2014 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 4 +
.../mapreduce/v2/app/rm/RMCommunicator.java | 12 +-
.../v2/app/rm/RMContainerAllocator.java | 272 +++++++++++--------
.../v2/app/rm/ResourceCalculatorUtils.java | 52 ++++
.../v2/app/rm/TestRMContainerAllocator.java | 82 ++++--
.../hadoop/yarn/util/resource/Resources.java | 4 +-
6 files changed, 288 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376233cd/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 815b3fb..859c899 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -254,6 +254,10 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-5891. Improved shuffle error handling across NM restarts
(Junping Du via jlowe)
+ MAPREDUCE-5279. Made MR headroom calculation honor cpu dimension when YARN
+ scheduler resource type is memory plus cpu. (Peng Zhang and Varun Vasudev
+ via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376233cd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
index 6e9f313..6c58a68 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.mapreduce.v2.app.rm;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
+import java.util.EnumSet;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -59,6 +60,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
/**
* Registers/unregisters to RM and sends heartbeats to RM.
@@ -90,6 +92,8 @@ public abstract class RMCommunicator extends AbstractService
private volatile boolean shouldUnregister = true;
private boolean isApplicationMasterRegistered = false;
+ private EnumSet<SchedulerResourceTypes> schedulerResourceTypes;
+
public RMCommunicator(ClientService clientService, AppContext context) {
super("RMCommunicator");
this.clientService = clientService;
@@ -98,6 +102,7 @@ public abstract class RMCommunicator extends AbstractService
this.applicationId = context.getApplicationID();
this.stopped = new AtomicBoolean(false);
this.heartbeatCallbacks = new ConcurrentLinkedQueue<Runnable>();
+ this.schedulerResourceTypes = EnumSet.of(SchedulerResourceTypes.MEMORY);
}
@Override
@@ -163,10 +168,11 @@ public abstract class RMCommunicator extends AbstractService
setClientToAMToken(response.getClientToAMTokenMasterKey());
}
this.applicationACLs = response.getApplicationACLs();
- LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory());
+ LOG.info("maxContainerCapability: " + maxContainerCapability);
String queue = response.getQueue();
LOG.info("queue: " + queue);
job.setQueueName(queue);
+ this.schedulerResourceTypes.addAll(response.getSchedulerResourceTypes());
} catch (Exception are) {
LOG.error("Exception while registering", are);
throw new YarnRuntimeException(are);
@@ -343,4 +349,8 @@ public abstract class RMCommunicator extends AbstractService
protected boolean isApplicationMasterRegistered() {
return isApplicationMasterRegistered;
}
+
+ public EnumSet<SchedulerResourceTypes> getSchedulerResourceTypes() {
+ return schedulerResourceTypes;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376233cd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index 6cb0191..fb8771a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.client.api.NMTokenCache;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -80,6 +81,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.RackResolver;
+import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
@@ -149,8 +151,8 @@ public class RMContainerAllocator extends RMContainerRequestor
private int lastCompletedTasks = 0;
private boolean recalculateReduceSchedule = false;
- private int mapResourceRequest;//memory
- private int reduceResourceRequest;//memory
+ private Resource mapResourceRequest = Resources.none();
+ private Resource reduceResourceRequest = Resources.none();
private boolean reduceStarted = false;
private float maxReduceRampupLimit = 0;
@@ -328,49 +330,61 @@ public class RMContainerAllocator extends RMContainerRequestor
if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
ContainerRequestEvent reqEvent = (ContainerRequestEvent) event;
JobId jobId = getJob().getID();
- int supportedMaxContainerCapability =
- getMaxContainerCapability().getMemory();
+ Resource supportedMaxContainerCapability = getMaxContainerCapability();
if (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP)) {
- if (mapResourceRequest == 0) {
- mapResourceRequest = reqEvent.getCapability().getMemory();
- eventHandler.handle(new JobHistoryEvent(jobId,
- new NormalizedResourceEvent(org.apache.hadoop.mapreduce.TaskType.MAP,
- mapResourceRequest)));
- LOG.info("mapResourceRequest:"+ mapResourceRequest);
- if (mapResourceRequest > supportedMaxContainerCapability) {
- String diagMsg = "MAP capability required is more than the supported " +
- "max container capability in the cluster. Killing the Job. mapResourceRequest: " +
- mapResourceRequest + " maxContainerCapability:" + supportedMaxContainerCapability;
+ if (mapResourceRequest.equals(Resources.none())) {
+ mapResourceRequest = reqEvent.getCapability();
+ eventHandler.handle(new JobHistoryEvent(jobId,
+ new NormalizedResourceEvent(
+ org.apache.hadoop.mapreduce.TaskType.MAP, mapResourceRequest
+ .getMemory())));
+ LOG.info("mapResourceRequest:" + mapResourceRequest);
+ if (mapResourceRequest.getMemory() > supportedMaxContainerCapability
+ .getMemory()
+ || mapResourceRequest.getVirtualCores() > supportedMaxContainerCapability
+ .getVirtualCores()) {
+ String diagMsg =
+ "MAP capability required is more than the supported "
+ + "max container capability in the cluster. Killing the Job. mapResourceRequest: "
+ + mapResourceRequest + " maxContainerCapability:"
+ + supportedMaxContainerCapability;
LOG.info(diagMsg);
- eventHandler.handle(new JobDiagnosticsUpdateEvent(
- jobId, diagMsg));
+ eventHandler.handle(new JobDiagnosticsUpdateEvent(jobId, diagMsg));
eventHandler.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
}
}
- //set the rounded off memory
- reqEvent.getCapability().setMemory(mapResourceRequest);
+ // set the resources
+ reqEvent.getCapability().setMemory(mapResourceRequest.getMemory());
+ reqEvent.getCapability().setVirtualCores(
+ mapResourceRequest.getVirtualCores());
scheduledRequests.addMap(reqEvent);//maps are immediately scheduled
} else {
- if (reduceResourceRequest == 0) {
- reduceResourceRequest = reqEvent.getCapability().getMemory();
- eventHandler.handle(new JobHistoryEvent(jobId,
- new NormalizedResourceEvent(
- org.apache.hadoop.mapreduce.TaskType.REDUCE,
- reduceResourceRequest)));
- LOG.info("reduceResourceRequest:"+ reduceResourceRequest);
- if (reduceResourceRequest > supportedMaxContainerCapability) {
- String diagMsg = "REDUCE capability required is more than the " +
- "supported max container capability in the cluster. Killing the " +
- "Job. reduceResourceRequest: " + reduceResourceRequest +
- " maxContainerCapability:" + supportedMaxContainerCapability;
+ if (reduceResourceRequest.equals(Resources.none())) {
+ reduceResourceRequest = reqEvent.getCapability();
+ eventHandler.handle(new JobHistoryEvent(jobId,
+ new NormalizedResourceEvent(
+ org.apache.hadoop.mapreduce.TaskType.REDUCE,
+ reduceResourceRequest.getMemory())));
+ LOG.info("reduceResourceRequest:" + reduceResourceRequest);
+ if (reduceResourceRequest.getMemory() > supportedMaxContainerCapability
+ .getMemory()
+ || reduceResourceRequest.getVirtualCores() > supportedMaxContainerCapability
+ .getVirtualCores()) {
+ String diagMsg =
+ "REDUCE capability required is more than the "
+ + "supported max container capability in the cluster. Killing the "
+ + "Job. reduceResourceRequest: " + reduceResourceRequest
+ + " maxContainerCapability:"
+ + supportedMaxContainerCapability;
LOG.info(diagMsg);
- eventHandler.handle(new JobDiagnosticsUpdateEvent(
- jobId, diagMsg));
+ eventHandler.handle(new JobDiagnosticsUpdateEvent(jobId, diagMsg));
eventHandler.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
}
}
- //set the rounded off memory
- reqEvent.getCapability().setMemory(reduceResourceRequest);
+ // set the resources
+ reqEvent.getCapability().setMemory(reduceResourceRequest.getMemory());
+ reqEvent.getCapability().setVirtualCores(
+ reduceResourceRequest.getVirtualCores());
if (reqEvent.getEarlierAttemptFailed()) {
//add to the front of queue for fail fast
pendingReduces.addFirst(new ContainerRequest(reqEvent, PRIORITY_REDUCE));
@@ -425,34 +439,40 @@ public class RMContainerAllocator extends RMContainerRequestor
@Private
@VisibleForTesting
- synchronized void setReduceResourceRequest(int mem) {
- this.reduceResourceRequest = mem;
+ synchronized void setReduceResourceRequest(Resource res) {
+ this.reduceResourceRequest = res;
}
@Private
@VisibleForTesting
- synchronized void setMapResourceRequest(int mem) {
- this.mapResourceRequest = mem;
+ synchronized void setMapResourceRequest(Resource res) {
+ this.mapResourceRequest = res;
}
@Private
@VisibleForTesting
void preemptReducesIfNeeded() {
- if (reduceResourceRequest == 0) {
- return; //no reduces
+ if (reduceResourceRequest.equals(Resources.none())) {
+ return; // no reduces
}
//check if reduces have taken over the whole cluster and there are
//unassigned maps
if (scheduledRequests.maps.size() > 0) {
- int memLimit = getMemLimit();
- int availableMemForMap = memLimit - ((assignedRequests.reduces.size() -
- assignedRequests.preemptionWaitingReduces.size()) * reduceResourceRequest);
- //availableMemForMap must be sufficient to run atleast 1 map
- if (availableMemForMap < mapResourceRequest) {
- //to make sure new containers are given to maps and not reduces
- //ramp down all scheduled reduces if any
- //(since reduces are scheduled at higher priority than maps)
- LOG.info("Ramping down all scheduled reduces:" + scheduledRequests.reduces.size());
+ Resource resourceLimit = getResourceLimit();
+ Resource availableResourceForMap =
+ Resources.subtract(
+ resourceLimit,
+ Resources.multiply(reduceResourceRequest,
+ assignedRequests.reduces.size()
+ - assignedRequests.preemptionWaitingReduces.size()));
+ // availableMemForMap must be sufficient to run at least 1 map
+ if (ResourceCalculatorUtils.computeAvailableContainers(availableResourceForMap,
+ mapResourceRequest, getSchedulerResourceTypes()) <= 0) {
+ // to make sure new containers are given to maps and not reduces
+ // ramp down all scheduled reduces if any
+ // (since reduces are scheduled at higher priority than maps)
+ LOG.info("Ramping down all scheduled reduces:"
+ + scheduledRequests.reduces.size());
for (ContainerRequest req : scheduledRequests.reduces.values()) {
pendingReduces.add(req);
}
@@ -462,17 +482,25 @@ public class RMContainerAllocator extends RMContainerRequestor
//hanging around for a while
int hangingMapRequests = getNumOfHangingRequests(scheduledRequests.maps);
if (hangingMapRequests > 0) {
- //preempt for making space for at least one map
- int premeptionLimit = Math.max(mapResourceRequest,
- (int) (maxReducePreemptionLimit * memLimit));
-
- int preemptMem = Math.min(hangingMapRequests * mapResourceRequest,
- premeptionLimit);
-
- int toPreempt = (int) Math.ceil((float) preemptMem / reduceResourceRequest);
- toPreempt = Math.min(toPreempt, assignedRequests.reduces.size());
-
- LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps");
+ // preempt for making space for at least one map
+ int preemptionReduceNumForOneMap =
+ ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest,
+ reduceResourceRequest, getSchedulerResourceTypes());
+ int preemptionReduceNumForPreemptionLimit =
+ ResourceCalculatorUtils.divideAndCeilContainers(
+ Resources.multiply(resourceLimit, maxReducePreemptionLimit),
+ reduceResourceRequest, getSchedulerResourceTypes());
+ int preemptionReduceNumForAllMaps =
+ ResourceCalculatorUtils.divideAndCeilContainers(
+ Resources.multiply(mapResourceRequest, hangingMapRequests),
+ reduceResourceRequest, getSchedulerResourceTypes());
+ int toPreempt =
+ Math.min(Math.max(preemptionReduceNumForOneMap,
+ preemptionReduceNumForPreemptionLimit),
+ preemptionReduceNumForAllMaps);
+
+ LOG.info("Going to preempt " + toPreempt
+ + " due to lack of space for maps");
assignedRequests.preemptReduce(toPreempt);
}
}
@@ -497,7 +525,7 @@ public class RMContainerAllocator extends RMContainerRequestor
int totalMaps, int completedMaps,
int scheduledMaps, int scheduledReduces,
int assignedMaps, int assignedReduces,
- int mapResourceReqt, int reduceResourceReqt,
+ Resource mapResourceReqt, Resource reduceResourceReqt,
int numPendingReduces,
float maxReduceRampupLimit, float reduceSlowStart) {
@@ -505,8 +533,12 @@ public class RMContainerAllocator extends RMContainerRequestor
return;
}
- int headRoom = getAvailableResources() != null ?
- getAvailableResources().getMemory() : 0;
+ // get available resources for this job
+ Resource headRoom = getAvailableResources();
+ if (headRoom == null) {
+ headRoom = Resources.none();
+ }
+
LOG.info("Recalculating schedule, headroom=" + headRoom);
//check for slow start
@@ -540,49 +572,60 @@ public class RMContainerAllocator extends RMContainerRequestor
completedMapPercent = 1;
}
- int netScheduledMapMem =
- (scheduledMaps + assignedMaps) * mapResourceReqt;
+ Resource netScheduledMapResource =
+ Resources.multiply(mapResourceReqt, (scheduledMaps + assignedMaps));
- int netScheduledReduceMem =
- (scheduledReduces + assignedReduces) * reduceResourceReqt;
+ Resource netScheduledReduceResource =
+ Resources.multiply(reduceResourceReqt,
+ (scheduledReduces + assignedReduces));
+
+ Resource finalMapResourceLimit;
+ Resource finalReduceResourceLimit;
- int finalMapMemLimit = 0;
- int finalReduceMemLimit = 0;
-
// ramp up the reduces based on completed map percentage
- int totalMemLimit = getMemLimit();
- int idealReduceMemLimit =
- Math.min(
- (int)(completedMapPercent * totalMemLimit),
- (int) (maxReduceRampupLimit * totalMemLimit));
- int idealMapMemLimit = totalMemLimit - idealReduceMemLimit;
+ Resource totalResourceLimit = getResourceLimit();
+
+ Resource idealReduceResourceLimit =
+ Resources.multiply(totalResourceLimit,
+ Math.min(completedMapPercent, maxReduceRampupLimit));
+ Resource ideaMapResourceLimit =
+ Resources.subtract(totalResourceLimit, idealReduceResourceLimit);
// check if there aren't enough maps scheduled, give the free map capacity
- // to reduce
- if (idealMapMemLimit > netScheduledMapMem) {
- int unusedMapMemLimit = idealMapMemLimit - netScheduledMapMem;
- finalReduceMemLimit = idealReduceMemLimit + unusedMapMemLimit;
- finalMapMemLimit = totalMemLimit - finalReduceMemLimit;
+ // to reduce.
+ // Even when container number equals, there may be unused resources in one
+ // dimension
+ if (ResourceCalculatorUtils.computeAvailableContainers(ideaMapResourceLimit,
+ mapResourceReqt, getSchedulerResourceTypes()) >= (scheduledMaps + assignedMaps)) {
+ // enough resource given to maps, given the remaining to reduces
+ Resource unusedMapResourceLimit =
+ Resources.subtract(ideaMapResourceLimit, netScheduledMapResource);
+ finalReduceResourceLimit =
+ Resources.add(idealReduceResourceLimit, unusedMapResourceLimit);
+ finalMapResourceLimit =
+ Resources.subtract(totalResourceLimit, finalReduceResourceLimit);
} else {
- finalMapMemLimit = idealMapMemLimit;
- finalReduceMemLimit = idealReduceMemLimit;
+ finalMapResourceLimit = ideaMapResourceLimit;
+ finalReduceResourceLimit = idealReduceResourceLimit;
}
-
- LOG.info("completedMapPercent " + completedMapPercent +
- " totalMemLimit:" + totalMemLimit +
- " finalMapMemLimit:" + finalMapMemLimit +
- " finalReduceMemLimit:" + finalReduceMemLimit +
- " netScheduledMapMem:" + netScheduledMapMem +
- " netScheduledReduceMem:" + netScheduledReduceMem);
-
- int rampUp =
- (finalReduceMemLimit - netScheduledReduceMem) / reduceResourceReqt;
-
+
+ LOG.info("completedMapPercent " + completedMapPercent
+ + " totalResourceLimit:" + totalResourceLimit
+ + " finalMapResourceLimit:" + finalMapResourceLimit
+ + " finalReduceResourceLimit:" + finalReduceResourceLimit
+ + " netScheduledMapResource:" + netScheduledMapResource
+ + " netScheduledReduceResource:" + netScheduledReduceResource);
+
+ int rampUp =
+ ResourceCalculatorUtils.computeAvailableContainers(Resources.subtract(
+ finalReduceResourceLimit, netScheduledReduceResource),
+ reduceResourceReqt, getSchedulerResourceTypes());
+
if (rampUp > 0) {
rampUp = Math.min(rampUp, numPendingReduces);
LOG.info("Ramping up " + rampUp);
rampUpReduces(rampUp);
- } else if (rampUp < 0){
+ } else if (rampUp < 0) {
int rampDown = -1 * rampUp;
rampDown = Math.min(rampDown, scheduledReduces);
LOG.info("Ramping down " + rampDown);
@@ -618,8 +661,10 @@ public class RMContainerAllocator extends RMContainerRequestor
@SuppressWarnings("unchecked")
private List<Container> getResources() throws Exception {
- int headRoom = getAvailableResources() != null
- ? getAvailableResources().getMemory() : 0;//first time it would be null
+ // will be null the first time
+ Resource headRoom =
+ getAvailableResources() == null ? Resources.none() :
+ Resources.clone(getAvailableResources());
AllocateResponse response;
/*
* If contact with RM is lost, the AM will wait MR_AM_TO_RM_WAIT_INTERVAL_MS
@@ -670,7 +715,9 @@ public class RMContainerAllocator extends RMContainerRequestor
throw new YarnRuntimeException(msg);
}
}
- int newHeadRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
+ Resource newHeadRoom =
+ getAvailableResources() == null ? Resources.none()
+ : getAvailableResources();
List<Container> newContainers = response.getAllocatedContainers();
// Setting NMTokens
if (response.getNMTokens() != null) {
@@ -694,10 +741,11 @@ public class RMContainerAllocator extends RMContainerRequestor
new PreemptionContext(assignedRequests), preemptReq);
}
- if (newContainers.size() + finishedContainers.size() > 0 || headRoom != newHeadRoom) {
+ if (newContainers.size() + finishedContainers.size() > 0
+ || !headRoom.equals(newHeadRoom)) {
//something changed
recalculateReduceSchedule = true;
- if (LOG.isDebugEnabled() && headRoom != newHeadRoom) {
+ if (LOG.isDebugEnabled() && !headRoom.equals(newHeadRoom)) {
LOG.debug("headroom=" + newHeadRoom);
}
}
@@ -802,10 +850,18 @@ public class RMContainerAllocator extends RMContainerRequestor
}
@Private
- public int getMemLimit() {
- int headRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
- return headRoom + assignedRequests.maps.size() * mapResourceRequest +
- assignedRequests.reduces.size() * reduceResourceRequest;
+ public Resource getResourceLimit() {
+ Resource headRoom = getAvailableResources();
+ if (headRoom == null) {
+ headRoom = Resources.none();
+ }
+ Resource assignedMapResource =
+ Resources.multiply(mapResourceRequest, assignedRequests.maps.size());
+ Resource assignedReduceResource =
+ Resources.multiply(reduceResourceRequest,
+ assignedRequests.reduces.size());
+ return Resources.add(headRoom,
+ Resources.add(assignedMapResource, assignedReduceResource));
}
@Private
@@ -914,10 +970,11 @@ public class RMContainerAllocator extends RMContainerRequestor
// a container to be assigned
boolean isAssignable = true;
Priority priority = allocated.getPriority();
- int allocatedMemory = allocated.getResource().getMemory();
+ Resource allocatedResource = allocated.getResource();
if (PRIORITY_FAST_FAIL_MAP.equals(priority)
|| PRIORITY_MAP.equals(priority)) {
- if (allocatedMemory < mapResourceRequest
+ if (ResourceCalculatorUtils.computeAvailableContainers(allocatedResource,
+ mapResourceRequest, getSchedulerResourceTypes()) <= 0
|| maps.isEmpty()) {
LOG.info("Cannot assign container " + allocated
+ " for a map as either "
@@ -928,7 +985,8 @@ public class RMContainerAllocator extends RMContainerRequestor
}
}
else if (PRIORITY_REDUCE.equals(priority)) {
- if (allocatedMemory < reduceResourceRequest
+ if (ResourceCalculatorUtils.computeAvailableContainers(allocatedResource,
+ reduceResourceRequest, getSchedulerResourceTypes()) <= 0
|| reduces.isEmpty()) {
LOG.info("Cannot assign container " + allocated
+ " for a reduce as either "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376233cd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java
new file mode 100644
index 0000000..b9bc8b5
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.app.rm;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.util.EnumSet;
+
+public class ResourceCalculatorUtils {
+ public static int divideAndCeil(int a, int b) {
+ if (b == 0) {
+ return 0;
+ }
+ return (a + (b - 1)) / b;
+ }
+
+ public static int computeAvailableContainers(Resource available,
+ Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) {
+ if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
+ return Math.min(available.getMemory() / required.getMemory(),
+ available.getVirtualCores() / required.getVirtualCores());
+ }
+ return available.getMemory() / required.getMemory();
+ }
+
+ public static int divideAndCeilContainers(Resource required, Resource factor,
+ EnumSet<SchedulerResourceTypes> resourceTypes) {
+ if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
+ return Math.max(divideAndCeil(required.getMemory(), factor.getMemory()),
+ divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()));
+ }
+ return divideAndCeil(required.getMemory(), factor.getMemory());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376233cd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 4c03b31..341e673 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapreduce.v2.app.rm;
+import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyFloat;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.isA;
@@ -30,19 +31,14 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.ClusterInfo;
import org.apache.hadoop.mapreduce.v2.app.MRApp;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -458,8 +454,8 @@ public class TestRMContainerAllocator {
0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
appAttemptId, mockJob, new SystemClock());
- allocator.setMapResourceRequest(1024);
- allocator.setReduceResourceRequest(1024);
+ allocator.setMapResourceRequest(BuilderUtils.newResource(1024, 1));
+ allocator.setReduceResourceRequest(BuilderUtils.newResource(1024, 1));
RMContainerAllocator.AssignedRequests assignedRequests =
allocator.getAssignedRequests();
RMContainerAllocator.ScheduledRequests scheduledRequests =
@@ -478,7 +474,7 @@ public class TestRMContainerAllocator {
@Test(timeout = 30000)
public void testNonAggressivelyPreemptReducers() throws Exception {
- LOG.info("Running testPreemptReducers");
+ LOG.info("Running testNonAggressivelyPreemptReducers");
final int preemptThreshold = 2; //sec
Configuration conf = new Configuration();
@@ -513,8 +509,8 @@ public class TestRMContainerAllocator {
clock.setTime(1);
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
appAttemptId, mockJob, clock);
- allocator.setMapResourceRequest(1024);
- allocator.setReduceResourceRequest(1024);
+ allocator.setMapResourceRequest(BuilderUtils.newResource(1024, 1));
+ allocator.setReduceResourceRequest(BuilderUtils.newResource(1024, 1));
RMContainerAllocator.AssignedRequests assignedRequests =
allocator.getAssignedRequests();
RMContainerAllocator.ScheduledRequests scheduledRequests =
@@ -1774,17 +1770,19 @@ public class TestRMContainerAllocator {
int scheduledReduces = 0;
int assignedMaps = 2;
int assignedReduces = 0;
- int mapResourceReqt = 1024;
- int reduceResourceReqt = 2*1024;
+ Resource mapResourceReqt = BuilderUtils.newResource(1024, 1);
+ Resource reduceResourceReqt = BuilderUtils.newResource(2 * 1024, 1);
int numPendingReduces = 4;
float maxReduceRampupLimit = 0.5f;
float reduceSlowStart = 0.2f;
RMContainerAllocator allocator = mock(RMContainerAllocator.class);
- doCallRealMethod().when(allocator).
- scheduleReduces(anyInt(), anyInt(), anyInt(), anyInt(), anyInt(),
- anyInt(), anyInt(), anyInt(), anyInt(), anyFloat(), anyFloat());
-
+ doCallRealMethod().when(allocator).scheduleReduces(anyInt(), anyInt(),
+ anyInt(), anyInt(), anyInt(), anyInt(), any(Resource.class),
+ any(Resource.class), anyInt(), anyFloat(), anyFloat());
+ doReturn(EnumSet.of(SchedulerResourceTypes.MEMORY)).when(allocator)
+ .getSchedulerResourceTypes();
+
// Test slow-start
allocator.scheduleReduces(
totalMaps, succeededMaps,
@@ -1808,6 +1806,7 @@ public class TestRMContainerAllocator {
verify(allocator, never()).scheduleAllReduces();
succeededMaps = 3;
+ doReturn(BuilderUtils.newResource(0, 0)).when(allocator).getResourceLimit();
allocator.scheduleReduces(
totalMaps, succeededMaps,
scheduledMaps, scheduledReduces,
@@ -1818,7 +1817,8 @@ public class TestRMContainerAllocator {
verify(allocator, times(1)).setIsReduceStarted(true);
// Test reduce ramp-up
- doReturn(100 * 1024).when(allocator).getMemLimit();
+ doReturn(BuilderUtils.newResource(100 * 1024, 100 * 1)).when(allocator)
+ .getResourceLimit();
allocator.scheduleReduces(
totalMaps, succeededMaps,
scheduledMaps, scheduledReduces,
@@ -1831,13 +1831,14 @@ public class TestRMContainerAllocator {
// Test reduce ramp-down
scheduledReduces = 3;
- doReturn(10 * 1024).when(allocator).getMemLimit();
+ doReturn(BuilderUtils.newResource(10 * 1024, 10 * 1)).when(allocator)
+ .getResourceLimit();
allocator.scheduleReduces(
- totalMaps, succeededMaps,
- scheduledMaps, scheduledReduces,
- assignedMaps, assignedReduces,
- mapResourceReqt, reduceResourceReqt,
- numPendingReduces,
+ totalMaps, succeededMaps,
+ scheduledMaps, scheduledReduces,
+ assignedMaps, assignedReduces,
+ mapResourceReqt, reduceResourceReqt,
+ numPendingReduces,
maxReduceRampupLimit, reduceSlowStart);
verify(allocator).rampDownReduces(anyInt());
@@ -1846,7 +1847,8 @@ public class TestRMContainerAllocator {
// should be invoked twice.
scheduledMaps = 2;
assignedReduces = 2;
- doReturn(10 * 1024).when(allocator).getMemLimit();
+ doReturn(BuilderUtils.newResource(10 * 1024, 10 * 1)).when(allocator)
+ .getResourceLimit();
allocator.scheduleReduces(
totalMaps, succeededMaps,
scheduledMaps, scheduledReduces,
@@ -1855,6 +1857,30 @@ public class TestRMContainerAllocator {
numPendingReduces,
maxReduceRampupLimit, reduceSlowStart);
verify(allocator, times(2)).rampDownReduces(anyInt());
+
+ doReturn(
+ EnumSet.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU))
+ .when(allocator).getSchedulerResourceTypes();
+
+ // Test ramp-down when enough memory but not enough cpu resource
+ scheduledMaps = 10;
+ assignedReduces = 0;
+ doReturn(BuilderUtils.newResource(100 * 1024, 5 * 1)).when(allocator)
+ .getResourceLimit();
+ allocator.scheduleReduces(totalMaps, succeededMaps, scheduledMaps,
+ scheduledReduces, assignedMaps, assignedReduces, mapResourceReqt,
+ reduceResourceReqt, numPendingReduces, maxReduceRampupLimit,
+ reduceSlowStart);
+ verify(allocator, times(3)).rampDownReduces(anyInt());
+
+ // Test ramp-down when enough cpu but not enough memory resource
+ doReturn(BuilderUtils.newResource(10 * 1024, 100 * 1)).when(allocator)
+ .getResourceLimit();
+ allocator.scheduleReduces(totalMaps, succeededMaps, scheduledMaps,
+ scheduledReduces, assignedMaps, assignedReduces, mapResourceReqt,
+ reduceResourceReqt, numPendingReduces, maxReduceRampupLimit,
+ reduceSlowStart);
+ verify(allocator, times(4)).rampDownReduces(anyInt());
}
private static class RecalculateContainerAllocator extends MyContainerAllocator {
@@ -1868,7 +1894,7 @@ public class TestRMContainerAllocator {
@Override
public void scheduleReduces(int totalMaps, int completedMaps,
int scheduledMaps, int scheduledReduces, int assignedMaps,
- int assignedReduces, int mapResourceReqt, int reduceResourceReqt,
+ int assignedReduces, Resource mapResourceReqt, Resource reduceResourceReqt,
int numPendingReduces, float maxReduceRampupLimit, float reduceSlowStart) {
recalculatedReduceSchedule = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376233cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 077c962..a205bd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -18,12 +18,12 @@
package org.apache.hadoop.yarn.util.resource;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
-@Private
+@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
@Unstable
public class Resources {
[43/50] [abbrv] YARN-1709. In-memory data structures used to track
resources over time to enable reservations.
Posted by cu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
new file mode 100644
index 0000000..6dcd41f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
@@ -0,0 +1,477 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestInMemoryPlan {
+
+ private String user = "yarn";
+ private String planName = "test-reservation";
+ private ResourceCalculator resCalc;
+ private Resource minAlloc;
+ private Resource maxAlloc;
+ private Resource totalCapacity;
+
+ private Clock clock;
+ private QueueMetrics queueMetrics;
+ private SharingPolicy policy;
+ private ReservationAgent agent;
+ private Planner replanner;
+
+ @Before
+ public void setUp() throws PlanningException {
+ resCalc = new DefaultResourceCalculator();
+ minAlloc = Resource.newInstance(1024, 1);
+ maxAlloc = Resource.newInstance(64 * 1024, 20);
+ totalCapacity = Resource.newInstance(100 * 1024, 100);
+
+ clock = mock(Clock.class);
+ queueMetrics = mock(QueueMetrics.class);
+ policy = mock(SharingPolicy.class);
+ replanner = mock(Planner.class);
+
+ when(clock.getTime()).thenReturn(1L);
+ }
+
+ @After
+ public void tearDown() {
+ resCalc = null;
+ minAlloc = null;
+ maxAlloc = null;
+ totalCapacity = null;
+
+ clock = null;
+ queueMetrics = null;
+ policy = null;
+ replanner = null;
+ }
+
+ @Test
+ public void testAddReservation() {
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true);
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ Map<ReservationInterval, ReservationRequest> allocations =
+ generateAllocation(start, alloc, false);
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length,
+ alloc.length, allocations.values());
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length, allocations, resCalc, minAlloc);
+ Assert.assertNull(plan.getReservationById(reservationID));
+ try {
+ plan.addReservation(rAllocation);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ doAssertions(plan, rAllocation);
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ plan.getConsumptionForUser(user, start + i));
+ }
+ }
+
+ @Test
+ public void testAddEmptyReservation() {
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true);
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = {};
+ int start = 100;
+ Map<ReservationInterval, ReservationRequest> allocations =
+ new HashMap<ReservationInterval, ReservationRequest>();
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length,
+ alloc.length, allocations.values());
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length, allocations, resCalc, minAlloc);
+ Assert.assertNull(plan.getReservationById(reservationID));
+ try {
+ plan.addReservation(rAllocation);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testAddReservationAlreadyExists() {
+ // First add a reservation
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true);
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ Map<ReservationInterval, ReservationRequest> allocations =
+ generateAllocation(start, alloc, false);
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length,
+ alloc.length, allocations.values());
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length, allocations, resCalc, minAlloc);
+ Assert.assertNull(plan.getReservationById(reservationID));
+ try {
+ plan.addReservation(rAllocation);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ doAssertions(plan, rAllocation);
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ plan.getConsumptionForUser(user, start + i));
+ }
+
+ // Try to add it again
+ try {
+ plan.addReservation(rAllocation);
+ Assert.fail("Add should fail as it already exists");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().endsWith("already exists"));
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ doAssertions(plan, rAllocation);
+ }
+
+ @Test
+ public void testUpdateReservation() {
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true);
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ // First add a reservation
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ Map<ReservationInterval, ReservationRequest> allocations =
+ generateAllocation(start, alloc, false);
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length,
+ alloc.length, allocations.values());
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length, allocations, resCalc, minAlloc);
+ Assert.assertNull(plan.getReservationById(reservationID));
+ try {
+ plan.addReservation(rAllocation);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ doAssertions(plan, rAllocation);
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ plan.getConsumptionForUser(user, start + i));
+ }
+
+ // Now update it
+ start = 110;
+ int[] updatedAlloc = { 0, 5, 10, 10, 5, 0 };
+ allocations = generateAllocation(start, updatedAlloc, true);
+ rDef =
+ createSimpleReservationDefinition(start, start + updatedAlloc.length,
+ updatedAlloc.length, allocations.values());
+ rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + updatedAlloc.length, allocations, resCalc, minAlloc);
+ try {
+ plan.updateReservation(rAllocation);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ doAssertions(plan, rAllocation);
+ for (int i = 0; i < updatedAlloc.length; i++) {
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (updatedAlloc[i] + i), updatedAlloc[i]
+ + i), plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (updatedAlloc[i] + i), updatedAlloc[i]
+ + i), plan.getConsumptionForUser(user, start + i));
+ }
+ }
+
+ @Test
+ public void testUpdateNonExistingReservation() {
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true);
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ // Try to update a reservation without adding
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ Map<ReservationInterval, ReservationRequest> allocations =
+ generateAllocation(start, alloc, false);
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length,
+ alloc.length, allocations.values());
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length, allocations, resCalc, minAlloc);
+ Assert.assertNull(plan.getReservationById(reservationID));
+ try {
+ plan.updateReservation(rAllocation);
+ Assert.fail("Update should fail as it does not exist in the plan");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().endsWith("does not exist in the plan"));
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNull(plan.getReservationById(reservationID));
+ }
+
+ @Test
+ public void testDeleteReservation() {
+ // First add a reservation
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true);
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ Map<ReservationInterval, ReservationRequest> allocations =
+ generateAllocation(start, alloc, true);
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length,
+ alloc.length, allocations.values());
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length, allocations, resCalc, minAlloc);
+ Assert.assertNull(plan.getReservationById(reservationID));
+ try {
+ plan.addReservation(rAllocation);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ doAssertions(plan, rAllocation);
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)),
+ plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)),
+ plan.getConsumptionForUser(user, start + i));
+ }
+
+ // Now delete it
+ try {
+ plan.deleteReservation(reservationID);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNull(plan.getReservationById(reservationID));
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ plan.getConsumptionForUser(user, start + i));
+ }
+ }
+
+ @Test
+ public void testDeleteNonExistingReservation() {
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true);
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ // Try to delete a reservation without adding
+ Assert.assertNull(plan.getReservationById(reservationID));
+ try {
+ plan.deleteReservation(reservationID);
+ Assert.fail("Delete should fail as it does not exist in the plan");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().endsWith("does not exist in the plan"));
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNull(plan.getReservationById(reservationID));
+ }
+
+ @Test
+ public void testArchiveCompletedReservations() {
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true);
+ ReservationId reservationID1 =
+ ReservationSystemTestUtil.getNewReservationId();
+ // First add a reservation
+ int[] alloc1 = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ Map<ReservationInterval, ReservationRequest> allocations1 =
+ generateAllocation(start, alloc1, false);
+ ReservationDefinition rDef1 =
+ createSimpleReservationDefinition(start, start + alloc1.length,
+ alloc1.length, allocations1.values());
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID1, rDef1, user,
+ planName, start, start + alloc1.length, allocations1, resCalc,
+ minAlloc);
+ Assert.assertNull(plan.getReservationById(reservationID1));
+ try {
+ plan.addReservation(rAllocation);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ doAssertions(plan, rAllocation);
+ for (int i = 0; i < alloc1.length; i++) {
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc1[i]), (alloc1[i])),
+ plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc1[i]), (alloc1[i])),
+ plan.getConsumptionForUser(user, start + i));
+ }
+
+ // Now add another one
+ ReservationId reservationID2 =
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc2 = { 0, 5, 10, 5, 0 };
+ Map<ReservationInterval, ReservationRequest> allocations2 =
+ generateAllocation(start, alloc2, true);
+ ReservationDefinition rDef2 =
+ createSimpleReservationDefinition(start, start + alloc2.length,
+ alloc2.length, allocations2.values());
+ rAllocation =
+ new InMemoryReservationAllocation(reservationID2, rDef2, user,
+ planName, start, start + alloc2.length, allocations2, resCalc,
+ minAlloc);
+ Assert.assertNull(plan.getReservationById(reservationID2));
+ try {
+ plan.addReservation(rAllocation);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(plan.getReservationById(reservationID2));
+ for (int i = 0; i < alloc2.length; i++) {
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i), alloc1[i]
+ + alloc2[i] + i), plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i), alloc1[i]
+ + alloc2[i] + i), plan.getConsumptionForUser(user, start + i));
+ }
+
+ // Now archive completed reservations
+ when(clock.getTime()).thenReturn(106L);
+ when(policy.getValidWindow()).thenReturn(1L);
+ try {
+ // will only remove 2nd reservation as only that has fallen out of the
+ // archival window
+ plan.archiveCompletedReservations(clock.getTime());
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(plan.getReservationById(reservationID1));
+ Assert.assertNull(plan.getReservationById(reservationID2));
+ for (int i = 0; i < alloc1.length; i++) {
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc1[i]), (alloc1[i])),
+ plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc1[i]), (alloc1[i])),
+ plan.getConsumptionForUser(user, start + i));
+ }
+ when(clock.getTime()).thenReturn(107L);
+ try {
+ // will remove 1st reservation also as it has fallen out of the archival
+ // window
+ plan.archiveCompletedReservations(clock.getTime());
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNull(plan.getReservationById(reservationID1));
+ for (int i = 0; i < alloc1.length; i++) {
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ plan.getConsumptionForUser(user, start + i));
+ }
+ }
+
+ private void doAssertions(Plan plan, ReservationAllocation rAllocation) {
+ ReservationId reservationID = rAllocation.getReservationId();
+ Assert.assertNotNull(plan.getReservationById(reservationID));
+ Assert.assertEquals(rAllocation, plan.getReservationById(reservationID));
+ Assert.assertTrue(((InMemoryPlan) plan).getAllReservations().size() == 1);
+ Assert.assertEquals(rAllocation.getEndTime(), plan.getLastEndTime());
+ Assert.assertEquals(totalCapacity, plan.getTotalCapacity());
+ Assert.assertEquals(minAlloc, plan.getMinimumAllocation());
+ Assert.assertEquals(maxAlloc, plan.getMaximumAllocation());
+ Assert.assertEquals(resCalc, plan.getResourceCalculator());
+ Assert.assertEquals(planName, plan.getQueueName());
+ Assert.assertTrue(plan.getMoveOnExpiry());
+ }
+
+ private ReservationDefinition createSimpleReservationDefinition(long arrival,
+ long deadline, long duration, Collection<ReservationRequest> resources) {
+ // create a request with a single atomic ask
+ ReservationDefinition rDef = new ReservationDefinitionPBImpl();
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setReservationResources(new ArrayList<ReservationRequest>(resources));
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
+ rDef.setReservationRequests(reqs);
+ rDef.setArrival(arrival);
+ rDef.setDeadline(deadline);
+ return rDef;
+ }
+
+ private Map<ReservationInterval, ReservationRequest> generateAllocation(
+ int startTime, int[] alloc, boolean isStep) {
+ Map<ReservationInterval, ReservationRequest> req =
+ new HashMap<ReservationInterval, ReservationRequest>();
+ int numContainers = 0;
+ for (int i = 0; i < alloc.length; i++) {
+ if (isStep) {
+ numContainers = alloc[i] + i;
+ } else {
+ numContainers = alloc[i];
+ }
+ ReservationRequest rr =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ (numContainers));
+ req.put(new ReservationInterval(startTime + i, startTime + i + 1), rr);
+ }
+ return req;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryReservationAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryReservationAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryReservationAllocation.java
new file mode 100644
index 0000000..f4c4581
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryReservationAllocation.java
@@ -0,0 +1,206 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestInMemoryReservationAllocation {
+
+ private String user = "yarn";
+ private String planName = "test-reservation";
+ private ResourceCalculator resCalc;
+ private Resource minAlloc;
+
+ private Random rand = new Random();
+
+ @Before
+ public void setUp() {
+ resCalc = new DefaultResourceCalculator();
+ minAlloc = Resource.newInstance(1, 1);
+ }
+
+ @After
+ public void tearDown() {
+ user = null;
+ planName = null;
+ resCalc = null;
+ minAlloc = null;
+ }
+
+ @Test
+ public void testBlocks() {
+ ReservationId reservationID =
+ ReservationId.newInstance(rand.nextLong(), rand.nextLong());
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length + 1,
+ alloc.length);
+ Map<ReservationInterval, ReservationRequest> allocations =
+ generateAllocation(start, alloc, false, false);
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length + 1, allocations, resCalc, minAlloc);
+ doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
+ Assert.assertFalse(rAllocation.containsGangs());
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ rAllocation.getResourcesAtTime(start + i));
+ }
+ }
+
+ @Test
+ public void testSteps() {
+ ReservationId reservationID =
+ ReservationId.newInstance(rand.nextLong(), rand.nextLong());
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length + 1,
+ alloc.length);
+ Map<ReservationInterval, ReservationRequest> allocations =
+ generateAllocation(start, alloc, true, false);
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length + 1, allocations, resCalc, minAlloc);
+ doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
+ Assert.assertFalse(rAllocation.containsGangs());
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)),
+ rAllocation.getResourcesAtTime(start + i));
+ }
+ }
+
+ @Test
+ public void testSkyline() {
+ ReservationId reservationID =
+ ReservationId.newInstance(rand.nextLong(), rand.nextLong());
+ int[] alloc = { 0, 5, 10, 10, 5, 0 };
+ int start = 100;
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length + 1,
+ alloc.length);
+ Map<ReservationInterval, ReservationRequest> allocations =
+ generateAllocation(start, alloc, true, false);
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length + 1, allocations, resCalc, minAlloc);
+ doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
+ Assert.assertFalse(rAllocation.containsGangs());
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)),
+ rAllocation.getResourcesAtTime(start + i));
+ }
+ }
+
+ @Test
+ public void testZeroAlloaction() {
+ ReservationId reservationID =
+ ReservationId.newInstance(rand.nextLong(), rand.nextLong());
+ int[] alloc = {};
+ long start = 0;
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length + 1,
+ alloc.length);
+ Map<ReservationInterval, ReservationRequest> allocations =
+ new HashMap<ReservationInterval, ReservationRequest>();
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length + 1, allocations, resCalc, minAlloc);
+ doAssertions(rAllocation, reservationID, rDef, allocations, (int) start,
+ alloc);
+ Assert.assertFalse(rAllocation.containsGangs());
+ }
+
+ @Test
+ public void testGangAlloaction() {
+ ReservationId reservationID =
+ ReservationId.newInstance(rand.nextLong(), rand.nextLong());
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ ReservationDefinition rDef =
+ createSimpleReservationDefinition(start, start + alloc.length + 1,
+ alloc.length);
+ Map<ReservationInterval, ReservationRequest> allocations =
+ generateAllocation(start, alloc, false, true);
+ ReservationAllocation rAllocation =
+ new InMemoryReservationAllocation(reservationID, rDef, user, planName,
+ start, start + alloc.length + 1, allocations, resCalc, minAlloc);
+ doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
+ Assert.assertTrue(rAllocation.containsGangs());
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ rAllocation.getResourcesAtTime(start + i));
+ }
+ }
+
+ private void doAssertions(ReservationAllocation rAllocation,
+ ReservationId reservationID, ReservationDefinition rDef,
+ Map<ReservationInterval, ReservationRequest> allocations, int start,
+ int[] alloc) {
+ Assert.assertEquals(reservationID, rAllocation.getReservationId());
+ Assert.assertEquals(rDef, rAllocation.getReservationDefinition());
+ Assert.assertEquals(allocations, rAllocation.getAllocationRequests());
+ Assert.assertEquals(user, rAllocation.getUser());
+ Assert.assertEquals(planName, rAllocation.getPlanName());
+ Assert.assertEquals(start, rAllocation.getStartTime());
+ Assert.assertEquals(start + alloc.length + 1, rAllocation.getEndTime());
+ }
+
+ private ReservationDefinition createSimpleReservationDefinition(long arrival,
+ long deadline, long duration) {
+ // create a request with a single atomic ask
+ ReservationRequest r =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1), 1, 1,
+ duration);
+ ReservationDefinition rDef = new ReservationDefinitionPBImpl();
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setReservationResources(Collections.singletonList(r));
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
+ rDef.setReservationRequests(reqs);
+ rDef.setArrival(arrival);
+ rDef.setDeadline(deadline);
+ return rDef;
+ }
+
+ private Map<ReservationInterval, ReservationRequest> generateAllocation(
+ int startTime, int[] alloc, boolean isStep, boolean isGang) {
+ Map<ReservationInterval, ReservationRequest> req =
+ new HashMap<ReservationInterval, ReservationRequest>();
+ int numContainers = 0;
+ for (int i = 0; i < alloc.length; i++) {
+ if (isStep) {
+ numContainers = alloc[i] + i;
+ } else {
+ numContainers = alloc[i];
+ }
+ ReservationRequest rr =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ (numContainers));
+ if (isGang) {
+ rr.setConcurrency(numContainers);
+ }
+ req.put(new ReservationInterval(startTime + i, startTime + i + 1), rr);
+ }
+ return req;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a83267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
new file mode 100644
index 0000000..ab0de6b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
@@ -0,0 +1,169 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestRLESparseResourceAllocation {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestRLESparseResourceAllocation.class);
+
+ @Test
+ public void testBlocks() {
+ ResourceCalculator resCalc = new DefaultResourceCalculator();
+ Resource minAlloc = Resource.newInstance(1, 1);
+
+ RLESparseResourceAllocation rleSparseVector =
+ new RLESparseResourceAllocation(resCalc, minAlloc);
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ Set<Entry<ReservationInterval, ReservationRequest>> inputs =
+ generateAllocation(start, alloc, false).entrySet();
+ for (Entry<ReservationInterval, ReservationRequest> ip : inputs) {
+ rleSparseVector.addInterval(ip.getKey(), ip.getValue());
+ }
+ LOG.info(rleSparseVector.toString());
+ Assert.assertFalse(rleSparseVector.isEmpty());
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(99));
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(start + alloc.length + 1));
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ rleSparseVector.getCapacityAtTime(start + i));
+ }
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(start + alloc.length + 2));
+ for (Entry<ReservationInterval, ReservationRequest> ip : inputs) {
+ rleSparseVector.removeInterval(ip.getKey(), ip.getValue());
+ }
+ LOG.info(rleSparseVector.toString());
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(start + i));
+ }
+ Assert.assertTrue(rleSparseVector.isEmpty());
+ }
+
+ @Test
+ public void testSteps() {
+ ResourceCalculator resCalc = new DefaultResourceCalculator();
+ Resource minAlloc = Resource.newInstance(1, 1);
+
+ RLESparseResourceAllocation rleSparseVector =
+ new RLESparseResourceAllocation(resCalc, minAlloc);
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ int start = 100;
+ Set<Entry<ReservationInterval, ReservationRequest>> inputs =
+ generateAllocation(start, alloc, true).entrySet();
+ for (Entry<ReservationInterval, ReservationRequest> ip : inputs) {
+ rleSparseVector.addInterval(ip.getKey(), ip.getValue());
+ }
+ LOG.info(rleSparseVector.toString());
+ Assert.assertFalse(rleSparseVector.isEmpty());
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(99));
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(start + alloc.length + 1));
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)),
+ rleSparseVector.getCapacityAtTime(start + i));
+ }
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(start + alloc.length + 2));
+ for (Entry<ReservationInterval, ReservationRequest> ip : inputs) {
+ rleSparseVector.removeInterval(ip.getKey(), ip.getValue());
+ }
+ LOG.info(rleSparseVector.toString());
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(start + i));
+ }
+ Assert.assertTrue(rleSparseVector.isEmpty());
+ }
+
+ @Test
+ public void testSkyline() {
+ ResourceCalculator resCalc = new DefaultResourceCalculator();
+ Resource minAlloc = Resource.newInstance(1, 1);
+
+ RLESparseResourceAllocation rleSparseVector =
+ new RLESparseResourceAllocation(resCalc, minAlloc);
+ int[] alloc = { 0, 5, 10, 10, 5, 0 };
+ int start = 100;
+ Set<Entry<ReservationInterval, ReservationRequest>> inputs =
+ generateAllocation(start, alloc, true).entrySet();
+ for (Entry<ReservationInterval, ReservationRequest> ip : inputs) {
+ rleSparseVector.addInterval(ip.getKey(), ip.getValue());
+ }
+ LOG.info(rleSparseVector.toString());
+ Assert.assertFalse(rleSparseVector.isEmpty());
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(99));
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(start + alloc.length + 1));
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)),
+ rleSparseVector.getCapacityAtTime(start + i));
+ }
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(start + alloc.length + 2));
+ for (Entry<ReservationInterval, ReservationRequest> ip : inputs) {
+ rleSparseVector.removeInterval(ip.getKey(), ip.getValue());
+ }
+ LOG.info(rleSparseVector.toString());
+ for (int i = 0; i < alloc.length; i++) {
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(start + i));
+ }
+ Assert.assertTrue(rleSparseVector.isEmpty());
+ }
+
+ @Test
+ public void testZeroAlloaction() {
+ ResourceCalculator resCalc = new DefaultResourceCalculator();
+ Resource minAlloc = Resource.newInstance(1, 1);
+ RLESparseResourceAllocation rleSparseVector =
+ new RLESparseResourceAllocation(resCalc, minAlloc);
+ rleSparseVector.addInterval(new ReservationInterval(0, Long.MAX_VALUE),
+ ReservationRequest.newInstance(Resource.newInstance(0, 0), (0)));
+ LOG.info(rleSparseVector.toString());
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ rleSparseVector.getCapacityAtTime(new Random().nextLong()));
+ Assert.assertTrue(rleSparseVector.isEmpty());
+ }
+
+ private Map<ReservationInterval, ReservationRequest> generateAllocation(
+ int startTime, int[] alloc, boolean isStep) {
+ Map<ReservationInterval, ReservationRequest> req =
+ new HashMap<ReservationInterval, ReservationRequest>();
+ int numContainers = 0;
+ for (int i = 0; i < alloc.length; i++) {
+ if (isStep) {
+ numContainers = alloc[i] + i;
+ } else {
+ numContainers = alloc[i];
+ }
+ req.put(new ReservationInterval(startTime + i, startTime + i + 1),
+
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ (numContainers)));
+ }
+ return req;
+ }
+
+}
[49/50] [abbrv] git commit: YARN-2080. Integrating reservation system
with ResourceManager and client-RM protocol. Contributed by Subru Krishnan
and Carlo Curino.
Posted by cu...@apache.org.
YARN-2080. Integrating reservation system with ResourceManager and client-RM protocol. Contributed by Subru Krishnan and Carlo Curino.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e53fdea3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e53fdea3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e53fdea3
Branch: refs/heads/YARN-1051
Commit: e53fdea301930244ba4d43cb55f5cae2450a9dd2
Parents: 4a79534
Author: subru <su...@outlook.com>
Authored: Thu Sep 18 15:30:27 2014 -0700
Committer: carlo curino <Carlo Curino>
Committed: Mon Sep 22 16:35:08 2014 -0700
----------------------------------------------------------------------
YARN-1051-CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 19 +
.../hadoop/yarn/client/api/YarnClient.java | 102 ++++
.../yarn/client/api/impl/YarnClientImpl.java | 25 +
.../yarn/client/api/impl/TestYarnClient.java | 115 ++++
.../server/resourcemanager/AdminService.java | 6 +
.../server/resourcemanager/ClientRMService.java | 201 +++++++
.../server/resourcemanager/RMAuditLogger.java | 5 +
.../yarn/server/resourcemanager/RMContext.java | 4 +
.../server/resourcemanager/RMContextImpl.java | 11 +
.../server/resourcemanager/ResourceManager.java | 38 ++
.../reservation/AbstractReservationSystem.java | 323 +++++++++++
.../reservation/CapacityReservationSystem.java | 146 +++++
.../reservation/ReservationInputValidator.java | 244 ++++++++
.../reservation/ReservationSystem.java | 125 +++++
.../server/resourcemanager/rmapp/RMApp.java | 3 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 15 +-
.../scheduler/event/AppAddedSchedulerEvent.java | 16 +-
.../resourcemanager/TestClientRMService.java | 116 ++++
.../TestCapacityReservationSystem.java | 102 ++++
.../TestReservationInputValidator.java | 560 +++++++++++++++++++
21 files changed, 2174 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/YARN-1051-CHANGES.txt
----------------------------------------------------------------------
diff --git a/YARN-1051-CHANGES.txt b/YARN-1051-CHANGES.txt
index 56b3c12..c4106b2 100644
--- a/YARN-1051-CHANGES.txt
+++ b/YARN-1051-CHANGES.txt
@@ -20,3 +20,6 @@ on user reservations. (Carlo Curino and Subru Krishnan via curino)
YARN-1712. Plan follower that synchronizes the current state of reservation
subsystem with the scheduler. (Subru Krishnan and Carlo Curino via subru)
+
+YARN-2080. Integrating reservation system with ResourceManager and
+client-RM protocol. (Subru Krishnan and Carlo Curino via subru)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index acc4a05..325c17f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -176,6 +176,25 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME =
false;
+ /** Whether the RM should enable Reservation System */
+ public static final String RM_RESERVATION_SYSTEM_ENABLE = RM_PREFIX
+ + "reservation-system.enable";
+ public static final boolean DEFAULT_RM_RESERVATION_SYSTEM_ENABLE = false;
+
+ /** The class to use as the Reservation System. */
+ public static final String RM_RESERVATION_SYSTEM_CLASS = RM_PREFIX
+ + "reservation-system.class";
+
+ /** The PlanFollower for the Reservation System. */
+ public static final String RM_RESERVATION_SYSTEM_PLAN_FOLLOWER = RM_PREFIX
+ + "reservation-system.plan.follower";
+
+ /** The step size of the Reservation System. */
+ public static final String RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP =
+ RM_PREFIX + "reservation-system.planfollower.time-step";
+ public static final long DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP =
+ 1000L;
+
/**
* Enable periodic monitor threads.
* @see #RM_SCHEDULER_MONITOR_POLICIES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 9e27de5..d697de9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -27,10 +27,17 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
@@ -43,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
@@ -474,4 +482,98 @@ public abstract class YarnClient extends AbstractService {
*/
public abstract void moveApplicationAcrossQueues(ApplicationId appId,
String queue) throws YarnException, IOException;
+
+ /**
+ * <p>
+ * The interface used by clients to submit a new reservation to the
+ * {@link ResourceManager}.
+ * </p>
+ *
+ * <p>
+ * The client packages all details of its request in a
+ * {@link ReservationRequest} object. This contains information about the
+ * amount of capacity, temporal constraints, and gang needs. Furthermore, the
+ * reservation might be composed of multiple stages, with ordering
+ * dependencies among them.
+ * </p>
+ *
+ * <p>
+ * In order to respond, a new admission control component in the
+ * {@link ResourceManager} performs an analysis of the resources that have
+ * been committed over the period of time the user is requesting, verify that
+ * the user requests can be fulfilled, and that it respect a sharing policy
+ * (e.g., {@link CapacityOverTimePolicy}). Once it has positively determined
+ * that the ReservationRequest is satisfiable the {@link ResourceManager}
+ * answers with a {@link ReservationResponse} that include a
+ * {@link ReservationId}. Upon failure to find a valid allocation the response
+ * is an exception with the message detailing the reason of failure.
+ * </p>
+ *
+ * <p>
+ * The semantics guarantees that the ReservationId returned, corresponds to a
+ * valid reservation existing in the time-range request by the user. The
+ * amount of capacity dedicated to such reservation can vary overtime,
+ * depending of the allocation that has been determined. But it is guaranteed
+ * to satisfy all the constraint expressed by the user in the
+ * {@link ReservationRequest}
+ * </p>
+ *
+ * @param request request to submit a new Reservation
+ * @return response contains the {@link ReservationId} on accepting the
+ * submission
+ * @throws YarnException if the reservation cannot be created successfully
+ * @throws IOException
+ *
+ */
+ @Public
+ @Unstable
+ public abstract ReservationSubmissionResponse submitReservation(
+ ReservationSubmissionRequest request) throws YarnException, IOException;
+
+ /**
+ * <p>
+ * The interface used by clients to update an existing Reservation. This is
+ * referred to as a re-negotiation process, in which a user that has
+ * previously submitted a Reservation.
+ * </p>
+ *
+ * <p>
+ * The allocation is attempted by virtually substituting all previous
+ * allocations related to this Reservation with new ones, that satisfy the new
+ * {@link ReservationRequest}. Upon success the previous allocation is
+ * atomically substituted by the new one, and on failure (i.e., if the system
+ * cannot find a valid allocation for the updated request), the previous
+ * allocation remains valid.
+ * </p>
+ *
+ * @param request to update an existing Reservation (the ReservationRequest
+ * should refer to an existing valid {@link ReservationId})
+ * @return response empty on successfully updating the existing reservation
+ * @throws YarnException if the request is invalid or reservation cannot be
+ * updated successfully
+ * @throws IOException
+ *
+ */
+ @Public
+ @Unstable
+ public abstract ReservationUpdateResponse updateReservation(
+ ReservationUpdateRequest request) throws YarnException, IOException;
+
+ /**
+ * <p>
+ * The interface used by clients to remove an existing Reservation.
+ * </p>
+ *
+ * @param request to remove an existing Reservation (the ReservationRequest
+ * should refer to an existing valid {@link ReservationId})
+ * @return response empty on successfully deleting the existing reservation
+ * @throws YarnException if the request is invalid or reservation cannot be
+ * deleted successfully
+ * @throws IOException
+ *
+ */
+ @Public
+ @Unstable
+ public abstract ReservationDeleteResponse deleteReservation(
+ ReservationDeleteRequest request) throws YarnException, IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index def6da5..02c5a74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -63,6 +63,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
@@ -646,4 +652,23 @@ public class YarnClientImpl extends YarnClient {
MoveApplicationAcrossQueuesRequest.newInstance(appId, queue);
rmClient.moveApplicationAcrossQueues(request);
}
+
+ @Override
+ public ReservationSubmissionResponse submitReservation(
+ ReservationSubmissionRequest request) throws YarnException, IOException {
+ return rmClient.submitReservation(request);
+ }
+
+ @Override
+ public ReservationUpdateResponse updateReservation(
+ ReservationUpdateRequest request) throws YarnException, IOException {
+ return rmClient.updateReservation(request);
+ }
+
+ @Override
+ public ReservationDeleteResponse deleteReservation(
+ ReservationDeleteRequest request) throws YarnException, IOException {
+ return rmClient.deleteReservation(request);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 3c1b1c1..d7bea7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -31,6 +31,7 @@ import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -63,6 +64,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
@@ -76,6 +83,11 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -89,8 +101,14 @@ import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.UTCClock;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
@@ -834,4 +852,101 @@ public class TestYarnClient {
client.stop();
}
}
+
+ @Test
+ public void testReservationAPIs() {
+ // initialize
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ ReservationSystemTestUtil.setupQueueConfiguration(conf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
+ MiniYARNCluster cluster =
+ new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
+ YarnClient client = null;
+ try {
+ cluster.init(conf);
+ cluster.start();
+ final Configuration yarnConf = cluster.getConfig();
+ client = YarnClient.createYarnClient();
+ client.init(yarnConf);
+ client.start();
+
+ // create a reservation
+ Clock clock = new UTCClock();
+ long arrival = clock.getTime();
+ long duration = 60000;
+ long deadline = (long) (arrival + 1.05 * duration);
+ ReservationSubmissionRequest sRequest =
+ createSimpleReservationRequest(4, arrival, deadline, duration);
+ ReservationSubmissionResponse sResponse = null;
+ try {
+ sResponse = client.submitReservation(sRequest);
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(sResponse);
+ ReservationId reservationID = sResponse.getReservationId();
+ Assert.assertNotNull(reservationID);
+ System.out.println("Submit reservation response: " + reservationID);
+
+ // Update the reservation
+ ReservationDefinition rDef = sRequest.getReservationDefinition();
+ ReservationRequest rr =
+ rDef.getReservationRequests().getReservationResources().get(0);
+ rr.setNumContainers(5);
+ arrival = clock.getTime();
+ duration = 30000;
+ deadline = (long) (arrival + 1.05 * duration);
+ rr.setDuration(duration);
+ rDef.setArrival(arrival);
+ rDef.setDeadline(deadline);
+ ReservationUpdateRequest uRequest =
+ ReservationUpdateRequest.newInstance(rDef, reservationID);
+ ReservationUpdateResponse uResponse = null;
+ try {
+ uResponse = client.updateReservation(uRequest);
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(sResponse);
+ System.out.println("Update reservation response: " + uResponse);
+
+ // Delete the reservation
+ ReservationDeleteRequest dRequest =
+ ReservationDeleteRequest.newInstance(reservationID);
+ ReservationDeleteResponse dResponse = null;
+ try {
+ dResponse = client.deleteReservation(dRequest);
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(sResponse);
+ System.out.println("Delete reservation response: " + dResponse);
+ } finally {
+ // clean-up
+ if (client != null) {
+ client.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+ private ReservationSubmissionRequest createSimpleReservationRequest(
+ int numContainers, long arrival, long deadline, long duration) {
+ // create a request with a single atomic ask
+ ReservationRequest r =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ numContainers, 1, duration);
+ ReservationRequests reqs =
+ ReservationRequests.newInstance(Collections.singletonList(r),
+ ReservationRequestInterpreter.R_ALL);
+ ReservationDefinition rDef =
+ ReservationDefinition.newInstance(arrival, deadline, reqs,
+ "testYarnClient#reservation");
+ ReservationSubmissionRequest request =
+ ReservationSubmissionRequest.newInstance(rDef,
+ ReservationSystemTestUtil.reservationQ);
+ return request;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index ff0a249..2b7797f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMapp
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
@@ -348,6 +349,11 @@ public class AdminService extends CompositeService implements
recordFactory.newRecordInstance(RefreshQueuesResponse.class);
try {
rmContext.getScheduler().reinitialize(getConfig(), this.rmContext);
+ // refresh the reservation system
+ ReservationSystem rSystem = rmContext.getReservationSystem();
+ if (rSystem != null) {
+ rSystem.reinitialize(getConfig(), rmContext);
+ }
RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
return response;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 71f873c..12811d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.security.AccessControlException;
+import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -79,6 +80,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesReq
import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@@ -93,6 +100,8 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
@@ -107,6 +116,10 @@ import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInputValidator;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -123,7 +136,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenS
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.UTCClock;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.Futures;
@@ -154,10 +169,23 @@ public class ClientRMService extends AbstractService implements
private final ApplicationACLsManager applicationsACLsManager;
private final QueueACLsManager queueACLsManager;
+ // For Reservation APIs
+ private Clock clock;
+ private ReservationSystem reservationSystem;
+ private ReservationInputValidator rValidator;
+
public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager,
QueueACLsManager queueACLsManager,
RMDelegationTokenSecretManager rmDTSecretManager) {
+ this(rmContext, scheduler, rmAppManager, applicationACLsManager,
+ queueACLsManager, rmDTSecretManager, new UTCClock());
+ }
+
+ public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
+ RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager,
+ QueueACLsManager queueACLsManager,
+ RMDelegationTokenSecretManager rmDTSecretManager, Clock clock) {
super(ClientRMService.class.getName());
this.scheduler = scheduler;
this.rmContext = rmContext;
@@ -165,6 +193,9 @@ public class ClientRMService extends AbstractService implements
this.applicationsACLsManager = applicationACLsManager;
this.queueACLsManager = queueACLsManager;
this.rmDTSecretManager = rmDTSecretManager;
+ this.reservationSystem = rmContext.getReservationSystem();
+ this.clock = clock;
+ this.rValidator = new ReservationInputValidator(clock);
}
@Override
@@ -1033,4 +1064,174 @@ public class ClientRMService extends AbstractService implements
public Server getServer() {
return this.server;
}
+
+ @Override
+ public ReservationSubmissionResponse submitReservation(
+ ReservationSubmissionRequest request) throws YarnException, IOException {
+ // Check if reservation system is enabled
+ checkReservationSytem(AuditConstants.SUBMIT_RESERVATION_REQUEST);
+ ReservationSubmissionResponse response =
+ recordFactory.newRecordInstance(ReservationSubmissionResponse.class);
+ // Create a new Reservation Id
+ ReservationId reservationId = reservationSystem.getNewReservationId();
+ // Validate the input
+ Plan plan =
+ rValidator.validateReservationSubmissionRequest(reservationSystem,
+ request, reservationId);
+ // Check ACLs
+ String queueName = request.getQueue();
+ String user =
+ checkReservationACLs(queueName,
+ AuditConstants.SUBMIT_RESERVATION_REQUEST);
+ try {
+ // Try to place the reservation using the agent
+ boolean result =
+ plan.getReservationAgent().createReservation(reservationId, user,
+ plan, request.getReservationDefinition());
+ if (result) {
+ // add the reservation id to valid ones maintained by reservation
+ // system
+ reservationSystem.setQueueForReservation(reservationId, queueName);
+ // create the reservation synchronously if required
+ refreshScheduler(queueName, request.getReservationDefinition(),
+ reservationId.toString());
+ // return the reservation id
+ response.setReservationId(reservationId);
+ }
+ } catch (PlanningException e) {
+ RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_RESERVATION_REQUEST,
+ e.getMessage(), "ClientRMService",
+ "Unable to create the reservation: " + reservationId);
+ throw RPCUtil.getRemoteException(e);
+ }
+ RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_RESERVATION_REQUEST,
+ "ClientRMService: " + reservationId);
+ return response;
+ }
+
+ @Override
+ public ReservationUpdateResponse updateReservation(
+ ReservationUpdateRequest request) throws YarnException, IOException {
+ // Check if reservation system is enabled
+ checkReservationSytem(AuditConstants.UPDATE_RESERVATION_REQUEST);
+ ReservationUpdateResponse response =
+ recordFactory.newRecordInstance(ReservationUpdateResponse.class);
+ // Validate the input
+ Plan plan =
+ rValidator.validateReservationUpdateRequest(reservationSystem, request);
+ ReservationId reservationId = request.getReservationId();
+ String queueName = reservationSystem.getQueueForReservation(reservationId);
+ // Check ACLs
+ String user =
+ checkReservationACLs(queueName,
+ AuditConstants.UPDATE_RESERVATION_REQUEST);
+ // Try to update the reservation using default agent
+ try {
+ boolean result =
+ plan.getReservationAgent().updateReservation(reservationId, user,
+ plan, request.getReservationDefinition());
+ if (!result) {
+ String errMsg = "Unable to update reservation: " + reservationId;
+ RMAuditLogger.logFailure(user,
+ AuditConstants.UPDATE_RESERVATION_REQUEST, errMsg,
+ "ClientRMService", errMsg);
+ throw RPCUtil.getRemoteException(errMsg);
+ }
+ } catch (PlanningException e) {
+ RMAuditLogger.logFailure(user, AuditConstants.UPDATE_RESERVATION_REQUEST,
+ e.getMessage(), "ClientRMService",
+ "Unable to update the reservation: " + reservationId);
+ throw RPCUtil.getRemoteException(e);
+ }
+ RMAuditLogger.logSuccess(user, AuditConstants.UPDATE_RESERVATION_REQUEST,
+ "ClientRMService: " + reservationId);
+ return response;
+ }
+
+ @Override
+ public ReservationDeleteResponse deleteReservation(
+ ReservationDeleteRequest request) throws YarnException, IOException {
+ // Check if reservation system is enabled
+ checkReservationSytem(AuditConstants.DELETE_RESERVATION_REQUEST);
+ ReservationDeleteResponse response =
+ recordFactory.newRecordInstance(ReservationDeleteResponse.class);
+ // Validate the input
+ Plan plan =
+ rValidator.validateReservationDeleteRequest(reservationSystem, request);
+ ReservationId reservationId = request.getReservationId();
+ String queueName = reservationSystem.getQueueForReservation(reservationId);
+ // Check ACLs
+ String user =
+ checkReservationACLs(queueName,
+ AuditConstants.DELETE_RESERVATION_REQUEST);
+ // Try to update the reservation using default agent
+ try {
+ boolean result =
+ plan.getReservationAgent().deleteReservation(reservationId, user,
+ plan);
+ if (!result) {
+ String errMsg = "Could not delete reservation: " + reservationId;
+ RMAuditLogger.logFailure(user,
+ AuditConstants.DELETE_RESERVATION_REQUEST, errMsg,
+ "ClientRMService", errMsg);
+ throw RPCUtil.getRemoteException(errMsg);
+ }
+ } catch (PlanningException e) {
+ RMAuditLogger.logFailure(user, AuditConstants.DELETE_RESERVATION_REQUEST,
+ e.getMessage(), "ClientRMService",
+ "Unable to delete the reservation: " + reservationId);
+ throw RPCUtil.getRemoteException(e);
+ }
+ RMAuditLogger.logSuccess(user, AuditConstants.DELETE_RESERVATION_REQUEST,
+ "ClientRMService: " + reservationId);
+ return response;
+ }
+
+ private void checkReservationSytem(String auditConstant) throws YarnException {
+ // Check if reservation is enabled
+ if (reservationSystem == null) {
+ throw RPCUtil.getRemoteException("Reservation is not enabled."
+ + " Please enable & try again");
+ }
+ }
+
+ private void refreshScheduler(String planName,
+ ReservationDefinition contract, String reservationId) {
+ if ((contract.getArrival() - clock.getTime()) < reservationSystem
+ .getPlanFollowerTimeStep()) {
+ LOG.debug(MessageFormat
+ .format(
+ "Reservation {0} is within threshold so attempting to create synchronously.",
+ reservationId));
+ reservationSystem.synchronizePlan(planName);
+ LOG.info(MessageFormat.format("Created reservation {0} synchronously.",
+ reservationId));
+ }
+ }
+
+ private String checkReservationACLs(String queueName, String auditConstant)
+ throws YarnException {
+ UserGroupInformation callerUGI;
+ try {
+ callerUGI = UserGroupInformation.getCurrentUser();
+ } catch (IOException ie) {
+ RMAuditLogger.logFailure("UNKNOWN", auditConstant, queueName,
+ "ClientRMService", "Error getting UGI");
+ throw RPCUtil.getRemoteException(ie);
+ }
+ // Check if user has access on the managed queue
+ if (!queueACLsManager.checkAccess(callerUGI, QueueACL.SUBMIT_APPLICATIONS,
+ queueName)) {
+ RMAuditLogger.logFailure(
+ callerUGI.getShortUserName(),
+ auditConstant,
+ "User doesn't have permissions to "
+ + QueueACL.SUBMIT_APPLICATIONS.toString(), "ClientRMService",
+ AuditConstants.UNAUTHORIZED_USER);
+ throw RPCUtil.getRemoteException(new AccessControlException("User "
+ + callerUGI.getShortUserName() + " cannot perform operation "
+ + QueueACL.SUBMIT_APPLICATIONS.name() + " on queue" + queueName));
+ }
+ return callerUGI.getShortUserName();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
index 9ae09a4..6dd67c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
@@ -56,6 +56,11 @@ public class RMAuditLogger {
// Some commonly used descriptions
public static final String UNAUTHORIZED_USER = "Unauthorized user";
+
+ // For Reservation system
+ public static final String SUBMIT_RESERVATION_REQUEST = "Submit Reservation Request";
+ public static final String UPDATE_RESERVATION_REQUEST = "Update Reservation Request";
+ public static final String DELETE_RESERVATION_REQUEST = "Delete Reservation Request";
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 60f88f6..436cba6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
@@ -110,4 +111,7 @@ public interface RMContext {
long getEpoch();
boolean isSchedulerReadyForAllocatingContainers();
+
+ ReservationSystem getReservationSystem();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 36eec04..78787ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWri
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
@@ -83,6 +84,7 @@ public class RMContextImpl implements RMContext {
private ClientRMService clientRMService;
private RMDelegationTokenSecretManager rmDelegationTokenSecretManager;
private ResourceScheduler scheduler;
+ private ReservationSystem reservationSystem;
private NodesListManager nodesListManager;
private ResourceTrackerService resourceTrackerService;
private ApplicationMasterService applicationMasterService;
@@ -209,6 +211,11 @@ public class RMContextImpl implements RMContext {
}
@Override
+ public ReservationSystem getReservationSystem() {
+ return this.reservationSystem;
+ }
+
+ @Override
public NodesListManager getNodesListManager() {
return this.nodesListManager;
}
@@ -303,6 +310,10 @@ public class RMContextImpl implements RMContext {
void setScheduler(ResourceScheduler scheduler) {
this.scheduler = scheduler;
}
+
+ void setReservationSystem(ReservationSystem reservationSystem) {
+ this.reservationSystem = reservationSystem;
+ }
void setDelegationTokenRenewer(
DelegationTokenRenewer delegationTokenRenewer) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 79af7a6..3e5f138 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -72,6 +72,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.AbstractReservationSystem;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -147,6 +149,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected RMSecretManagerService rmSecretManagerService;
protected ResourceScheduler scheduler;
+ protected ReservationSystem reservationSystem;
private ClientRMService clientRM;
protected ApplicationMasterService masterService;
protected NMLivelinessMonitor nmLivelinessMonitor;
@@ -281,6 +284,29 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
}
+ protected ReservationSystem createReservationSystem() {
+ String reservationClassName =
+ conf.get(YarnConfiguration.RM_RESERVATION_SYSTEM_CLASS,
+ AbstractReservationSystem.getDefaultReservationSystem(scheduler));
+ if (reservationClassName == null) {
+ return null;
+ }
+ LOG.info("Using ReservationSystem: " + reservationClassName);
+ try {
+ Class<?> reservationClazz = Class.forName(reservationClassName);
+ if (ReservationSystem.class.isAssignableFrom(reservationClazz)) {
+ return (ReservationSystem) ReflectionUtils.newInstance(
+ reservationClazz, this.conf);
+ } else {
+ throw new YarnRuntimeException("Class: " + reservationClassName
+ + " not instance of " + ReservationSystem.class.getCanonicalName());
+ }
+ } catch (ClassNotFoundException e) {
+ throw new YarnRuntimeException(
+ "Could not instantiate ReservationSystem: " + reservationClassName, e);
+ }
+ }
+
protected ApplicationMasterLauncher createAMLauncher() {
return new ApplicationMasterLauncher(this.rmContext);
}
@@ -456,6 +482,18 @@ public class ResourceManager extends CompositeService implements Recoverable {
DefaultMetricsSystem.initialize("ResourceManager");
JvmMetrics.initSingleton("ResourceManager", null);
+ // Initialize the Reservation system
+ if (conf.getBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE,
+ YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_ENABLE)) {
+ reservationSystem = createReservationSystem();
+ if (reservationSystem != null) {
+ reservationSystem.setRMContext(rmContext);
+ addIfService(reservationSystem);
+ rmContext.setReservationSystem(reservationSystem);
+ LOG.info("Initialized Reservation system");
+ }
+ }
+
// creating monitors that handle preemption
createPolicyMonitors();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
new file mode 100644
index 0000000..f0a9543
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -0,0 +1,323 @@
+/**
+ *
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.UTCClock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is the implementation of {@link ReservationSystem} based on the
+ * {@link ResourceScheduler}
+ */
+@LimitedPrivate("yarn")
+@Unstable
+public abstract class AbstractReservationSystem extends AbstractService
+ implements ReservationSystem {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AbstractReservationSystem.class);
+
+ // private static final String DEFAULT_CAPACITY_SCHEDULER_PLAN
+
+ private final ReentrantReadWriteLock readWriteLock =
+ new ReentrantReadWriteLock(true);
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+ private boolean initialized = false;
+
+ private final Clock clock = new UTCClock();
+
+ private AtomicLong resCounter = new AtomicLong();
+
+ private Map<String, Plan> plans = new HashMap<String, Plan>();
+
+ private Map<ReservationId, String> resQMap =
+ new HashMap<ReservationId, String>();
+
+ private RMContext rmContext;
+
+ private ResourceScheduler scheduler;
+
+ private ScheduledExecutorService scheduledExecutorService;
+
+ protected Configuration conf;
+
+ protected long planStepSize;
+
+ private PlanFollower planFollower;
+
+ /**
+ * Construct the service.
+ *
+ * @param name service name
+ */
+ public AbstractReservationSystem(String name) {
+ super(name);
+ }
+
+ @Override
+ public void setRMContext(RMContext rmContext) {
+ writeLock.lock();
+ try {
+ this.rmContext = rmContext;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void reinitialize(Configuration conf, RMContext rmContext)
+ throws YarnException {
+ writeLock.lock();
+ try {
+ if (!initialized) {
+ initialize(conf);
+ initialized = true;
+ } else {
+ initializeNewPlans(conf);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void initialize(Configuration conf) throws YarnException {
+ LOG.info("Initializing Reservation system");
+ this.conf = conf;
+ scheduler = rmContext.getScheduler();
+ // Get the plan step size
+ planStepSize =
+ conf.getTimeDuration(
+ YarnConfiguration.RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP,
+ YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP,
+ TimeUnit.MILLISECONDS);
+ if (planStepSize < 0) {
+ planStepSize =
+ YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP;
+ }
+ // Create a plan corresponding to every reservable queue
+ Set<String> planQueueNames = scheduler.getPlanQueues();
+ for (String planQueueName : planQueueNames) {
+ Plan plan = initializePlan(planQueueName);
+ plans.put(planQueueName, plan);
+ }
+ }
+
+ private void initializeNewPlans(Configuration conf) {
+ LOG.info("Refreshing Reservation system");
+ writeLock.lock();
+ try {
+ // Create a plan corresponding to every new reservable queue
+ Set<String> planQueueNames = scheduler.getPlanQueues();
+ for (String planQueueName : planQueueNames) {
+ if (!plans.containsKey(planQueueName)) {
+ Plan plan = initializePlan(planQueueName);
+ plans.put(planQueueName, plan);
+ } else {
+ LOG.warn("Plan based on reservation queue {0} already exists.",
+ planQueueName);
+ }
+ }
+ // Update the plan follower with the active plans
+ if (planFollower != null) {
+ planFollower.setPlans(plans.values());
+ }
+ } catch (YarnException e) {
+ LOG.warn("Exception while trying to refresh reservable queues", e);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private PlanFollower createPlanFollower() {
+ String planFollowerPolicyClassName =
+ conf.get(YarnConfiguration.RM_RESERVATION_SYSTEM_PLAN_FOLLOWER,
+ getDefaultPlanFollower());
+ if (planFollowerPolicyClassName == null) {
+ return null;
+ }
+ LOG.info("Using PlanFollowerPolicy: " + planFollowerPolicyClassName);
+ try {
+ Class<?> planFollowerPolicyClazz =
+ conf.getClassByName(planFollowerPolicyClassName);
+ if (PlanFollower.class.isAssignableFrom(planFollowerPolicyClazz)) {
+ return (PlanFollower) ReflectionUtils.newInstance(
+ planFollowerPolicyClazz, conf);
+ } else {
+ throw new YarnRuntimeException("Class: " + planFollowerPolicyClassName
+ + " not instance of " + PlanFollower.class.getCanonicalName());
+ }
+ } catch (ClassNotFoundException e) {
+ throw new YarnRuntimeException(
+ "Could not instantiate PlanFollowerPolicy: "
+ + planFollowerPolicyClassName, e);
+ }
+ }
+
+ private String getDefaultPlanFollower() {
+ // currently only capacity scheduler is supported
+ if (scheduler instanceof CapacityScheduler) {
+ return CapacitySchedulerPlanFollower.class.getName();
+ }
+ return null;
+ }
+
+ @Override
+ public Plan getPlan(String planName) {
+ readLock.lock();
+ try {
+ return plans.get(planName);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * @return the planStepSize
+ */
+ @Override
+ public long getPlanFollowerTimeStep() {
+ readLock.lock();
+ try {
+ return planStepSize;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public void synchronizePlan(String planName) {
+ writeLock.lock();
+ try {
+ Plan plan = plans.get(planName);
+ if (plan != null) {
+ planFollower.synchronizePlan(plan);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ Configuration configuration = new Configuration(conf);
+ reinitialize(configuration, rmContext);
+ // Create the plan follower with the active plans
+ planFollower = createPlanFollower();
+ if (planFollower != null) {
+ planFollower.init(clock, scheduler, plans.values());
+ }
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (planFollower != null) {
+ scheduledExecutorService = new ScheduledThreadPoolExecutor(1);
+ scheduledExecutorService.scheduleWithFixedDelay(planFollower, 0L,
+ planStepSize, TimeUnit.MILLISECONDS);
+ }
+ super.serviceStart();
+ }
+
+ @Override
+ public void serviceStop() {
+ // Stop the plan follower
+ if (scheduledExecutorService != null
+ && !scheduledExecutorService.isShutdown()) {
+ scheduledExecutorService.shutdown();
+ }
+ // Clear the plans
+ plans.clear();
+ }
+
+ @Override
+ public String getQueueForReservation(ReservationId reservationId) {
+ readLock.lock();
+ try {
+ return resQMap.get(reservationId);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public void setQueueForReservation(ReservationId reservationId,
+ String queueName) {
+ writeLock.lock();
+ try {
+ resQMap.put(reservationId, queueName);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public ReservationId getNewReservationId() {
+ writeLock.lock();
+ try {
+ ReservationId resId =
+ ReservationId.newInstance(ResourceManager.getClusterTimeStamp(),
+ resCounter.incrementAndGet());
+ LOG.info("Allocated new reservationId: " + resId);
+ return resId;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public Map<String, Plan> getAllPlans() {
+ return plans;
+ }
+
+ /**
+ * Get the default reservation system corresponding to the scheduler
+ *
+ * @param scheduler the scheduler for which the reservation system is required
+ */
+ public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
+ // currently only capacity scheduler is supported
+ if (scheduler instanceof CapacityScheduler) {
+ return CapacityReservationSystem.class.getName();
+ }
+ return null;
+ }
+
+ protected abstract Plan initializePlan(String planQueueName)
+ throws YarnException;
+
+ protected abstract Planner getReplanner(String planQueueName);
+
+ protected abstract ReservationAgent getAgent(String queueName);
+
+ protected abstract SharingPolicy getAdmissionPolicy(String queueName);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java
new file mode 100644
index 0000000..548fde1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java
@@ -0,0 +1,146 @@
+/**
+ *
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is the implementation of {@link ReservationSystem} based on the
+ * {@link CapacityScheduler}
+ */
+@LimitedPrivate("yarn")
+@Unstable
+public class CapacityReservationSystem extends AbstractReservationSystem {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(CapacityReservationSystem.class);
+
+ private CapacityScheduler capScheduler;
+
+ public CapacityReservationSystem() {
+ super(CapacityReservationSystem.class.getName());
+ }
+
+ @Override
+ public void reinitialize(Configuration conf, RMContext rmContext)
+ throws YarnException {
+ // Validate if the scheduler is capacity based
+ ResourceScheduler scheduler = rmContext.getScheduler();
+ if (!(scheduler instanceof CapacityScheduler)) {
+ throw new YarnRuntimeException("Class "
+ + scheduler.getClass().getCanonicalName() + " not instance of "
+ + CapacityScheduler.class.getCanonicalName());
+ }
+ capScheduler = (CapacityScheduler) scheduler;
+ this.conf = conf;
+ super.reinitialize(conf, rmContext);
+ }
+
+ @Override
+ protected Plan initializePlan(String planQueueName) throws YarnException {
+ SharingPolicy adPolicy = getAdmissionPolicy(planQueueName);
+ String planQueuePath = capScheduler.getQueue(planQueueName).getQueuePath();
+ adPolicy.init(planQueuePath, capScheduler.getConfiguration());
+ CSQueue planQueue = capScheduler.getQueue(planQueueName);
+ // Calculate the max plan capacity
+ Resource minAllocation = capScheduler.getMinimumResourceCapability();
+ ResourceCalculator rescCalc = capScheduler.getResourceCalculator();
+ Resource totCap =
+ rescCalc.multiplyAndNormalizeDown(capScheduler.getClusterResource(),
+ planQueue.getAbsoluteCapacity(), minAllocation);
+ Plan plan =
+ new InMemoryPlan(capScheduler.getRootQueueMetrics(), adPolicy,
+ getAgent(planQueuePath), totCap, planStepSize, rescCalc,
+ minAllocation, capScheduler.getMaximumResourceCapability(),
+ planQueueName, getReplanner(planQueuePath), capScheduler
+ .getConfiguration().getMoveOnExpiry(planQueuePath));
+ LOG.info("Intialized plan {0} based on reservable queue {1}",
+ plan.toString(), planQueueName);
+ return plan;
+ }
+
+ @Override
+ protected Planner getReplanner(String planQueueName) {
+ CapacitySchedulerConfiguration capSchedulerConfig =
+ capScheduler.getConfiguration();
+ String plannerClassName = capSchedulerConfig.getReplanner(planQueueName);
+ LOG.info("Using Replanner: " + plannerClassName + " for queue: "
+ + planQueueName);
+ try {
+ Class<?> plannerClazz =
+ capSchedulerConfig.getClassByName(plannerClassName);
+ if (Planner.class.isAssignableFrom(plannerClazz)) {
+ Planner planner =
+ (Planner) ReflectionUtils.newInstance(plannerClazz, conf);
+ planner.init(planQueueName, capSchedulerConfig);
+ return planner;
+ } else {
+ throw new YarnRuntimeException("Class: " + plannerClazz
+ + " not instance of " + Planner.class.getCanonicalName());
+ }
+ } catch (ClassNotFoundException e) {
+ throw new YarnRuntimeException("Could not instantiate Planner: "
+ + plannerClassName + " for queue: " + planQueueName, e);
+ }
+ }
+
+ @Override
+ protected ReservationAgent getAgent(String queueName) {
+ CapacitySchedulerConfiguration capSchedulerConfig =
+ capScheduler.getConfiguration();
+ String agentClassName = capSchedulerConfig.getReservationAgent(queueName);
+ LOG.info("Using Agent: " + agentClassName + " for queue: " + queueName);
+ try {
+ Class<?> agentClazz = capSchedulerConfig.getClassByName(agentClassName);
+ if (ReservationAgent.class.isAssignableFrom(agentClazz)) {
+ return (ReservationAgent) ReflectionUtils.newInstance(agentClazz, conf);
+ } else {
+ throw new YarnRuntimeException("Class: " + agentClassName
+ + " not instance of " + ReservationAgent.class.getCanonicalName());
+ }
+ } catch (ClassNotFoundException e) {
+ throw new YarnRuntimeException("Could not instantiate Agent: "
+ + agentClassName + " for queue: " + queueName, e);
+ }
+ }
+
+ @Override
+ protected SharingPolicy getAdmissionPolicy(String queueName) {
+ CapacitySchedulerConfiguration capSchedulerConfig =
+ capScheduler.getConfiguration();
+ String admissionPolicyClassName =
+ capSchedulerConfig.getReservationAdmissionPolicy(queueName);
+ LOG.info("Using AdmissionPolicy: " + admissionPolicyClassName
+ + " for queue: " + queueName);
+ try {
+ Class<?> admissionPolicyClazz =
+ capSchedulerConfig.getClassByName(admissionPolicyClassName);
+ if (SharingPolicy.class.isAssignableFrom(admissionPolicyClazz)) {
+ return (SharingPolicy) ReflectionUtils.newInstance(
+ admissionPolicyClazz, conf);
+ } else {
+ throw new YarnRuntimeException("Class: " + admissionPolicyClassName
+ + " not instance of " + SharingPolicy.class.getCanonicalName());
+ }
+ } catch (ClassNotFoundException e) {
+ throw new YarnRuntimeException("Could not instantiate AdmissionPolicy: "
+ + admissionPolicyClassName + " for queue: " + queueName, e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
new file mode 100644
index 0000000..678773d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
@@ -0,0 +1,244 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+public class ReservationInputValidator {
+
+ private final Clock clock;
+
+ /**
+ * Utility class to validate reservation requests.
+ */
+ public ReservationInputValidator(Clock clock) {
+ this.clock = clock;
+ }
+
+ private Plan validateReservation(ReservationSystem reservationSystem,
+ ReservationId reservationId, String auditConstant) throws YarnException {
+ String message = "";
+ // check if the reservation id is valid
+ if (reservationId == null) {
+ message =
+ "Missing reservation id."
+ + " Please try again by specifying a reservation id.";
+ RMAuditLogger.logFailure("UNKNOWN", auditConstant,
+ "validate reservation input", "ClientRMService", message);
+ throw RPCUtil.getRemoteException(message);
+ }
+ String queueName = reservationSystem.getQueueForReservation(reservationId);
+ if (queueName == null) {
+ message =
+ "The specified reservation with ID: " + reservationId
+ + " is unknown. Please try again with a valid reservation.";
+ RMAuditLogger.logFailure("UNKNOWN", auditConstant,
+ "validate reservation input", "ClientRMService", message);
+ throw RPCUtil.getRemoteException(message);
+ }
+ // check if the associated plan is valid
+ Plan plan = reservationSystem.getPlan(queueName);
+ if (plan == null) {
+ message =
+ "The specified reservation: " + reservationId
+ + " is not associated with any valid plan."
+ + " Please try again with a valid reservation.";
+ RMAuditLogger.logFailure("UNKNOWN", auditConstant,
+ "validate reservation input", "ClientRMService", message);
+ throw RPCUtil.getRemoteException(message);
+ }
+ return plan;
+ }
+
+ private void validateReservationDefinition(ReservationId reservationId,
+ ReservationDefinition contract, Plan plan, String auditConstant)
+ throws YarnException {
+ String message = "";
+ // check if deadline is in the past
+ if (contract == null) {
+ message =
+ "Missing reservation definition."
+ + " Please try again by specifying a reservation definition.";
+ RMAuditLogger.logFailure("UNKNOWN", auditConstant,
+ "validate reservation input definition", "ClientRMService", message);
+ throw RPCUtil.getRemoteException(message);
+ }
+ if (contract.getDeadline() <= clock.getTime()) {
+ message =
+ "The specified deadline: " + contract.getDeadline()
+ + " is the past. Please try again with deadline in the future.";
+ RMAuditLogger.logFailure("UNKNOWN", auditConstant,
+ "validate reservation input definition", "ClientRMService", message);
+ throw RPCUtil.getRemoteException(message);
+ }
+ // Check if at least one RR has been specified
+ ReservationRequests resReqs = contract.getReservationRequests();
+ if (resReqs == null) {
+ message =
+ "No resources have been specified to reserve."
+ + "Please try again by specifying the resources to reserve.";
+ RMAuditLogger.logFailure("UNKNOWN", auditConstant,
+ "validate reservation input definition", "ClientRMService", message);
+ throw RPCUtil.getRemoteException(message);
+ }
+ List<ReservationRequest> resReq = resReqs.getReservationResources();
+ if (resReq == null || resReq.isEmpty()) {
+ message =
+ "No resources have been specified to reserve."
+ + " Please try again by specifying the resources to reserve.";
+ RMAuditLogger.logFailure("UNKNOWN", auditConstant,
+ "validate reservation input definition", "ClientRMService", message);
+ throw RPCUtil.getRemoteException(message);
+ }
+ // compute minimum duration and max gang size
+ long minDuration = 0;
+ Resource maxGangSize = Resource.newInstance(0, 0);
+ ReservationRequestInterpreter type =
+ contract.getReservationRequests().getInterpreter();
+ for (ReservationRequest rr : resReq) {
+ if (type == ReservationRequestInterpreter.R_ALL
+ || type == ReservationRequestInterpreter.R_ANY) {
+ minDuration = Math.max(minDuration, rr.getDuration());
+ } else {
+ minDuration += rr.getDuration();
+ }
+ maxGangSize =
+ Resources.max(plan.getResourceCalculator(), plan.getTotalCapacity(),
+ maxGangSize,
+ Resources.multiply(rr.getCapability(), rr.getConcurrency()));
+ }
+ // verify the allocation is possible (skip for ANY)
+ if (contract.getDeadline() - contract.getArrival() < minDuration
+ && type != ReservationRequestInterpreter.R_ANY) {
+ message =
+ "The time difference ("
+ + (contract.getDeadline() - contract.getArrival())
+ + ") between arrival (" + contract.getArrival() + ") "
+ + "and deadline (" + contract.getDeadline() + ") must "
+ + " be greater or equal to the minimum resource duration ("
+ + minDuration + ")";
+ RMAuditLogger.logFailure("UNKNOWN", auditConstant,
+ "validate reservation input definition", "ClientRMService", message);
+ throw RPCUtil.getRemoteException(message);
+ }
+ // check that the largest gang does not exceed the inventory available
+ // capacity (skip for ANY)
+ if (Resources.greaterThan(plan.getResourceCalculator(),
+ plan.getTotalCapacity(), maxGangSize, plan.getTotalCapacity())
+ && type != ReservationRequestInterpreter.R_ANY) {
+ message =
+ "The size of the largest gang in the reservation refinition ("
+ + maxGangSize + ") exceed the capacity available ("
+ + plan.getTotalCapacity() + " )";
+ RMAuditLogger.logFailure("UNKNOWN", auditConstant,
+ "validate reservation input definition", "ClientRMService", message);
+ throw RPCUtil.getRemoteException(message);
+ }
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast) the input and returns the appropriate {@link Plan} associated with
+ * the specified {@link Queue} or throws an exception message illustrating the
+ * details of any validation check failures
+ *
+ * @param reservationSystem the {@link ReservationSystem} to validate against
+ * @param request the {@link ReservationSubmissionRequest} defining the
+ * resources required over time for the request
+ * @param reservationId the {@link ReservationId} associated with the current
+ * request
+ * @return the {@link Plan} to submit the request to
+ * @throws YarnException
+ */
+ public Plan validateReservationSubmissionRequest(
+ ReservationSystem reservationSystem,
+ ReservationSubmissionRequest request, ReservationId reservationId)
+ throws YarnException {
+ // Check if it is a managed queue
+ String queueName = request.getQueue();
+ if (queueName == null || queueName.isEmpty()) {
+ String errMsg =
+ "The queue to submit is not specified."
+ + " Please try again with a valid reservable queue.";
+ RMAuditLogger.logFailure("UNKNOWN",
+ AuditConstants.SUBMIT_RESERVATION_REQUEST,
+ "validate reservation input", "ClientRMService", errMsg);
+ throw RPCUtil.getRemoteException(errMsg);
+ }
+ Plan plan = reservationSystem.getPlan(queueName);
+ if (plan == null) {
+ String errMsg =
+ "The specified queue: " + queueName
+ + " is not managed by reservation system."
+ + " Please try again with a valid reservable queue.";
+ RMAuditLogger.logFailure("UNKNOWN",
+ AuditConstants.SUBMIT_RESERVATION_REQUEST,
+ "validate reservation input", "ClientRMService", errMsg);
+ throw RPCUtil.getRemoteException(errMsg);
+ }
+ validateReservationDefinition(reservationId,
+ request.getReservationDefinition(), plan,
+ AuditConstants.SUBMIT_RESERVATION_REQUEST);
+ return plan;
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast) the input and returns the appropriate {@link Plan} associated with
+ * the specified {@link Queue} or throws an exception message illustrating the
+ * details of any validation check failures
+ *
+ * @param reservationSystem the {@link ReservationSystem} to validate against
+ * @param request the {@link ReservationUpdateRequest} defining the resources
+ * required over time for the request
+ * @return the {@link Plan} to submit the request to
+ * @throws YarnException
+ */
+ public Plan validateReservationUpdateRequest(
+ ReservationSystem reservationSystem, ReservationUpdateRequest request)
+ throws YarnException {
+ ReservationId reservationId = request.getReservationId();
+ Plan plan =
+ validateReservation(reservationSystem, reservationId,
+ AuditConstants.UPDATE_RESERVATION_REQUEST);
+ validateReservationDefinition(reservationId,
+ request.getReservationDefinition(), plan,
+ AuditConstants.UPDATE_RESERVATION_REQUEST);
+ return plan;
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast) the input and returns the appropriate {@link Plan} associated with
+ * the specified {@link Queue} or throws an exception message illustrating the
+ * details of any validation check failures
+ *
+ * @param reservationSystem the {@link ReservationSystem} to validate against
+ * @param request the {@link ReservationDeleteRequest} defining the resources
+ * required over time for the request
+ * @return the {@link Plan} to submit the request to
+ * @throws YarnException
+ */
+ public Plan validateReservationDeleteRequest(
+ ReservationSystem reservationSystem, ReservationDeleteRequest request)
+ throws YarnException {
+ return validateReservation(reservationSystem, request.getReservationId(),
+ AuditConstants.DELETE_RESERVATION_REQUEST);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
new file mode 100644
index 0000000..cb76dcf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+
+/**
+ * This interface is the one implemented by any system that wants to support
+ * Reservations i.e. make {@link Resource} allocations in future. Implementors
+ * need to bootstrap all configured {@link Plan}s in the active
+ * {@link ResourceScheduler} along with their corresponding
+ * {@link ReservationAgent} and {@link SharingPolicy}. It is also responsible
+ * for managing the {@link PlanFollower} to ensure the {@link Plan}s are in sync
+ * with the {@link ResourceScheduler}.
+ */
+@LimitedPrivate("yarn")
+@Unstable
+public interface ReservationSystem {
+
+ /**
+ * Set RMContext for {@link ReservationSystem}. This method should be called
+ * immediately after instantiating a reservation system once.
+ *
+ * @param rmContext created by {@link ResourceManager}
+ */
+ void setRMContext(RMContext rmContext);
+
+ /**
+ * Re-initialize the {@link ReservationSystem}.
+ *
+ * @param conf configuration
+ * @param rmContext current context of the {@link ResourceManager}
+ * @throws YarnException
+ */
+ void reinitialize(Configuration conf, RMContext rmContext)
+ throws YarnException;
+
+ /**
+ * Get an existing {@link Plan} that has been initialized.
+ *
+ * @param planName the name of the {@link Plan}
+ * @return the {@link Plan} identified by name
+ *
+ */
+ Plan getPlan(String planName);
+
+ /**
+ * Return a map containing all the plans known to this ReservationSystem
+ * (useful for UI)
+ *
+ * @return a Map of Plan names and Plan objects
+ */
+ Map<String, Plan> getAllPlans();
+
+ /**
+ * Invokes {@link PlanFollower} to synchronize the specified {@link Plan} with
+ * the {@link ResourceScheduler}
+ *
+ * @param planName the name of the {@link Plan} to be synchronized
+ */
+ void synchronizePlan(String planName);
+
+ /**
+ * Return the time step (ms) at which the {@link PlanFollower} is invoked
+ *
+ * @return the time step (ms) at which the {@link PlanFollower} is invoked
+ */
+ long getPlanFollowerTimeStep();
+
+ /**
+ * Get a new unique {@link ReservationId}.
+ *
+ * @return a new unique {@link ReservationId}
+ *
+ */
+ ReservationId getNewReservationId();
+
+ /**
+ * Get the {@link Queue} that an existing {@link ReservationId} is associated
+ * with.
+ *
+ * @param reservationId the unique id of the reservation
+ * @return the name of the associated Queue
+ *
+ */
+ String getQueueForReservation(ReservationId reservationId);
+
+ /**
+ * Set the {@link Queue} that an existing {@link ReservationId} should be
+ * associated with.
+ *
+ * @param reservationId the unique id of the reservation
+ * @param queueName the name of Queue to associate the reservation with
+ *
+ */
+ void setQueueForReservation(ReservationId reservationId, String queueName);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index a1ae3ca..624aa18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -236,4 +237,6 @@ public interface RMApp extends EventHandler<RMAppEvent> {
* @return metrics
*/
RMAppMetrics getRMAppMetrics();
+
+ ReservationId getReservationId();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e53fdea3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 0b8f321..20d773c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -846,14 +847,16 @@ public class RMAppImpl implements RMApp, Recoverable {
// started or started but not yet saved.
if (app.attempts.isEmpty()) {
app.scheduler.handle(new AppAddedSchedulerEvent(app.applicationId,
- app.submissionContext.getQueue(), app.user));
+ app.submissionContext.getQueue(), app.user, app.submissionContext
+ .getReservationID()));
return RMAppState.SUBMITTED;
}
// Add application to scheduler synchronously to guarantee scheduler
// knows applications before AM or NM re-registers.
app.scheduler.handle(new AppAddedSchedulerEvent(app.applicationId,
- app.submissionContext.getQueue(), app.user, true));
+ app.submissionContext.getQueue(), app.user, true,
+ app.submissionContext.getReservationID()));
// recover attempts
app.recoverAppAttempts();
@@ -880,7 +883,8 @@ public class RMAppImpl implements RMApp, Recoverable {
@Override
public void transition(RMAppImpl app, RMAppEvent event) {
app.handler.handle(new AppAddedSchedulerEvent(app.applicationId,
- app.submissionContext.getQueue(), app.user));
+ app.submissionContext.getQueue(), app.user, app.submissionContext
+ .getReservationID()));
}
}
@@ -1293,4 +1297,9 @@ public class RMAppImpl implements RMApp, Recoverable {
public void setSystemClock(Clock clock) {
this.systemClock = clock;
}
+
+ @Override
+ public ReservationId getReservationId() {
+ return submissionContext.getReservationID();
+ }
}
[32/50] [abbrv] git commit: HDFS-7115. TestEncryptionZones assumes
Unix path separator for KMS key store path. Contributed by Xiaoyu Yao.
Posted by cu...@apache.org.
HDFS-7115. TestEncryptionZones assumes Unix path separator for KMS key store path. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26cba7f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26cba7f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26cba7f3
Branch: refs/heads/YARN-1051
Commit: 26cba7f35ff24262afa5d8f9ed22f3a7f01d9a71
Parents: 7039b98
Author: cnauroth <cn...@apache.org>
Authored: Mon Sep 22 09:07:07 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Mon Sep 22 09:07:07 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/TestEncryptionZones.java | 10 +++++-----
2 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26cba7f3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d95e228..c4e3e27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -917,6 +917,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7110. Skip tests related to short-circuit read on platforms that do not
currently implement short-circuit read. (cnauroth)
+ HDFS-7115. TestEncryptionZones assumes Unix path separator for KMS key store
+ path. (Xiaoyu Yao via cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26cba7f3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 52ca942..b80fd45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -116,8 +116,8 @@ public class TestEncryptionZones {
protected FileContextTestWrapper fcWrapper;
protected String getKeyProviderURI() {
- return JavaKeyStoreProvider.SCHEME_NAME + "://file" + testRootDir +
- "/test.jks";
+ return JavaKeyStoreProvider.SCHEME_NAME + "://file" +
+ new Path(testRootDir.toString(), "test.jks").toUri();
}
@Before
@@ -1043,7 +1043,7 @@ public class TestEncryptionZones {
dfsAdmin.createEncryptionZone(zone, TEST_KEY);
DFSTestUtil.createFile(fs, zoneFile, len, (short) 1, 0xFEED);
String contents = DFSTestUtil.readFile(fs, zoneFile);
- final Path snap1 = fs.createSnapshot(zoneParent);
+ final Path snap1 = fs.createSnapshot(zoneParent, "snap1");
final Path snap1Zone = new Path(snap1, zone.getName());
assertEquals("Got unexpected ez path", zone.toString(),
dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString());
@@ -1052,14 +1052,14 @@ public class TestEncryptionZones {
// snapshot
fsWrapper.delete(zone, true);
fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
- final Path snap2 = fs.createSnapshot(zoneParent);
+ final Path snap2 = fs.createSnapshot(zoneParent, "snap2");
final Path snap2Zone = new Path(snap2, zone.getName());
assertNull("Expected null ez path",
dfsAdmin.getEncryptionZoneForPath(snap2Zone));
// Create the encryption zone again
dfsAdmin.createEncryptionZone(zone, TEST_KEY2);
- final Path snap3 = fs.createSnapshot(zoneParent);
+ final Path snap3 = fs.createSnapshot(zoneParent, "snap3");
final Path snap3Zone = new Path(snap3, zone.getName());
// Check that snap3's EZ has the correct settings
EncryptionZone ezSnap3 = dfsAdmin.getEncryptionZoneForPath(snap3Zone);
[46/50] [abbrv] git commit: YARN-1711. Policy to enforce
instantaneous and over-time quotas on user reservation. Contributed by Carlo
Curino and Subru Krishnan.
Posted by cu...@apache.org.
YARN-1711. Policy to enforce instantaneous and over-time quotas on user reservation. Contributed by Carlo Curino and Subru Krishnan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5800b72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5800b72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5800b72
Branch: refs/heads/YARN-1051
Commit: a5800b72ef0186d1f18769b2eaa2fb20827ce1a1
Parents: d297491
Author: carlo curino <Carlo Curino>
Authored: Tue Sep 16 13:20:57 2014 -0700
Committer: carlo curino <Carlo Curino>
Committed: Mon Sep 22 16:32:11 2014 -0700
----------------------------------------------------------------------
YARN-1051-CHANGES.txt | 3 +
.../reservation/CapacityOverTimePolicy.java | 231 +++++++++++++++++++
.../reservation/NoOverCommitPolicy.java | 74 ++++++
.../reservation/SharingPolicy.java | 49 ++++
.../exceptions/ContractValidationException.java | 9 +-
.../exceptions/MismatchedUserException.java | 28 +++
.../exceptions/PlanningException.java | 9 +-
.../exceptions/PlanningQuotaException.java | 28 +++
.../exceptions/ResourceOverCommitException.java | 28 +++
.../CapacitySchedulerConfiguration.java | 154 +++++++++++++
.../reservation/TestCapacityOverTimePolicy.java | 222 ++++++++++++++++++
.../reservation/TestNoOverCommitPolicy.java | 144 ++++++++++++
12 files changed, 977 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/YARN-1051-CHANGES.txt
----------------------------------------------------------------------
diff --git a/YARN-1051-CHANGES.txt b/YARN-1051-CHANGES.txt
index deece7c..e9ec691 100644
--- a/YARN-1051-CHANGES.txt
+++ b/YARN-1051-CHANGES.txt
@@ -14,3 +14,6 @@ subru)
YARN-1710. Logic to find allocations within a Plan that satisfy
user ReservationRequest(s). (Carlo Curino and Subru Krishnan via
curino)
+
+YARN-1711. Policy to enforce instantaneous and over-time quotas
+on user reservations. (Carlo Curino and Subru Krishnan via curino)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java
new file mode 100644
index 0000000..38c0207
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java
@@ -0,0 +1,231 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.Date;
+
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.MismatchedUserException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningQuotaException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ResourceOverCommitException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * This policy enforces a time-extended notion of Capacity. In particular it
+ * guarantees that the allocation received in input when combined with all
+ * previous allocation for the user does not violate an instantaneous max limit
+ * on the resources received, and that for every window of time of length
+ * validWindow, the integral of the allocations for a user (sum of the currently
+ * submitted allocation and all prior allocations for the user) does not exceed
+ * validWindow * maxAvg.
+ *
+ * This allows flexibility, in the sense that an allocation can instantaneously
+ * use large portions of the available capacity, but prevents abuses by bounding
+ * the average use over time.
+ *
+ * By controlling maxInst, maxAvg, validWindow the administrator configuring
+ * this policy can obtain a behavior ranging from instantaneously enforced
+ * capacity (akin to existing queues), or fully flexible allocations (likely
+ * reserved to super-users, or trusted systems).
+ */
+@LimitedPrivate("yarn")
+@Unstable
+public class CapacityOverTimePolicy implements SharingPolicy {
+
+ private CapacitySchedulerConfiguration conf;
+ private long validWindow;
+ private float maxInst;
+ private float maxAvg;
+
+ // For now this is CapacityScheduler specific, but given a hierarchy in the
+ // configuration structure of the schedulers (e.g., SchedulerConfiguration)
+ // it should be easy to remove this limitation
+ @Override
+ public void init(String reservationQueuePath, Configuration conf) {
+ this.conf = (CapacitySchedulerConfiguration) conf;
+ validWindow = this.conf.getReservationWindow(reservationQueuePath);
+ maxInst = this.conf.getInstantaneousMaxCapacity(reservationQueuePath) / 100;
+ maxAvg = this.conf.getAverageCapacity(reservationQueuePath) / 100;
+ };
+
+ @Override
+ public void validate(Plan plan, ReservationAllocation reservation)
+ throws PlanningException {
+
+ // this is entire method invoked under a write-lock on the plan, no need
+ // to synchronize accesses to the plan further
+
+ // Try to verify whether there is already a reservation with this ID in
+ // the system (remove its contribution during validation to simulate a
+ // try-n-swap
+ // update).
+ ReservationAllocation oldReservation =
+ plan.getReservationById(reservation.getReservationId());
+
+ // sanity check that the update of a reservation is not changing username
+ if (oldReservation != null
+ && !oldReservation.getUser().equals(reservation.getUser())) {
+ throw new MismatchedUserException(
+ "Updating an existing reservation with mismatched user:"
+ + oldReservation.getUser() + " != " + reservation.getUser());
+ }
+
+ long startTime = reservation.getStartTime();
+ long endTime = reservation.getEndTime();
+ long step = plan.getStep();
+
+ Resource planTotalCapacity = plan.getTotalCapacity();
+
+ Resource maxAvgRes = Resources.multiply(planTotalCapacity, maxAvg);
+ Resource maxInsRes = Resources.multiply(planTotalCapacity, maxInst);
+
+ // define variable that will store integral of resources (need diff class to
+ // avoid overflow issues for long/large allocations)
+ IntegralResource runningTot = new IntegralResource(0L, 0L);
+ IntegralResource maxAllowed = new IntegralResource(maxAvgRes);
+ maxAllowed.multiplyBy(validWindow / step);
+
+ // check that the resources offered to the user during any window of length
+ // "validWindow" overlapping this allocation are within maxAllowed
+ // also enforce instantaneous and physical constraints during this pass
+ for (long t = startTime - validWindow; t < endTime + validWindow; t += step) {
+
+ Resource currExistingAllocTot = plan.getTotalCommittedResources(t);
+ Resource currExistingAllocForUser =
+ plan.getConsumptionForUser(reservation.getUser(), t);
+ Resource currNewAlloc = reservation.getResourcesAtTime(t);
+ Resource currOldAlloc = Resources.none();
+ if (oldReservation != null) {
+ currOldAlloc = oldReservation.getResourcesAtTime(t);
+ }
+
+ // throw exception if the cluster is overcommitted
+ // tot_allocated - old + new > capacity
+ Resource inst =
+ Resources.subtract(Resources.add(currExistingAllocTot, currNewAlloc),
+ currOldAlloc);
+ if (Resources.greaterThan(plan.getResourceCalculator(),
+ planTotalCapacity, inst, planTotalCapacity)) {
+ throw new ResourceOverCommitException(" Resources at time " + t
+ + " would be overcommitted (" + inst + " over "
+ + plan.getTotalCapacity() + ") by accepting reservation: "
+ + reservation.getReservationId());
+ }
+
+ // throw exception if instantaneous limits are violated
+ // tot_alloc_to_this_user - old + new > inst_limit
+ if (Resources.greaterThan(plan.getResourceCalculator(),
+ planTotalCapacity, Resources.subtract(
+ Resources.add(currExistingAllocForUser, currNewAlloc),
+ currOldAlloc), maxInsRes)) {
+ throw new PlanningQuotaException("Instantaneous quota capacity "
+ + maxInst + " would be passed at time " + t
+ + " by accepting reservation: " + reservation.getReservationId());
+ }
+
+ // throw exception if the running integral of utilization over validWindow
+ // is violated. We perform a delta check, adding/removing instants at the
+ // boundary of the window from runningTot.
+
+ // runningTot = previous_runningTot + currExistingAllocForUser +
+ // currNewAlloc - currOldAlloc - pastNewAlloc - pastOldAlloc;
+
+ // Where:
+ // 1) currNewAlloc, currExistingAllocForUser represent the contribution of
+ // the instant in time added in this pass.
+ // 2) pastNewAlloc, pastOldAlloc are the contributions relative to time
+ // instants that are being retired from the the window
+ // 3) currOldAlloc is the contribution (if any) of the previous version of
+ // this reservation (the one we are updating)
+
+ runningTot.add(currExistingAllocForUser);
+ runningTot.add(currNewAlloc);
+ runningTot.subtract(currOldAlloc);
+
+ // expire contributions from instant in time before (t - validWindow)
+ if (t > startTime) {
+ Resource pastOldAlloc =
+ plan.getConsumptionForUser(reservation.getUser(), t - validWindow);
+ Resource pastNewAlloc = reservation.getResourcesAtTime(t - validWindow);
+
+ // runningTot = runningTot - pastExistingAlloc - pastNewAlloc;
+ runningTot.subtract(pastOldAlloc);
+ runningTot.subtract(pastNewAlloc);
+ }
+
+ // check integral
+ // runningTot > maxAvg * validWindow
+ // NOTE: we need to use comparator of IntegralResource directly, as
+ // Resource and ResourceCalculator assume "int" amount of resources,
+ // which is not sufficient when comparing integrals (out-of-bound)
+ if (maxAllowed.compareTo(runningTot) < 0) {
+ throw new PlanningQuotaException(
+ "Integral (avg over time) quota capacity " + maxAvg
+ + " over a window of " + validWindow / 1000 + " seconds, "
+ + " would be passed at time " + t + "(" + new Date(t)
+ + ") by accepting reservation: "
+ + reservation.getReservationId());
+ }
+ }
+ }
+
+ @Override
+ public long getValidWindow() {
+ return validWindow;
+ }
+
+ /**
+ * This class provides support for Resource-like book-keeping, based on
+ * long(s), as using Resource to store the "integral" of the allocation over
+ * time leads to integer overflows for large allocations/clusters. (Evolving
+ * Resource to use long is too disruptive at this point.)
+ *
+ * The comparison/multiplication behaviors of IntegralResource are consistent
+ * with the DefaultResourceCalculator.
+ */
+ public class IntegralResource {
+ long memory;
+ long vcores;
+
+ public IntegralResource(Resource resource) {
+ this.memory = resource.getMemory();
+ this.vcores = resource.getVirtualCores();
+ }
+
+ public IntegralResource(long mem, long vcores) {
+ this.memory = mem;
+ this.vcores = vcores;
+ }
+
+ public void add(Resource r) {
+ memory += r.getMemory();
+ vcores += r.getVirtualCores();
+ }
+
+ public void subtract(Resource r) {
+ memory -= r.getMemory();
+ vcores -= r.getVirtualCores();
+ }
+
+ public void multiplyBy(long window) {
+ memory = memory * window;
+ vcores = vcores * window;
+ }
+
+ public long compareTo(IntegralResource other) {
+ long diff = memory - other.memory;
+ if (diff == 0) {
+ diff = vcores - other.vcores;
+ }
+ return diff;
+ }
+
+ @Override
+ public String toString() {
+ return "<memory:" + memory + ", vCores:" + vcores + ">";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
new file mode 100644
index 0000000..cbe2b78
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
@@ -0,0 +1,74 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.MismatchedUserException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ResourceOverCommitException;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * This policy enforce a simple physical cluster capacity constraints, by
+ * validating that the allocation proposed fits in the current plan. This
+ * validation is compatible with "updates" and in verifying the capacity
+ * constraints it conceptually remove the prior version of the reservation.
+ */
+@LimitedPrivate("yarn")
+@Unstable
+public class NoOverCommitPolicy implements SharingPolicy {
+
+ @Override
+ public void validate(Plan plan, ReservationAllocation reservation)
+ throws PlanningException {
+
+ ReservationAllocation oldReservation =
+ plan.getReservationById(reservation.getReservationId());
+
+ // check updates are using same name
+ if (oldReservation != null
+ && !oldReservation.getUser().equals(reservation.getUser())) {
+ throw new MismatchedUserException(
+ "Updating an existing reservation with mismatching user:"
+ + oldReservation.getUser() + " != " + reservation.getUser());
+ }
+
+ long startTime = reservation.getStartTime();
+ long endTime = reservation.getEndTime();
+ long step = plan.getStep();
+
+ // for every instant in time, check we are respecting cluster capacity
+ for (long t = startTime; t < endTime; t += step) {
+ Resource currExistingAllocTot = plan.getTotalCommittedResources(t);
+ Resource currNewAlloc = reservation.getResourcesAtTime(t);
+ Resource currOldAlloc = Resource.newInstance(0, 0);
+ if (oldReservation != null) {
+ oldReservation.getResourcesAtTime(t);
+ }
+ // check the cluster is never over committed
+ // currExistingAllocTot + currNewAlloc - currOldAlloc >
+ // capPlan.getTotalCapacity()
+ if (Resources.greaterThan(plan.getResourceCalculator(), plan
+ .getTotalCapacity(), Resources.subtract(
+ Resources.add(currExistingAllocTot, currNewAlloc), currOldAlloc),
+ plan.getTotalCapacity())) {
+ throw new ResourceOverCommitException("Resources at time " + t
+ + " would be overcommitted by " + "accepting reservation: "
+ + reservation.getReservationId());
+ }
+ }
+ }
+
+ @Override
+ public long getValidWindow() {
+ // this policy has no "memory" so the valid window is set to zero
+ return 0;
+ }
+
+ @Override
+ public void init(String inventoryQueuePath, Configuration conf) {
+ // nothing to do for this policy
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
new file mode 100644
index 0000000..d917764
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
@@ -0,0 +1,49 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+
+/**
+ * This is the interface for policy that validate new
+ * {@link ReservationAllocation}s for allocations being added to a {@link Plan}.
+ * Individual policies will be enforcing different invariants.
+ */
+@LimitedPrivate("yarn")
+@Unstable
+public interface SharingPolicy {
+
+ /**
+ * Initialize this policy
+ *
+ * @param inventoryQueuePath the name of the queue for this plan
+ * @param conf the system configuration
+ */
+ public void init(String inventoryQueuePath, Configuration conf);
+
+ /**
+ * This method runs the policy validation logic, and return true/false on
+ * whether the {@link ReservationAllocation} is acceptable according to this
+ * sharing policy.
+ *
+ * @param plan the {@link Plan} we validate against
+ * @param newAllocation the allocation proposed to be added to the
+ * {@link Plan}
+ * @throws PlanningException if the policy is respected if we add this
+ * {@link ReservationAllocation} to the {@link Plan}
+ */
+ public void validate(Plan plan, ReservationAllocation newAllocation)
+ throws PlanningException;
+
+ /**
+ * Returns the time range before and after the current reservation considered
+ * by this policy. In particular, this informs the archival process for the
+ * {@link Plan}, i.e., reservations regarding times before (now - validWindow)
+ * can be deleted.
+ *
+ * @return validWindow the window of validity considered by the policy.
+ */
+ public long getValidWindow();
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ContractValidationException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ContractValidationException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ContractValidationException.java
index 7ee5a76..cd82a9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ContractValidationException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ContractValidationException.java
@@ -1,5 +1,13 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * This exception is thrown if the request made is not syntactically valid.
+ */
+@Public
+@Unstable
public class ContractValidationException extends PlanningException {
private static final long serialVersionUID = 1L;
@@ -8,5 +16,4 @@ public class ContractValidationException extends PlanningException {
super(message);
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/MismatchedUserException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/MismatchedUserException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/MismatchedUserException.java
new file mode 100644
index 0000000..0a443f3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/MismatchedUserException.java
@@ -0,0 +1,28 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * Exception thrown when an update to an existing reservation is performed
+ * by a user that is not the reservation owner.
+ */
+@Public
+@Unstable
+public class MismatchedUserException extends PlanningException {
+
+ private static final long serialVersionUID = 8313222590561668413L;
+
+ public MismatchedUserException(String message) {
+ super(message);
+ }
+
+ public MismatchedUserException(Throwable cause) {
+ super(cause);
+ }
+
+ public MismatchedUserException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningException.java
index aa9e9fb..0699856 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningException.java
@@ -2,10 +2,17 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
/**
* Exception thrown by the admission control subsystem when there is a problem
- * in trying to find an allocation for a user {@link ReservationSubmissionRequest}.
+ * in trying to find an allocation for a user
+ * {@link ReservationSubmissionRequest}.
*/
+
+@Public
+@Unstable
public class PlanningException extends Exception {
private static final long serialVersionUID = -684069387367879218L;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningQuotaException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningQuotaException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningQuotaException.java
new file mode 100644
index 0000000..aad4ee8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/PlanningQuotaException.java
@@ -0,0 +1,28 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * This exception is thrown if the user quota is exceed while accepting or
+ * updating a reservation.
+ */
+@Public
+@Unstable
+public class PlanningQuotaException extends PlanningException {
+
+ private static final long serialVersionUID = 8206629288380246166L;
+
+ public PlanningQuotaException(String message) {
+ super(message);
+ }
+
+ public PlanningQuotaException(Throwable cause) {
+ super(cause);
+ }
+
+ public PlanningQuotaException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ResourceOverCommitException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ResourceOverCommitException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ResourceOverCommitException.java
new file mode 100644
index 0000000..a4c2b07
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/ResourceOverCommitException.java
@@ -0,0 +1,28 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * This exception indicate that the reservation that has been attempted, would
+ * exceed the physical resources available in the {@link Plan} at the moment.
+ */
+@Public
+@Unstable
+public class ResourceOverCommitException extends PlanningException {
+
+ private static final long serialVersionUID = 7070699407526521032L;
+
+ public ResourceOverCommitException(String message) {
+ super(message);
+ }
+
+ public ResourceOverCommitException(Throwable cause) {
+ super(cause);
+ }
+
+ public ResourceOverCommitException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index af6bdc3..2c915c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -183,6 +183,63 @@ public class CapacitySchedulerConfiguration extends Configuration {
}
}
+ @Private
+ public static final String AVERAGE_CAPACITY = "average-capacity";
+
+ @Private
+ public static final String IS_RESERVABLE = "reservable";
+
+ @Private
+ public static final String RESERVATION_WINDOW = "reservation-window";
+
+ @Private
+ public static final String INSTANTANEOUS_MAX_CAPACITY =
+ "instantaneous-max-capacity";
+
+ @Private
+ public static final long DEFAULT_RESERVATION_WINDOW = 0L;
+
+ @Private
+ public static final String RESERVATION_ADMISSION_POLICY =
+ "reservation-policy";
+
+ @Private
+ public static final String RESERVATION_AGENT_NAME = "reservation-agent";
+
+ @Private
+ public static final String RESERVATION_SHOW_RESERVATION_AS_QUEUE =
+ "show-reservations-as-queues";
+
+ @Private
+ public static final String DEFAULT_RESERVATION_ADMISSION_POLICY =
+ "org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityOverTimePolicy";
+
+ @Private
+ public static final String DEFAULT_RESERVATION_AGENT_NAME =
+ "org.apache.hadoop.yarn.server.resourcemanager.reservation.GreedyReservationAgent";
+
+ @Private
+ public static final String RESERVATION_PLANNER_NAME = "reservation-planner";
+
+ @Private
+ public static final String DEFAULT_RESERVATION_PLANNER_NAME =
+ "org.apache.hadoop.yarn.server.resourcemanager.reservation.SimpleCapacityReplanner";
+
+ @Private
+ public static final String RESERVATION_MOVE_ON_EXPIRY =
+ "reservation-move-on-expiry";
+
+ @Private
+ public static final boolean DEFAULT_RESERVATION_MOVE_ON_EXPIRY = true;
+
+ @Private
+ public static final String RESERVATION_ENFORCEMENT_WINDOW =
+ "reservation-enforcement-window";
+
+ // default to 1h lookahead enforcement
+ @Private
+ public static final long DEFAULT_RESERVATION_ENFORCEMENT_WINDOW = 3600000;
+
public CapacitySchedulerConfiguration() {
this(new Configuration());
}
@@ -493,4 +550,101 @@ public class CapacitySchedulerConfiguration extends Configuration {
return mappings;
}
+
+ public boolean isReservable(String queue) {
+ boolean isReservable =
+ getBoolean(getQueuePrefix(queue) + IS_RESERVABLE, false);
+ return isReservable;
+ }
+
+ public void setReservable(String queue, boolean isReservable) {
+ setBoolean(getQueuePrefix(queue) + IS_RESERVABLE, isReservable);
+ LOG.debug("here setReservableQueue: queuePrefix=" + getQueuePrefix(queue)
+ + ", isReservableQueue=" + isReservable(queue));
+ }
+
+ public long getReservationWindow(String queue) {
+ long reservationWindow =
+ getLong(getQueuePrefix(queue) + RESERVATION_WINDOW,
+ DEFAULT_RESERVATION_WINDOW);
+ return reservationWindow;
+ }
+
+ public float getAverageCapacity(String queue) {
+ float avgCapacity =
+ getFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY,
+ MAXIMUM_CAPACITY_VALUE);
+ return avgCapacity;
+ }
+
+ public float getInstantaneousMaxCapacity(String queue) {
+ float instMaxCapacity =
+ getFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY,
+ MAXIMUM_CAPACITY_VALUE);
+ return instMaxCapacity;
+ }
+
+ public void setInstantaneousMaxCapacity(String queue, float instMaxCapacity) {
+ setFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY,
+ instMaxCapacity);
+ }
+
+ public void setReservationWindow(String queue, long reservationWindow) {
+ setLong(getQueuePrefix(queue) + RESERVATION_WINDOW, reservationWindow);
+ }
+
+ public void setAverageCapacity(String queue, float avgCapacity) {
+ setFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY, avgCapacity);
+ }
+
+ public String getReservationAdmissionPolicy(String queue) {
+ String reservationPolicy =
+ get(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY,
+ DEFAULT_RESERVATION_ADMISSION_POLICY);
+ return reservationPolicy;
+ }
+
+ public void setReservationAdmissionPolicy(String queue,
+ String reservationPolicy) {
+ set(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY, reservationPolicy);
+ }
+
+ public String getReservationAgent(String queue) {
+ String reservationAgent =
+ get(getQueuePrefix(queue) + RESERVATION_AGENT_NAME,
+ DEFAULT_RESERVATION_AGENT_NAME);
+ return reservationAgent;
+ }
+
+ public void setReservationAgent(String queue, String reservationPolicy) {
+ set(getQueuePrefix(queue) + RESERVATION_AGENT_NAME, reservationPolicy);
+ }
+
+ public boolean getShowReservationAsQueues(String queuePath) {
+ boolean showReservationAsQueues =
+ getBoolean(getQueuePrefix(queuePath)
+ + RESERVATION_SHOW_RESERVATION_AS_QUEUE, false);
+ return showReservationAsQueues;
+ }
+
+ public String getReplanner(String queue) {
+ String replanner =
+ get(getQueuePrefix(queue) + RESERVATION_PLANNER_NAME,
+ DEFAULT_RESERVATION_PLANNER_NAME);
+ return replanner;
+ }
+
+ public boolean getMoveOnExpiry(String queue) {
+ boolean killOnExpiry =
+ getBoolean(getQueuePrefix(queue) + RESERVATION_MOVE_ON_EXPIRY,
+ DEFAULT_RESERVATION_MOVE_ON_EXPIRY);
+ return killOnExpiry;
+ }
+
+ public long getEnforcementWindow(String queue) {
+ long enforcementWindow =
+ getLong(getQueuePrefix(queue) + RESERVATION_ENFORCEMENT_WINDOW,
+ DEFAULT_RESERVATION_ENFORCEMENT_WINDOW);
+ return enforcementWindow;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java
new file mode 100644
index 0000000..83d6d3f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java
@@ -0,0 +1,222 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningQuotaException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ResourceOverCommitException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCapacityOverTimePolicy {
+
+ long timeWindow;
+ long step;
+ float avgConstraint;
+ float instConstraint;
+ long initTime;
+
+ InMemoryPlan plan;
+ ReservationAgent mAgent;
+ Resource minAlloc;
+ ResourceCalculator res;
+ Resource maxAlloc;
+
+ int totCont = 1000000;
+
+ @Before
+ public void setup() throws Exception {
+
+ // 24h window
+ timeWindow = 86400000L;
+ // 1 sec step
+ step = 1000L;
+
+ // 25% avg cap on capacity
+ avgConstraint = 25;
+
+ // 70% instantaneous cap on capacity
+ instConstraint = 70;
+
+ initTime = System.currentTimeMillis();
+ minAlloc = Resource.newInstance(1024, 1);
+ res = new DefaultResourceCalculator();
+ maxAlloc = Resource.newInstance(1024 * 8, 8);
+
+ mAgent = mock(ReservationAgent.class);
+ ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+ CapacityScheduler scheduler = testUtil.mockCapacityScheduler(totCont);
+ String reservationQ = testUtil.getFullReservationQueueName();
+ CapacitySchedulerConfiguration capConf = scheduler.getConfiguration();
+ capConf.setReservationWindow(reservationQ, timeWindow);
+ capConf.setInstantaneousMaxCapacity(reservationQ, instConstraint);
+ capConf.setAverageCapacity(reservationQ, avgConstraint);
+ CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
+ policy.init(reservationQ, capConf);
+
+ plan =
+ new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, mAgent,
+ scheduler.getClusterResource(), step, res, minAlloc, maxAlloc,
+ "dedicated", null, true);
+ }
+
+ public int[] generateData(int length, int val) {
+ int[] data = new int[length];
+ for (int i = 0; i < length; i++) {
+ data[i] = val;
+ }
+ return data;
+ }
+
+ @Test
+ public void testSimplePass() throws IOException, PlanningException {
+ // generate allocation that simply fit within all constraints
+ int[] f = generateData(3600, (int) Math.ceil(0.2 * totCont));
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ }
+
+ @Test
+ public void testSimplePass2() throws IOException, PlanningException {
+ // generate allocation from single tenant that exceed avg momentarily but
+ // fit within
+ // max instantanesou
+ int[] f = generateData(3600, (int) Math.ceil(0.69 * totCont));
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ }
+
+ @Test
+ public void testMultiTenantPass() throws IOException, PlanningException {
+ // generate allocation from multiple tenants that barely fit in tot capacity
+ int[] f = generateData(3600, (int) Math.ceil(0.25 * totCont));
+ for (int i = 0; i < 4; i++) {
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u" + i,
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ }
+ }
+
+ @Test(expected = ResourceOverCommitException.class)
+ public void testMultiTenantFail() throws IOException, PlanningException {
+ // generate allocation from multiple tenants that exceed tot capacity
+ int[] f = generateData(3600, (int) Math.ceil(0.25 * totCont));
+ for (int i = 0; i < 5; i++) {
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u" + i,
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ }
+ }
+
+ @Test(expected = PlanningQuotaException.class)
+ public void testInstFail() throws IOException, PlanningException {
+ // generate allocation that exceed the instantaneous cap single-show
+ int[] f = generateData(3600, (int) Math.ceil(0.71 * totCont));
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ Assert.fail("should not have accepted this");
+ }
+
+ @Test
+ public void testInstFailBySum() throws IOException, PlanningException {
+ // generate allocation that exceed the instantaneous cap by sum
+ int[] f = generateData(3600, (int) Math.ceil(0.3 * totCont));
+
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ try {
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ Assert.fail();
+ } catch (PlanningQuotaException p) {
+ // expected
+ }
+ }
+
+ @Test(expected = PlanningQuotaException.class)
+ public void testFailAvg() throws IOException, PlanningException {
+ // generate an allocation which violates the 25% average single-shot
+ Map<ReservationInterval, ReservationRequest> req =
+ new TreeMap<ReservationInterval, ReservationRequest>();
+ long win = timeWindow / 2 + 100;
+ int cont = (int) Math.ceil(0.5 * totCont);
+ req.put(new ReservationInterval(initTime, initTime + win),
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1), cont));
+
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + win, req, res, minAlloc)));
+ }
+
+ @Test
+ public void testFailAvgBySum() throws IOException, PlanningException {
+ // generate an allocation which violates the 25% average by sum
+ Map<ReservationInterval, ReservationRequest> req =
+ new TreeMap<ReservationInterval, ReservationRequest>();
+ long win = 86400000 / 4 + 1;
+ int cont = (int) Math.ceil(0.5 * totCont);
+ req.put(new ReservationInterval(initTime, initTime + win),
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1), cont));
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + win, req, res, minAlloc)));
+
+ try {
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + win, req, res, minAlloc)));
+
+ Assert.fail("should not have accepted this");
+ } catch (PlanningQuotaException e) {
+ // expected
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5800b72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java
new file mode 100644
index 0000000..2ceead3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java
@@ -0,0 +1,144 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.MismatchedUserException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ResourceOverCommitException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestNoOverCommitPolicy {
+
+ long step;
+ long initTime;
+
+ InMemoryPlan plan;
+ ReservationAgent mAgent;
+ Resource minAlloc;
+ ResourceCalculator res;
+ Resource maxAlloc;
+
+ int totCont = 1000000;
+
+ @Before
+ public void setup() throws Exception {
+
+ // 1 sec step
+ step = 1000L;
+
+ initTime = System.currentTimeMillis();
+ minAlloc = Resource.newInstance(1024, 1);
+ res = new DefaultResourceCalculator();
+ maxAlloc = Resource.newInstance(1024 * 8, 8);
+
+ mAgent = mock(ReservationAgent.class);
+ ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+ CapacityScheduler scheduler = testUtil.mockCapacityScheduler(totCont);
+ String reservationQ = testUtil.getFullReservationQueueName();
+ CapacitySchedulerConfiguration capConf = scheduler.getConfiguration();
+ NoOverCommitPolicy policy = new NoOverCommitPolicy();
+ policy.init(reservationQ, capConf);
+
+ plan =
+ new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, mAgent,
+ scheduler.getClusterResource(), step, res, minAlloc, maxAlloc,
+ "dedicated", null, true);
+ }
+
+ public int[] generateData(int length, int val) {
+ int[] data = new int[length];
+ for (int i = 0; i < length; i++) {
+ data[i] = val;
+ }
+ return data;
+ }
+
+ @Test
+ public void testSingleUserEasyFitPass() throws IOException, PlanningException {
+ // generate allocation that easily fit within resource constraints
+ int[] f = generateData(3600, (int) Math.ceil(0.2 * totCont));
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ }
+
+ @Test
+ public void testSingleUserBarelyFitPass() throws IOException,
+ PlanningException {
+ // generate allocation from single tenant that barely fit
+ int[] f = generateData(3600, totCont);
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ }
+
+ @Test(expected = ResourceOverCommitException.class)
+ public void testSingleFail() throws IOException, PlanningException {
+ // generate allocation from single tenant that exceed capacity
+ int[] f = generateData(3600, (int) (1.1 * totCont));
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u1",
+ "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil
+ .generateAllocation(initTime, step, f), res, minAlloc));
+ }
+
+ @Test(expected = MismatchedUserException.class)
+ public void testUserMismatch() throws IOException, PlanningException {
+ // generate allocation from single tenant that exceed capacity
+ int[] f = generateData(3600, (int) (0.5 * totCont));
+
+ ReservationId rid = ReservationSystemTestUtil.getNewReservationId();
+ plan.addReservation(new InMemoryReservationAllocation(rid, null, "u1",
+ "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil
+ .generateAllocation(initTime, step, f), res, minAlloc));
+
+ // trying to update a reservation with a mismatching user
+ plan.updateReservation(new InMemoryReservationAllocation(rid, null, "u2",
+ "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil
+ .generateAllocation(initTime, step, f), res, minAlloc));
+ }
+
+ @Test
+ public void testMultiTenantPass() throws IOException, PlanningException {
+ // generate allocation from multiple tenants that barely fit in tot capacity
+ int[] f = generateData(3600, (int) Math.ceil(0.25 * totCont));
+ for (int i = 0; i < 4; i++) {
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u" + i,
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ }
+ }
+
+ @Test(expected = ResourceOverCommitException.class)
+ public void testMultiTenantFail() throws IOException, PlanningException {
+ // generate allocation from multiple tenants that exceed tot capacity
+ int[] f = generateData(3600, (int) Math.ceil(0.25 * totCont));
+ for (int i = 0; i < 5; i++) {
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(
+ ReservationSystemTestUtil.getNewReservationId(), null, "u" + i,
+ "dedicated", initTime, initTime + f.length,
+ ReservationSystemTestUtil.generateAllocation(initTime, step, f),
+ res, minAlloc)));
+ }
+ }
+}
[15/50] [abbrv] git commit: MAPREDUCE-6091. YARNRunner.getJobStatus()
fails with ApplicationNotFoundException if the job rolled off the RM view.
Contributed by Sangjin Lee
Posted by cu...@apache.org.
MAPREDUCE-6091. YARNRunner.getJobStatus() fails with ApplicationNotFoundException if the job rolled off the RM view. Contributed by Sangjin Lee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/951847ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/951847ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/951847ba
Branch: refs/heads/YARN-1051
Commit: 951847ba94442b8d928435decfbea9ea20882912
Parents: 9f03a7c
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Sep 19 20:15:54 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Sep 19 20:15:54 2014 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 4 ++++
.../org/apache/hadoop/mapred/ClientServiceDelegate.java | 3 +++
.../org/apache/hadoop/mapred/TestClientServiceDelegate.java | 8 +++++---
.../org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java | 9 ++-------
4 files changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951847ba/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 0c6bbf7..eb4c251 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -356,6 +356,10 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6086. mapreduce.job.credentials.binary should allow all URIs.
(Zhihai Xu via kasha)
+ MAPREDUCE-6091. YARNRunner.getJobStatus() fails with
+ ApplicationNotFoundException if the job rolled off the RM view (Sangjin
+ Lee via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951847ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
index c3eee2c..686fa0c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -150,6 +151,8 @@ public class ClientServiceDelegate {
ApplicationReport application = null;
try {
application = rm.getApplicationReport(appId);
+ } catch (ApplicationNotFoundException e) {
+ application = null;
} catch (YarnException e2) {
throw new IOException(e2);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951847ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
index 5639e5d..7d6b2f3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
@@ -31,8 +31,6 @@ import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobStatus;
@@ -56,8 +54,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.Records;
+import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -488,7 +488,9 @@ public class TestClientServiceDelegate {
private ResourceMgrDelegate getRMDelegate() throws IOException {
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
try {
- when(rm.getApplicationReport(jobId.getAppId())).thenReturn(null);
+ ApplicationId appId = jobId.getAppId();
+ when(rm.getApplicationReport(appId)).
+ thenThrow(new ApplicationNotFoundException(appId + " not found"));
} catch (YarnException e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951847ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java
index d4f5f84..b6947f3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java
@@ -90,13 +90,8 @@ public class TestNonExistentJob extends TestCase {
}
public void testGetInvalidJob() throws Exception {
- try {
- RunningJob runJob = new JobClient(getJobConf()).getJob(JobID.forName("job_0_0"));
- fail("Exception is expected to thrown ahead!");
- } catch (Exception e) {
- assertTrue(e instanceof IOException);
- assertTrue(e.getMessage().contains("ApplicationNotFoundException"));
- }
+ RunningJob runJob = new JobClient(getJobConf()).getJob(JobID.forName("job_0_0"));
+ assertNull(runJob);
}
}
[47/50] [abbrv] git commit: YARN-1712. Plan follower that
synchronizes the current state of reservation subsystem with the scheduler.
Contributed by Subru Krishnan and Carlo Curino.
Posted by cu...@apache.org.
YARN-1712. Plan follower that synchronizes the current state of reservation subsystem with the scheduler. Contributed by Subru Krishnan and Carlo Curino.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a795348
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a795348
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a795348
Branch: refs/heads/YARN-1051
Commit: 4a7953484ac38ff8c86b8909f7b77842e103718a
Parents: a5800b7
Author: subru <su...@outlook.com>
Authored: Tue Sep 16 16:45:45 2014 -0700
Committer: carlo curino <Carlo Curino>
Committed: Mon Sep 22 16:32:12 2014 -0700
----------------------------------------------------------------------
YARN-1051-CHANGES.txt | 7 +-
.../CapacitySchedulerPlanFollower.java | 367 +++++++++++++++++++
.../reservation/PlanFollower.java | 67 ++++
.../TestCapacitySchedulerPlanFollower.java | 319 ++++++++++++++++
4 files changed, 758 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a795348/YARN-1051-CHANGES.txt
----------------------------------------------------------------------
diff --git a/YARN-1051-CHANGES.txt b/YARN-1051-CHANGES.txt
index e9ec691..56b3c12 100644
--- a/YARN-1051-CHANGES.txt
+++ b/YARN-1051-CHANGES.txt
@@ -5,10 +5,10 @@ YARN-2475. Logic for responding to capacity drops for the
ReservationSystem. (Carlo Curino and Subru Krishnan via curino)
YARN-1708. Public YARN APIs for creating/updating/deleting
-reservations. (Carlo Curino and Subru Krishnan via subru)
+reservations. (Subru Krishnan and Carlo Curino via subru)
YARN-1709. In-memory data structures used to track resources over
-time to enable reservations. (Carlo Curino and Subru Krishnan via
+time to enable reservations. (Subru Krishnan and Carlo Curino via
subru)
YARN-1710. Logic to find allocations within a Plan that satisfy
@@ -17,3 +17,6 @@ curino)
YARN-1711. Policy to enforce instantaneous and over-time quotas
on user reservations. (Carlo Curino and Subru Krishnan via curino)
+
+YARN-1712. Plan follower that synchronizes the current state of reservation
+subsystem with the scheduler. (Subru Krishnan and Carlo Curino via subru)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a795348/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
new file mode 100644
index 0000000..cfa172c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
@@ -0,0 +1,367 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ReservationQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class implements a {@link PlanFollower}. This is invoked on a timer, and
+ * it is in charge to publish the state of the {@link Plan}s to the underlying
+ * {@link CapacityScheduler}. This implementation does so, by
+ * adding/removing/resizing leaf queues in the scheduler, thus affecting the
+ * dynamic behavior of the scheduler in a way that is consistent with the
+ * content of the plan. It also updates the plan's view on how much resources
+ * are available in the cluster.
+ *
+ * This implementation of PlanFollower is relatively stateless, and it can
+ * synchronize schedulers and Plans that have arbitrary changes (performing set
+ * differences among existing queues). This makes it resilient to frequency of
+ * synchronization, and RM restart issues (no "catch up" is necessary).
+ */
+public class CapacitySchedulerPlanFollower implements PlanFollower {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(CapacitySchedulerPlanFollower.class);
+
+ private Collection<Plan> plans = new ArrayList<Plan>();
+
+ private Clock clock;
+ private CapacityScheduler scheduler;
+
+ @Override
+ public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans) {
+ LOG.info("Initializing Plan Follower Policy:"
+ + this.getClass().getCanonicalName());
+ if (!(sched instanceof CapacityScheduler)) {
+ throw new YarnRuntimeException(
+ "CapacitySchedulerPlanFollower can only work with CapacityScheduler");
+ }
+ this.clock = clock;
+ this.scheduler = (CapacityScheduler) sched;
+ this.plans.addAll(plans);
+ }
+
+ @Override
+ public synchronized void run() {
+ for (Plan plan : plans) {
+ synchronizePlan(plan);
+ }
+ }
+
+ @Override
+ public synchronized void synchronizePlan(Plan plan) {
+ String planQueueName = plan.getQueueName();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Running plan follower edit policy for plan: " + planQueueName);
+ }
+ // align with plan step
+ long step = plan.getStep();
+ long now = clock.getTime();
+ if (now % step != 0) {
+ now += step - (now % step);
+ }
+ CSQueue queue = scheduler.getQueue(planQueueName);
+ if (!(queue instanceof PlanQueue)) {
+ LOG.error("The Plan is not an PlanQueue!");
+ return;
+ }
+ PlanQueue planQueue = (PlanQueue) queue;
+ // first we publish to the plan the current availability of resources
+ Resource clusterResources = scheduler.getClusterResource();
+ float planAbsCap = planQueue.getAbsoluteCapacity();
+ Resource planResources = Resources.multiply(clusterResources, planAbsCap);
+ plan.setTotalCapacity(planResources);
+
+ Set<ReservationAllocation> currentReservations =
+ plan.getReservationsAtTime(now);
+ Set<String> curReservationNames = new HashSet<String>();
+ Resource reservedResources = Resource.newInstance(0, 0);
+ int numRes = 0;
+ if (currentReservations != null) {
+ numRes = currentReservations.size();
+ for (ReservationAllocation reservation : currentReservations) {
+ curReservationNames.add(reservation.getReservationId().toString());
+ Resources.addTo(reservedResources, reservation.getResourcesAtTime(now));
+ }
+ }
+ // create the default reservation queue if it doesnt exist
+ String defReservationQueue = planQueueName + PlanQueue.DEFAULT_QUEUE_SUFFIX;
+ if (scheduler.getQueue(defReservationQueue) == null) {
+ ReservationQueue defQueue =
+ new ReservationQueue(scheduler, defReservationQueue, planQueue);
+ try {
+ scheduler.addQueue(defQueue);
+ } catch (SchedulerDynamicEditException e) {
+ LOG.warn(
+ "Exception while trying to create default reservation queue for plan: {}",
+ planQueueName, e);
+ }
+ }
+ curReservationNames.add(defReservationQueue);
+ // if the resources dedicated to this plan has shrunk invoke replanner
+ if (Resources.greaterThan(scheduler.getResourceCalculator(),
+ clusterResources, reservedResources, planResources)) {
+ try {
+ plan.getReplanner().plan(plan, null);
+ } catch (PlanningException e) {
+ LOG.warn("Exception while trying to replan: {}", planQueueName, e);
+ }
+ }
+ // identify the reservations that have expired and new reservations that
+ // have to be activated
+ List<CSQueue> resQueues = planQueue.getChildQueues();
+ Set<String> expired = new HashSet<String>();
+ for (CSQueue resQueue : resQueues) {
+ String resQueueName = resQueue.getQueueName();
+ if (curReservationNames.contains(resQueueName)) {
+ // it is already existing reservation, so needed not create new
+ // reservation queue
+ curReservationNames.remove(resQueueName);
+ } else {
+ // the reservation has termination, mark for cleanup
+ expired.add(resQueueName);
+ }
+ }
+ // garbage collect expired reservations
+ cleanupExpiredQueues(plan.getMoveOnExpiry(), expired, defReservationQueue);
+
+ // Add new reservations and update existing ones
+ float totalAssignedCapacity = 0f;
+ if (currentReservations != null) {
+ // first release all excess capacity in default queue
+ try {
+ scheduler.setEntitlement(defReservationQueue, new QueueEntitlement(0f,
+ 1.0f));
+ } catch (YarnException e) {
+ LOG.warn(
+ "Exception while trying to release default queue capacity for plan: {}",
+ planQueueName, e);
+ }
+ // sort allocations from the one giving up the most resources, to the
+ // one asking for the most
+ // avoid order-of-operation errors that temporarily violate 100%
+ // capacity bound
+ List<ReservationAllocation> sortedAllocations =
+ sortByDelta(
+ new ArrayList<ReservationAllocation>(currentReservations), now);
+ for (ReservationAllocation res : sortedAllocations) {
+ String currResId = res.getReservationId().toString();
+ if (curReservationNames.contains(currResId)) {
+ ReservationQueue resQueue =
+ new ReservationQueue(scheduler, currResId, planQueue);
+ try {
+ scheduler.addQueue(resQueue);
+ } catch (SchedulerDynamicEditException e) {
+ LOG.warn(
+ "Exception while trying to activate reservation: {} for plan: {}",
+ currResId, planQueueName, e);
+ }
+ }
+ Resource capToAssign = res.getResourcesAtTime(now);
+ float targetCapacity = 0f;
+ if (planResources.getMemory() > 0
+ && planResources.getVirtualCores() > 0) {
+ targetCapacity =
+ Resources.divide(scheduler.getResourceCalculator(),
+ clusterResources, capToAssign, planResources);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Assigning capacity of {} to queue {} with target capacity {}",
+ capToAssign, currResId, targetCapacity);
+ }
+ // set maxCapacity to 100% unless the job requires gang, in which
+ // case we stick to capacity (as running early/before is likely a
+ // waste of resources)
+ float maxCapacity = 1.0f;
+ if (res.containsGangs()) {
+ maxCapacity = targetCapacity;
+ }
+ try {
+ scheduler.setEntitlement(currResId, new QueueEntitlement(
+ targetCapacity, maxCapacity));
+ } catch (YarnException e) {
+ LOG.warn("Exception while trying to size reservation for plan: {}",
+ currResId, planQueueName, e);
+ }
+ totalAssignedCapacity += targetCapacity;
+ }
+ }
+ // compute the default queue capacity
+ float defQCap = 1.0f - totalAssignedCapacity;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("PlanFollowerEditPolicyTask: total Plan Capacity: {} "
+ + "currReservation: {} default-queue capacity: {}", planResources,
+ numRes, defQCap);
+ }
+ // set the default queue to eat-up all remaining capacity
+ try {
+ scheduler.setEntitlement(defReservationQueue, new QueueEntitlement(
+ defQCap, 1.0f));
+ } catch (YarnException e) {
+ LOG.warn(
+ "Exception while trying to reclaim default queue capacity for plan: {}",
+ planQueueName, e);
+ }
+ // garbage collect finished reservations from plan
+ try {
+ plan.archiveCompletedReservations(now);
+ } catch (PlanningException e) {
+ LOG.error("Exception in archiving completed reservations: ", e);
+ }
+ LOG.info("Finished iteration of plan follower edit policy for plan: "
+ + planQueueName);
+
+ // Extension: update plan with app states,
+ // useful to support smart replanning
+ }
+
+ /**
+ * Move all apps in the set of queues to the parent plan queue's default
+ * reservation queue in a synchronous fashion
+ */
+ private void moveAppsInQueueSync(String expiredReservation,
+ String defReservationQueue) {
+ List<ApplicationAttemptId> activeApps =
+ scheduler.getAppsInQueue(expiredReservation);
+ if (activeApps.isEmpty()) {
+ return;
+ }
+ for (ApplicationAttemptId app : activeApps) {
+ // fallback to parent's default queue
+ try {
+ scheduler.moveApplication(app.getApplicationId(), defReservationQueue);
+ } catch (YarnException e) {
+ LOG.warn(
+ "Encountered unexpected error during migration of application: {} from reservation: {}",
+ app, expiredReservation, e);
+ }
+ }
+ }
+
+ /**
+ * First sets entitlement of queues to zero to prevent new app submission.
+ * Then move all apps in the set of queues to the parent plan queue's default
+ * reservation queue if move is enabled. Finally cleanups the queue by killing
+ * any apps (if move is disabled or move failed) and removing the queue
+ */
+ private void cleanupExpiredQueues(boolean shouldMove, Set<String> toRemove,
+ String defReservationQueue) {
+ for (String expiredReservation : toRemove) {
+ try {
+ // reduce entitlement to 0
+ scheduler.setEntitlement(expiredReservation, new QueueEntitlement(0.0f,
+ 0.0f));
+ if (shouldMove) {
+ moveAppsInQueueSync(expiredReservation, defReservationQueue);
+ }
+ if (scheduler.getAppsInQueue(expiredReservation).size() > 0) {
+ scheduler.killAllAppsInQueue(expiredReservation);
+ LOG.info("Killing applications in queue: {}", expiredReservation);
+ } else {
+ scheduler.removeQueue(expiredReservation);
+ LOG.info("Queue: " + expiredReservation + " removed");
+ }
+ } catch (YarnException e) {
+ LOG.warn("Exception while trying to expire reservation: {}",
+ expiredReservation, e);
+ }
+ }
+ }
+
+ @Override
+ public synchronized void setPlans(Collection<Plan> plans) {
+ this.plans.clear();
+ this.plans.addAll(plans);
+ }
+
+ /**
+ * Sort in the order from the least new amount of resources asked (likely
+ * negative) to the highest. This prevents "order-of-operation" errors related
+ * to exceeding 100% capacity temporarily.
+ */
+ private List<ReservationAllocation> sortByDelta(
+ List<ReservationAllocation> currentReservations, long now) {
+ Collections.sort(currentReservations, new ReservationAllocationComparator(
+ scheduler, now));
+ return currentReservations;
+ }
+
+ private class ReservationAllocationComparator implements
+ Comparator<ReservationAllocation> {
+ CapacityScheduler scheduler;
+ long now;
+
+ ReservationAllocationComparator(CapacityScheduler scheduler, long now) {
+ this.scheduler = scheduler;
+ this.now = now;
+ }
+
+ private Resource getUnallocatedReservedResources(
+ ReservationAllocation reservation) {
+ Resource resResource;
+ CSQueue resQueue =
+ scheduler.getQueue(reservation.getReservationId().toString());
+ if (resQueue != null) {
+ resResource =
+ Resources.subtract(
+ reservation.getResourcesAtTime(now),
+ Resources.multiply(scheduler.getClusterResource(),
+ resQueue.getAbsoluteCapacity()));
+ } else {
+ resResource = reservation.getResourcesAtTime(now);
+ }
+ return resResource;
+ }
+
+ @Override
+ public int compare(ReservationAllocation lhs, ReservationAllocation rhs) {
+ // compute delta between current and previous reservation, and compare
+ // based on that
+ Resource lhsRes = getUnallocatedReservedResources(lhs);
+ Resource rhsRes = getUnallocatedReservedResources(rhs);
+ return lhsRes.compareTo(rhsRes);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a795348/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanFollower.java
new file mode 100644
index 0000000..9d00366
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanFollower.java
@@ -0,0 +1,67 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.Collection;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.util.Clock;
+
+/**
+ * A PlanFollower is a component that runs on a timer, and synchronizes the
+ * underlying {@link ResourceScheduler} with the {@link Plan}(s) and viceversa.
+ *
+ * While different implementations might operate differently, the key idea is to
+ * map the current allocation of resources for each active reservation in the
+ * plan(s), to a corresponding notion in the underlying scheduler (e.g., tuning
+ * capacity of queues, set pool weights, or tweak application priorities). The
+ * goal is to affect the dynamic allocation of resources done by the scheduler
+ * so that the jobs obtain access to resources in a way that is consistent with
+ * the reservations in the plan. A key conceptual step here is to convert the
+ * absolute-valued promises made in the reservations to appropriate relative
+ * priorities/queue sizes etc.
+ *
+ * Symmetrically the PlanFollower exposes changes in cluster conditions (as
+ * tracked by the scheduler) to the plan, e.g., the overall amount of physical
+ * resources available. The Plan in turn can react by replanning its allocations
+ * if appropriate.
+ *
+ * The implementation can assume that is run frequently enough to be able to
+ * observe and react to normal operational changes in cluster conditions on the
+ * fly (e.g., if cluster resources drop, we can update the relative weights of a
+ * queue so that the absolute promises made to the job at reservation time are
+ * respected).
+ *
+ * However, due to RM restarts and the related downtime, it is advisable for
+ * implementations to operate in a stateless way, and be able to synchronize the
+ * state of plans/scheduler regardless of how big is the time gap between
+ * executions.
+ */
+public interface PlanFollower extends Runnable {
+
+ /**
+ * Init function that configures the PlanFollower, by providing:
+ *
+ * @param clock a reference to the system clock.
+ * @param sched a reference to the underlying scheduler
+ * @param plans references to the plans we should keep synchronized at every
+ * time tick.
+ */
+ public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans);
+
+ /**
+ * The function performing the actual synchronization operation for a given
+ * Plan. This is normally invoked by the run method, but it can be invoked
+ * synchronously to avoid race conditions when a user's reservation request
+ * start time is imminent.
+ *
+ * @param plan the Plan to synchronize
+ */
+ public void synchronizePlan(Plan plan);
+
+ /**
+ * Setter for the list of plans.
+ *
+ * @param plans the collection of Plans we operate on at every time tick.
+ */
+ public void setPlans(Collection<Plan> plans);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a795348/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
new file mode 100644
index 0000000..4eedd42
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
@@ -0,0 +1,319 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+
+public class TestCapacitySchedulerPlanFollower {
+
+ final static int GB = 1024;
+
+ private Clock mClock = null;
+ private CapacityScheduler scheduler = null;
+ private RMContext rmContext;
+ private RMContext spyRMContext;
+ private CapacitySchedulerContext csContext;
+ private ReservationAgent mAgent;
+ private Plan plan;
+ private Resource minAlloc = Resource.newInstance(GB, 1);
+ private Resource maxAlloc = Resource.newInstance(GB * 8, 8);
+ private ResourceCalculator res = new DefaultResourceCalculator();
+ private CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
+
+ @Rule
+ public TestName name = new TestName();
+
+ @Before
+ public void setUp() throws Exception {
+ CapacityScheduler spyCs = new CapacityScheduler();
+ scheduler = spy(spyCs);
+ rmContext = TestUtils.getMockRMContext();
+ spyRMContext = spy(rmContext);
+
+ ConcurrentMap<ApplicationId, RMApp> spyApps =
+ spy(new ConcurrentHashMap<ApplicationId, RMApp>());
+ RMApp rmApp = mock(RMApp.class);
+ when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any()))
+ .thenReturn(null);
+ Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
+ when(spyRMContext.getRMApps()).thenReturn(spyApps);
+
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ ReservationSystemTestUtil.setupQueueConfiguration(csConf);
+
+ scheduler.setConf(csConf);
+
+ csContext = mock(CapacitySchedulerContext.class);
+ when(csContext.getConfiguration()).thenReturn(csConf);
+ when(csContext.getConf()).thenReturn(csConf);
+ when(csContext.getMinimumResourceCapability()).thenReturn(minAlloc);
+ when(csContext.getMaximumResourceCapability()).thenReturn(maxAlloc);
+ when(csContext.getClusterResource()).thenReturn(
+ Resources.createResource(100 * 16 * GB, 100 * 32));
+ when(scheduler.getClusterResource()).thenReturn(
+ Resources.createResource(125 * GB, 125));
+ when(csContext.getResourceCalculator()).thenReturn(
+ new DefaultResourceCalculator());
+ RMContainerTokenSecretManager containerTokenSecretManager =
+ new RMContainerTokenSecretManager(csConf);
+ containerTokenSecretManager.rollMasterKey();
+ when(csContext.getContainerTokenSecretManager()).thenReturn(
+ containerTokenSecretManager);
+
+ scheduler.setRMContext(spyRMContext);
+ scheduler.init(csConf);
+ scheduler.start();
+
+ setupPlanFollower();
+ }
+
+ private void setupPlanFollower() throws Exception {
+ ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+ mClock = mock(Clock.class);
+ mAgent = mock(ReservationAgent.class);
+
+ String reservationQ = testUtil.getFullReservationQueueName();
+ CapacitySchedulerConfiguration csConf = scheduler.getConfiguration();
+ csConf.setReservationWindow(reservationQ, 20L);
+ csConf.setMaximumCapacity(reservationQ, 40);
+ csConf.setAverageCapacity(reservationQ, 20);
+ policy.init(reservationQ, csConf);
+ }
+
+ @Test
+ public void testWithMoveOnExpiry() throws PlanningException,
+ InterruptedException, AccessControlException {
+ // invoke plan follower test with move
+ testPlanFollower(true);
+ }
+
+ @Test
+ public void testWithKillOnExpiry() throws PlanningException,
+ InterruptedException, AccessControlException {
+ // invoke plan follower test with kill
+ testPlanFollower(false);
+ }
+
+ private void testPlanFollower(boolean isMove) throws PlanningException,
+ InterruptedException, AccessControlException {
+ // Initialize plan based on move flag
+ plan =
+ new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, mAgent,
+ scheduler.getClusterResource(), 1L, res,
+ scheduler.getMinimumResourceCapability(), maxAlloc, "dedicated",
+ null, isMove);
+
+ // add a few reservations to the plan
+ long ts = System.currentTimeMillis();
+ ReservationId r1 = ReservationId.newInstance(ts, 1);
+ int[] f1 = { 10, 10, 10, 10, 10 };
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(r1, null, "u3",
+ "dedicated", 0, 0 + f1.length, ReservationSystemTestUtil
+ .generateAllocation(0L, 1L, f1), res, minAlloc)));
+
+ ReservationId r2 = ReservationId.newInstance(ts, 2);
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(r2, null, "u3",
+ "dedicated", 3, 3 + f1.length, ReservationSystemTestUtil
+ .generateAllocation(3L, 1L, f1), res, minAlloc)));
+
+ ReservationId r3 = ReservationId.newInstance(ts, 3);
+ int[] f2 = { 0, 10, 20, 10, 0 };
+ assertTrue(plan.toString(),
+ plan.addReservation(new InMemoryReservationAllocation(r3, null, "u4",
+ "dedicated", 10, 10 + f2.length, ReservationSystemTestUtil
+ .generateAllocation(10L, 1L, f2), res, minAlloc)));
+
+ CapacitySchedulerPlanFollower planFollower =
+ new CapacitySchedulerPlanFollower();
+ planFollower.init(mClock, scheduler, Collections.singletonList(plan));
+
+ when(mClock.getTime()).thenReturn(0L);
+ planFollower.run();
+
+ CSQueue defQ =
+ scheduler.getQueue("dedicated" + PlanQueue.DEFAULT_QUEUE_SUFFIX);
+ CSQueue q = scheduler.getQueue(r1.toString());
+ assertNotNull(q);
+ // submit an app to r1
+ String user_0 = "test-user";
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId_0 =
+ ApplicationAttemptId.newInstance(appId, 0);
+ AppAddedSchedulerEvent addAppEvent =
+ new AppAddedSchedulerEvent(appId, q.getQueueName(), user_0);
+ scheduler.handle(addAppEvent);
+ AppAttemptAddedSchedulerEvent appAttemptAddedEvent =
+ new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
+ scheduler.handle(appAttemptAddedEvent);
+
+ // initial default reservation queue should have no apps
+ Assert.assertEquals(0, defQ.getNumApplications());
+
+ Assert.assertEquals(0.1, q.getCapacity(), 0.01);
+ Assert.assertEquals(0.1, q.getMaximumCapacity(), 1.0);
+ Assert.assertEquals(1, q.getNumApplications());
+
+ CSQueue q2 = scheduler.getQueue(r2.toString());
+ assertNull(q2);
+ CSQueue q3 = scheduler.getQueue(r3.toString());
+ assertNull(q3);
+
+ when(mClock.getTime()).thenReturn(3L);
+ planFollower.run();
+
+ Assert.assertEquals(0, defQ.getNumApplications());
+ q = scheduler.getQueue(r1.toString());
+ assertNotNull(q);
+ Assert.assertEquals(0.1, q.getCapacity(), 0.01);
+ Assert.assertEquals(0.1, q.getMaximumCapacity(), 1.0);
+ Assert.assertEquals(1, q.getNumApplications());
+ q2 = scheduler.getQueue(r2.toString());
+ assertNotNull(q2);
+ Assert.assertEquals(0.1, q.getCapacity(), 0.01);
+ Assert.assertEquals(0.1, q.getMaximumCapacity(), 1.0);
+ q3 = scheduler.getQueue(r3.toString());
+ assertNull(q3);
+
+ when(mClock.getTime()).thenReturn(10L);
+ planFollower.run();
+
+ q = scheduler.getQueue(r1.toString());
+ if (isMove) {
+ // app should have been moved to default reservation queue
+ Assert.assertEquals(1, defQ.getNumApplications());
+ assertNull(q);
+ } else {
+ // app should be killed
+ Assert.assertEquals(0, defQ.getNumApplications());
+ assertNotNull(q);
+ AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent =
+ new AppAttemptRemovedSchedulerEvent(appAttemptId_0,
+ RMAppAttemptState.KILLED, false);
+ scheduler.handle(appAttemptRemovedEvent);
+ }
+ q2 = scheduler.getQueue(r2.toString());
+ assertNull(q2);
+ q3 = scheduler.getQueue(r3.toString());
+ assertNotNull(q3);
+ Assert.assertEquals(0, q3.getCapacity(), 0.01);
+ Assert.assertEquals(1.0, q3.getMaximumCapacity(), 1.0);
+
+ when(mClock.getTime()).thenReturn(11L);
+ planFollower.run();
+
+ if (isMove) {
+ // app should have been moved to default reservation queue
+ Assert.assertEquals(1, defQ.getNumApplications());
+ } else {
+ // app should be killed
+ Assert.assertEquals(0, defQ.getNumApplications());
+ }
+ q = scheduler.getQueue(r1.toString());
+ assertNull(q);
+ q2 = scheduler.getQueue(r2.toString());
+ assertNull(q2);
+ q3 = scheduler.getQueue(r3.toString());
+ assertNotNull(q3);
+ Assert.assertEquals(0.1, q3.getCapacity(), 0.01);
+ Assert.assertEquals(0.1, q3.getMaximumCapacity(), 1.0);
+
+ when(mClock.getTime()).thenReturn(12L);
+ planFollower.run();
+
+ q = scheduler.getQueue(r1.toString());
+ assertNull(q);
+ q2 = scheduler.getQueue(r2.toString());
+ assertNull(q2);
+ q3 = scheduler.getQueue(r3.toString());
+ assertNotNull(q3);
+ Assert.assertEquals(0.2, q3.getCapacity(), 0.01);
+ Assert.assertEquals(0.2, q3.getMaximumCapacity(), 1.0);
+
+ when(mClock.getTime()).thenReturn(16L);
+ planFollower.run();
+
+ q = scheduler.getQueue(r1.toString());
+ assertNull(q);
+ q2 = scheduler.getQueue(r2.toString());
+ assertNull(q2);
+ q3 = scheduler.getQueue(r3.toString());
+ assertNull(q3);
+
+ assertTrue(defQ.getCapacity() > 0.9);
+
+ }
+
+ public static ApplicationACLsManager mockAppACLsManager() {
+ Configuration conf = new Configuration();
+ return new ApplicationACLsManager(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (scheduler != null) {
+ scheduler.stop();
+ }
+ }
+
+}
[30/50] [abbrv] git commit: YARN-2453.
TestProportionalCapacityPreemptionPolicy fails with FairScheduler. (Zhihai Xu
via kasha)
Posted by cu...@apache.org.
YARN-2453. TestProportionalCapacityPreemptionPolicy fails with FairScheduler. (Zhihai Xu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9721e2c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9721e2c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9721e2c1
Branch: refs/heads/YARN-1051
Commit: 9721e2c1feb5aecea3a6dab5bda96af1cd0f8de3
Parents: eab15af
Author: Karthik Kambatla <ka...@apache.org>
Authored: Sun Sep 21 23:13:45 2014 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Sun Sep 21 23:13:45 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../TestProportionalCapacityPreemptionPolicy.java | 12 +++++++-----
2 files changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9721e2c1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3052ed3..5c9cafd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -409,6 +409,9 @@ Release 2.6.0 - UNRELEASED
YARN-2452. TestRMApplicationHistoryWriter fails with FairScheduler.
(Zhihai Xu via kasha)
+ YARN-2453. TestProportionalCapacityPreemptionPolicy fails with
+ FairScheduler. (Zhihai Xu via kasha)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9721e2c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index 8a2840e..a0c2b01 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -112,6 +112,13 @@ public class TestProportionalCapacityPreemptionPolicy {
// report "ideal" preempt
conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0);
+ conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
+ ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
+ // FairScheduler doesn't support this test,
+ // Set CapacityScheduler as the scheduler for this test.
+ conf.set("yarn.resourcemanager.scheduler.class",
+ CapacityScheduler.class.getName());
mClock = mock(Clock.class);
mCS = mock(CapacityScheduler.class);
@@ -441,11 +448,6 @@ public class TestProportionalCapacityPreemptionPolicy {
@Test
public void testPolicyInitializeAfterSchedulerInitialized() {
- Configuration conf = new Configuration();
- conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
- ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
- conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
-
@SuppressWarnings("resource")
MockRM rm = new MockRM(conf);
rm.init(conf);
[07/50] [abbrv] git commit: HDFS-6840. Clients are always sent to the
same datanode when read is off rack. (wang)
Posted by cu...@apache.org.
HDFS-6840. Clients are always sent to the same datanode when read is off rack. (wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e730844
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e730844
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e730844
Branch: refs/heads/YARN-1051
Commit: 8e73084491c9f317bc8cc3590f93ca67a63687a8
Parents: 20a076b
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Sep 18 17:49:17 2014 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Sep 18 17:49:17 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/net/NetworkTopology.java | 36 +++++++++---------
.../net/NetworkTopologyWithNodeGroup.java | 23 +++++------
.../net/TestNetworkTopologyWithNodeGroup.java | 12 ++----
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 --
.../server/blockmanagement/DatanodeManager.java | 6 +--
.../hdfs/server/namenode/FSNamesystem.java | 12 +-----
.../src/main/resources/hdfs-default.xml | 13 -------
.../apache/hadoop/net/TestNetworkTopology.java | 40 ++++++++++----------
9 files changed, 57 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e730844/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 6ee6db7..50d5116 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -26,6 +26,7 @@ import java.util.TreeMap;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -689,6 +690,12 @@ public class NetworkTopology {
return rand;
}
+ @VisibleForTesting
+ void setRandomSeed(long seed) {
+ Random rand = getRandom();
+ rand.setSeed(seed);
+ }
+
/** randomly choose one node from <i>scope</i>
* if scope starts with ~, choose one from the all nodes except for the
* ones in <i>scope</i>; otherwise, choose one from <i>scope</i>
@@ -870,21 +877,19 @@ public class NetworkTopology {
/**
* Sort nodes array by network distance to <i>reader</i>.
* <p/>
- * In a three-level topology, a node can be either local, on the same rack, or
- * on a different rack from the reader. Sorting the nodes based on network
- * distance from the reader reduces network traffic and improves performance.
+ * In a three-level topology, a node can be either local, on the same rack,
+ * or on a different rack from the reader. Sorting the nodes based on network
+ * distance from the reader reduces network traffic and improves
+ * performance.
* <p/>
* As an additional twist, we also randomize the nodes at each network
- * distance using the provided random seed. This helps with load balancing
- * when there is data skew.
- *
- * @param reader Node where data will be read
- * @param nodes Available replicas with the requested data
- * @param seed Used to seed the pseudo-random generator that randomizes the
- * set of nodes at each network distance.
+ * distance. This helps with load balancing when there is data skew.
+ *
+ * @param reader Node where data will be read
+ * @param nodes Available replicas with the requested data
+ * @param activeLen Number of active nodes at the front of the array
*/
- public void sortByDistance(Node reader, Node[] nodes, int activeLen,
- long seed, boolean randomizeBlockLocationsPerBlock) {
+ public void sortByDistance(Node reader, Node[] nodes, int activeLen) {
/** Sort weights for the nodes array */
int[] weights = new int[activeLen];
for (int i=0; i<activeLen; i++) {
@@ -903,14 +908,7 @@ public class NetworkTopology {
list.add(node);
}
- // Seed is normally the block id
- // This means we use the same pseudo-random order for each block, for
- // potentially better page cache usage.
- // Seed is not used if we want to randomize block location for every block
Random rand = getRandom();
- if (!randomizeBlockLocationsPerBlock) {
- rand.setSeed(seed);
- }
int idx = 0;
for (List<Node> list: tree.values()) {
if (list != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e730844/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
index cc598c0..13160eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
@@ -268,19 +268,17 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology {
/**
* Sort nodes array by their distances to <i>reader</i>.
* <p/>
- * This is the same as
- * {@link NetworkTopology#sortByDistance(Node, Node[], long)} except with a
- * four-level network topology which contains the additional network distance
- * of a "node group" which is between local and same rack.
- *
- * @param reader Node where data will be read
- * @param nodes Available replicas with the requested data
- * @param seed Used to seed the pseudo-random generator that randomizes the
- * set of nodes at each network distance.
+ * This is the same as {@link NetworkTopology#sortByDistance(Node, Node[],
+ * int)} except with a four-level network topology which contains the
+ * additional network distance of a "node group" which is between local and
+ * same rack.
+ *
+ * @param reader Node where data will be read
+ * @param nodes Available replicas with the requested data
+ * @param activeLen Number of active nodes at the front of the array
*/
@Override
- public void sortByDistance(Node reader, Node[] nodes, int activeLen,
- long seed, boolean randomizeBlockLocationsPerBlock) {
+ public void sortByDistance(Node reader, Node[] nodes, int activeLen) {
// If reader is not a datanode (not in NetworkTopology tree), we need to
// replace this reader with a sibling leaf node in tree.
if (reader != null && !this.contains(reader)) {
@@ -293,8 +291,7 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology {
return;
}
}
- super.sortByDistance(reader, nodes, activeLen, seed,
- randomizeBlockLocationsPerBlock);
+ super.sortByDistance(reader, nodes, activeLen);
}
/** InnerNodeWithNodeGroup represents a switch/router of a data center, rack
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e730844/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
index 657fae3..15bd9fe 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
@@ -104,8 +104,7 @@ public class TestNetworkTopologyWithNodeGroup {
testNodes[1] = dataNodes[2];
testNodes[2] = dataNodes[3];
testNodes[3] = dataNodes[0];
- cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF, false);
+ cluster.sortByDistance(dataNodes[0], testNodes, testNodes.length);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
assertTrue(testNodes[2] == dataNodes[2]);
@@ -116,8 +115,7 @@ public class TestNetworkTopologyWithNodeGroup {
testNodes[1] = dataNodes[4];
testNodes[2] = dataNodes[1];
testNodes[3] = dataNodes[0];
- cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF, false);
+ cluster.sortByDistance(dataNodes[0], testNodes, testNodes.length);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
@@ -126,8 +124,7 @@ public class TestNetworkTopologyWithNodeGroup {
testNodes[1] = dataNodes[3];
testNodes[2] = dataNodes[2];
testNodes[3] = dataNodes[0];
- cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF, false);
+ cluster.sortByDistance(dataNodes[0], testNodes, testNodes.length);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[2]);
@@ -136,8 +133,7 @@ public class TestNetworkTopologyWithNodeGroup {
testNodes[1] = dataNodes[7];
testNodes[2] = dataNodes[2];
testNodes[3] = dataNodes[0];
- cluster.sortByDistance(computeNode, testNodes,
- testNodes.length, 0xDEADBEEF, false);
+ cluster.sortByDistance(computeNode, testNodes, testNodes.length);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[2]);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e730844/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c7339f0..436d2f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -771,6 +771,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7078. Fix listEZs to work correctly with snapshots. (wang)
+ HDFS-6840. Clients are always sent to the same datanode when read
+ is off rack. (wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e730844/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 7f04fc2..609b4c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -221,9 +221,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY = "dfs.namenode.min.supported.datanode.version";
public static final String DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_DEFAULT = "3.0.0-SNAPSHOT";
- public static final String DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK = "dfs.namenode.randomize-block-locations-per-block";
- public static final boolean DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK_DEFAULT = false;
-
public static final String DFS_NAMENODE_EDITS_DIR_MINIMUM_KEY = "dfs.namenode.edits.dir.minimum";
public static final int DFS_NAMENODE_EDITS_DIR_MINIMUM_DEFAULT = 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e730844/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 709f060..5314f09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -348,8 +348,7 @@ public class DatanodeManager {
/** Sort the located blocks by the distance to the target host. */
public void sortLocatedBlocks(final String targethost,
- final List<LocatedBlock> locatedblocks,
- boolean randomizeBlockLocationsPerBlock) {
+ final List<LocatedBlock> locatedblocks) {
//sort the blocks
// As it is possible for the separation of node manager and datanode,
// here we should get node but not datanode only .
@@ -376,8 +375,7 @@ public class DatanodeManager {
--lastActiveIndex;
}
int activeLen = lastActiveIndex + 1;
- networktopology.sortByDistance(client, b.getLocations(), activeLen, b
- .getBlock().getBlockId(), randomizeBlockLocationsPerBlock);
+ networktopology.sortByDistance(client, b.getLocations(), activeLen);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e730844/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index d0a1af6..2b03362 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -65,8 +65,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CAC
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY;
@@ -547,8 +545,6 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
private final FSImage fsImage;
- private boolean randomizeBlockLocationsPerBlock;
-
/**
* Notify that loading of this FSDirectory is complete, and
* it is imageLoaded for use
@@ -865,10 +861,6 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT);
- this.randomizeBlockLocationsPerBlock = conf.getBoolean(
- DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK,
- DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK_DEFAULT);
-
this.dtSecretManager = createDelegationTokenSecretManager(conf);
this.dir = new FSDirectory(this, conf);
this.snapshotManager = new SnapshotManager(dir);
@@ -1739,7 +1731,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
true);
if (blocks != null) {
blockManager.getDatanodeManager().sortLocatedBlocks(clientMachine,
- blocks.getLocatedBlocks(), randomizeBlockLocationsPerBlock);
+ blocks.getLocatedBlocks());
// lastBlock is not part of getLocatedBlocks(), might need to sort it too
LocatedBlock lastBlock = blocks.getLastLocatedBlock();
@@ -1748,7 +1740,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
Lists.newArrayListWithCapacity(1);
lastBlockList.add(lastBlock);
blockManager.getDatanodeManager().sortLocatedBlocks(clientMachine,
- lastBlockList, randomizeBlockLocationsPerBlock);
+ lastBlockList);
}
}
return blocks;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e730844/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index b28b216..d404c1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2070,19 +2070,6 @@
</property>
<property>
- <name>dfs.namenode.randomize-block-locations-per-block</name>
- <value>false</value>
- <description>When fetching replica locations of a block, the replicas
- are sorted based on network distance. This configuration parameter
- determines whether the replicas at the same network distance are randomly
- shuffled. By default, this is false, such that repeated requests for a block's
- replicas always result in the same order. This potentially improves page cache
- behavior. However, for some network topologies, it is desirable to shuffle this
- order for better load balancing.
- </description>
-</property>
-
-<property>
<name>dfs.datanode.block.id.layout.upgrade.threads</name>
<value>12</value>
<description>The number of threads to use when creating hard links from
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e730844/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
index faf9460..1758807 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
@@ -139,8 +139,8 @@ public class TestNetworkTopology {
testNodes[0] = dataNodes[1];
testNodes[1] = dataNodes[2];
testNodes[2] = dataNodes[0];
- cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF, false);
+ cluster.setRandomSeed(0xDEADBEEF);
+ cluster.sortByDistance(dataNodes[0], testNodes, testNodes.length);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
assertTrue(testNodes[2] == dataNodes[2]);
@@ -152,8 +152,8 @@ public class TestNetworkTopology {
dtestNodes[2] = dataNodes[11];
dtestNodes[3] = dataNodes[9];
dtestNodes[4] = dataNodes[10];
- cluster.sortByDistance(dataNodes[8], dtestNodes,
- dtestNodes.length - 2, 0xDEADBEEF, false);
+ cluster.setRandomSeed(0xDEADBEEF);
+ cluster.sortByDistance(dataNodes[8], dtestNodes, dtestNodes.length - 2);
assertTrue(dtestNodes[0] == dataNodes[8]);
assertTrue(dtestNodes[1] == dataNodes[11]);
assertTrue(dtestNodes[2] == dataNodes[12]);
@@ -164,8 +164,8 @@ public class TestNetworkTopology {
testNodes[0] = dataNodes[1];
testNodes[1] = dataNodes[3];
testNodes[2] = dataNodes[0];
- cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF, false);
+ cluster.setRandomSeed(0xDEADBEEF);
+ cluster.sortByDistance(dataNodes[0], testNodes, testNodes.length);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
assertTrue(testNodes[2] == dataNodes[3]);
@@ -174,8 +174,8 @@ public class TestNetworkTopology {
testNodes[0] = dataNodes[5];
testNodes[1] = dataNodes[3];
testNodes[2] = dataNodes[1];
- cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF, false);
+ cluster.setRandomSeed(0xDEADBEEF);
+ cluster.sortByDistance(dataNodes[0], testNodes, testNodes.length);
assertTrue(testNodes[0] == dataNodes[1]);
assertTrue(testNodes[1] == dataNodes[3]);
assertTrue(testNodes[2] == dataNodes[5]);
@@ -184,8 +184,8 @@ public class TestNetworkTopology {
testNodes[0] = dataNodes[1];
testNodes[1] = dataNodes[5];
testNodes[2] = dataNodes[3];
- cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF, false);
+ cluster.setRandomSeed(0xDEADBEEF);
+ cluster.sortByDistance(dataNodes[0], testNodes, testNodes.length);
assertTrue(testNodes[0] == dataNodes[1]);
assertTrue(testNodes[1] == dataNodes[3]);
assertTrue(testNodes[2] == dataNodes[5]);
@@ -194,24 +194,23 @@ public class TestNetworkTopology {
testNodes[0] = dataNodes[1];
testNodes[1] = dataNodes[5];
testNodes[2] = dataNodes[3];
- cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEAD, false);
+ cluster.setRandomSeed(0xDEAD);
+ cluster.sortByDistance(dataNodes[0], testNodes, testNodes.length);
// sortByDistance does not take the "data center" layer into consideration
// and it doesn't sort by getDistance, so 1, 5, 3 is also valid here
assertTrue(testNodes[0] == dataNodes[1]);
assertTrue(testNodes[1] == dataNodes[5]);
assertTrue(testNodes[2] == dataNodes[3]);
- // Array is just local rack nodes
- // Expect a random first node depending on the seed (normally the block ID).
+ // Array of just rack-local nodes
+ // Expect a random first node
DatanodeDescriptor first = null;
boolean foundRandom = false;
for (int i=5; i<=7; i++) {
testNodes[0] = dataNodes[5];
testNodes[1] = dataNodes[6];
testNodes[2] = dataNodes[7];
- cluster.sortByDistance(dataNodes[i], testNodes,
- testNodes.length, 0xBEADED+i, false);
+ cluster.sortByDistance(dataNodes[i], testNodes, testNodes.length);
if (first == null) {
first = testNodes[0];
} else {
@@ -222,16 +221,15 @@ public class TestNetworkTopology {
}
}
assertTrue("Expected to find a different first location", foundRandom);
- // Array of rack local nodes with randomizeBlockLocationsPerBlock set to
- // true
- // Expect random order of block locations for same block
+
+ // Array of just remote nodes
+ // Expect random first node
first = null;
for (int i = 1; i <= 4; i++) {
testNodes[0] = dataNodes[13];
testNodes[1] = dataNodes[14];
testNodes[2] = dataNodes[15];
- cluster.sortByDistance(dataNodes[15 + i], testNodes, testNodes.length,
- 0xBEADED, true);
+ cluster.sortByDistance(dataNodes[i], testNodes, testNodes.length);
if (first == null) {
first = testNodes[0];
} else {
[14/50] [abbrv] git commit: HADOOP-10946. Fix a bunch of typos in log
messages (Ray Chiang via aw)
Posted by cu...@apache.org.
HADOOP-10946. Fix a bunch of typos in log messages (Ray Chiang via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f03a7c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f03a7c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f03a7c0
Branch: refs/heads/YARN-1051
Commit: 9f03a7c018bb2c497cd0ef758f1a3e08e8163d06
Parents: 444acf8
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 19 11:32:55 2014 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Sep 19 11:33:07 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../main/java/org/apache/hadoop/io/VersionMismatchException.java | 2 +-
.../org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java | 2 +-
.../src/main/java/org/apache/hadoop/util/DiskChecker.java | 2 +-
.../java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java | 2 +-
.../src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java | 2 +-
.../src/main/java/org/apache/hadoop/streaming/StreamJob.java | 2 +-
7 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03a7c0/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 90053fc..7fc3b5b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -838,6 +838,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11109. Site build is broken. (Jian He via atm)
+ HADOOP-10946. Fix a bunch of typos in log messages (Ray Chiang via aw)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03a7c0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VersionMismatchException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VersionMismatchException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VersionMismatchException.java
index a72be58..015c15e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VersionMismatchException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VersionMismatchException.java
@@ -41,7 +41,7 @@ public class VersionMismatchException extends IOException {
/** Returns a string representation of this object. */
@Override
public String toString(){
- return "A record version mismatch occured. Expecting v"
+ return "A record version mismatch occurred. Expecting v"
+ expectedVersion + ", found v" + foundVersion;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03a7c0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java
index 57014d5..9c9164e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java
@@ -160,7 +160,7 @@ public abstract class MetricsDynamicMBeanBase implements DynamicMBean {
else if (attributeName.endsWith(MAX_TIME))
return or.getMaxTime();
else {
- MetricsUtil.LOG.error("Unexpected attrubute suffix");
+ MetricsUtil.LOG.error("Unexpected attribute suffix");
throw new AttributeNotFoundException();
}
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03a7c0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
index f2ee446..6b27ae5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
@@ -102,7 +102,7 @@ public class DiskChecker {
*/
public static void checkDir(File dir) throws DiskErrorException {
if (!mkdirsWithExistsCheck(dir)) {
- throw new DiskErrorException("Can not create directory: "
+ throw new DiskErrorException("Cannot create directory: "
+ dir.toString());
}
checkDirAccess(dir);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03a7c0/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 577711f..dae957e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -1095,7 +1095,7 @@ public class NativeAzureFileSystem extends FileSystem {
if (dstKey.startsWith(srcKey + PATH_DELIMITER)) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Renaming directory to a itself is disallowed. src=" + src
+ LOG.debug("Renaming directory to itself is disallowed. src=" + src
+ " dest=" + dst);
}
return false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03a7c0/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
index 4620cfc..8ac590b 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
@@ -681,7 +681,7 @@ public class Gridmix extends Configured implements Tool {
} catch (IOException e) {
LOG.warn("Failure killing " + job.getJobName(), e);
} catch (Exception e) {
- LOG.error("Unexcpected exception", e);
+ LOG.error("Unexpected exception", e);
}
}
LOG.info("Done.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03a7c0/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
index c64c0b6..7ff5641 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
@@ -1016,7 +1016,7 @@ public class StreamJob implements Tool {
if (background_) {
LOG.info("Job is running in background.");
} else if (!jc_.monitorAndPrintJob(jobConf_, running_)) {
- LOG.error("Job not Successful!");
+ LOG.error("Job not successful!");
return 1;
}
LOG.info("Output directory: " + output_);