You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/06/09 03:04:16 UTC
[01/29] hadoop git commit: HADOOP-14478. Optimize
NativeAzureFsInputStream for positional reads. Contributed by Rajesh
Balamohan [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-5734 985e77beb -> a48d47572 (forced update)
HADOOP-14478. Optimize NativeAzureFsInputStream for positional reads. Contributed by Rajesh Balamohan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fd9742c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fd9742c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fd9742c
Branch: refs/heads/YARN-5734
Commit: 5fd9742c83fbeae96bf0913bdcdf77fafbf15b2f
Parents: 8355609
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Jun 5 15:56:43 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Jun 5 15:56:43 2017 -0700
----------------------------------------------------------------------
.../fs/azure/AzureNativeFileSystemStore.java | 5 +---
.../hadoop/fs/azure/NativeAzureFileSystem.java | 31 ++++++++++++++++++--
2 files changed, 29 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fd9742c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index a5bb370..534919e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.fs.azure;
import static org.apache.hadoop.fs.azure.NativeAzureFileSystem.PATH_DELIMITER;
-import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -2043,11 +2042,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// Get blob reference and open the input buffer stream.
CloudBlobWrapper blob = getBlobReference(key);
- BufferedInputStream inBufStream = new BufferedInputStream(
- openInputStream(blob));
// Return a data input stream.
- DataInputStream inDataStream = new DataInputStream(inBufStream);
+ DataInputStream inDataStream = new DataInputStream(openInputStream(blob));
return inDataStream;
} catch (Exception e) {
// Re-throw as an Azure storage exception.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fd9742c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index b61baab..b2cc4ea 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -816,6 +816,27 @@ public class NativeAzureFileSystem extends FileSystem {
}
}
+ @Override
+ public synchronized void readFully(long position, byte[] buffer, int offset, int length)
+ throws IOException {
+ validatePositionedReadArgs(position, buffer, offset, length);
+
+ int nread = 0;
+ while (nread < length) {
+ // In case BlobInputStream is used, mark() can act as a hint to read ahead only this
+ // length instead of 4 MB boundary.
+ in.mark(length - nread);
+ int nbytes = read(position + nread,
+ buffer,
+ offset + nread,
+ length - nread);
+ if (nbytes < 0) {
+ throw new EOFException(FSExceptionMessages.EOF_IN_READ_FULLY);
+ }
+ nread += nbytes;
+ }
+ }
+
/*
* Reads up to len bytes of data from the input stream into an array of
* bytes. An attempt is made to read as many as len bytes, but a smaller
@@ -886,9 +907,13 @@ public class NativeAzureFileSystem extends FileSystem {
if (pos < 0) {
throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
}
- IOUtils.closeStream(in);
- in = store.retrieve(key);
- this.pos = in.skip(pos);
+ if (this.pos > pos) {
+ IOUtils.closeStream(in);
+ in = store.retrieve(key);
+ this.pos = in.skip(pos);
+ } else {
+ this.pos += in.skip(pos - this.pos);
+ }
LOG.debug("Seek to position {}. Bytes skipped {}", pos,
this.pos);
} catch(IOException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/29] hadoop git commit: HADOOP-14283. Upgrade AWS SDK to 1.11.134.
Contributed by Aaron Fabbri
Posted by jh...@apache.org.
HADOOP-14283. Upgrade AWS SDK to 1.11.134. Contributed by Aaron Fabbri
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c9694c5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c9694c5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c9694c5
Branch: refs/heads/YARN-5734
Commit: 7c9694c56712e20589d83b31e71c7bbddc131b8d
Parents: 1a33c9d
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Jun 8 15:06:30 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Jun 8 15:06:30 2017 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9694c5/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3cfdc18..29b3237 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -130,7 +130,7 @@
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.11.86</aws-java-sdk.version>
+ <aws-java-sdk.version>1.11.134</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version>
<!-- the version of Hadoop declared in the version resources; can be overridden
so that Hadoop 3.x can declare itself a 2.x artifact. -->
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/29] hadoop git commit: HDFS-11932. BPServiceActor thread name is
not correctly set. Contributed by Chen Liang.
Posted by jh...@apache.org.
HDFS-11932. BPServiceActor thread name is not correctly set. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/867903db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/867903db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/867903db
Branch: refs/heads/YARN-5734
Commit: 867903db2b38a59c86e71f1aaef3b0f9a87bfe17
Parents: 536f057
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Jun 6 13:51:02 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Jun 6 13:51:02 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/datanode/BPOfferService.java | 4 ++--
.../org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/867903db/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index 042169a..2644c0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -189,8 +189,8 @@ class BPOfferService {
if (bpNSInfo != null) {
return bpNSInfo.getBlockPoolID();
} else {
- LOG.warn("Block pool ID needed, but service not yet registered with NN",
- new Exception("trace"));
+ LOG.warn("Block pool ID needed, but service not yet registered with " +
+ "NN, trace:", new Exception());
return null;
}
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/867903db/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index fcf7d5e..0896844 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -541,7 +541,7 @@ class BPServiceActor implements Runnable {
//Thread is started already
return;
}
- bpThread = new Thread(this, formatThreadName("heartbeating", nnAddr));
+ bpThread = new Thread(this);
bpThread.setDaemon(true); // needed for JUnit testing
bpThread.start();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/29] hadoop git commit: YARN-6604. Allow metric TTL for
Application table to be specified through cmd (Haibo Chen via Varun Saxena)
Posted by jh...@apache.org.
YARN-6604. Allow metric TTL for Application table to be specified through cmd (Haibo Chen via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b7bff70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b7bff70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b7bff70
Branch: refs/heads/YARN-5734
Commit: 0b7bff706e9a5c4a17e0f46deceb2521168d25b9
Parents: 5ec7163
Author: Varun Saxena <va...@apache.org>
Authored: Wed Jun 7 21:51:07 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Wed Jun 7 21:51:07 2017 +0530
----------------------------------------------------------------------
.../storage/TimelineSchemaCreator.java | 36 ++++++++++++++------
1 file changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b7bff70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index b436eec..b3b749e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -60,9 +60,10 @@ public final class TimelineSchemaCreator {
final static String NAME = TimelineSchemaCreator.class.getSimpleName();
private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
private static final String SKIP_EXISTING_TABLE_OPTION_SHORT = "s";
+ private static final String APP_METRICS_TTL_OPTION_SHORT = "ma";
private static final String APP_TABLE_NAME_SHORT = "a";
private static final String APP_TO_FLOW_TABLE_NAME_SHORT = "a2f";
- private static final String TTL_OPTION_SHORT = "m";
+ private static final String ENTITY_METRICS_TTL_OPTION_SHORT = "me";
private static final String ENTITY_TABLE_NAME_SHORT = "e";
private static final String HELP_SHORT = "h";
private static final String CREATE_TABLES_SHORT = "c";
@@ -87,12 +88,12 @@ public final class TimelineSchemaCreator {
if (StringUtils.isNotBlank(entityTableName)) {
hbaseConf.set(EntityTable.TABLE_NAME_CONF_NAME, entityTableName);
}
- // Grab the TTL argument
- String entityTableTTLMetrics =commandLine.getOptionValue(
- TTL_OPTION_SHORT);
- if (StringUtils.isNotBlank(entityTableTTLMetrics)) {
- int metricsTTL = Integer.parseInt(entityTableTTLMetrics);
- new EntityTable().setMetricsTTL(metricsTTL, hbaseConf);
+ // Grab the entity metrics TTL
+ String entityTableMetricsTTL = commandLine.getOptionValue(
+ ENTITY_METRICS_TTL_OPTION_SHORT);
+ if (StringUtils.isNotBlank(entityTableMetricsTTL)) {
+ int entityMetricsTTL = Integer.parseInt(entityTableMetricsTTL);
+ new EntityTable().setMetricsTTL(entityMetricsTTL, hbaseConf);
}
// Grab the appToflowTableName argument
String appToflowTableName = commandLine.getOptionValue(
@@ -107,6 +108,13 @@ public final class TimelineSchemaCreator {
hbaseConf.set(ApplicationTable.TABLE_NAME_CONF_NAME,
applicationTableName);
}
+ // Grab the application metrics TTL
+ String applicationTableMetricsTTL = commandLine.getOptionValue(
+ APP_METRICS_TTL_OPTION_SHORT);
+ if (StringUtils.isNotBlank(applicationTableMetricsTTL)) {
+ int appMetricsTTL = Integer.parseInt(applicationTableMetricsTTL);
+ new ApplicationTable().setMetricsTTL(appMetricsTTL, hbaseConf);
+ }
// create all table schemas in hbase
final boolean skipExisting = commandLine.hasOption(
@@ -145,9 +153,9 @@ public final class TimelineSchemaCreator {
o.setRequired(false);
options.addOption(o);
- o = new Option(TTL_OPTION_SHORT, "metricsTTL", true,
+ o = new Option(ENTITY_METRICS_TTL_OPTION_SHORT, "entityMetricsTTL", true,
"TTL for metrics column family");
- o.setArgName("metricsTTL");
+ o.setArgName("entityMetricsTTL");
o.setRequired(false);
options.addOption(o);
@@ -163,6 +171,12 @@ public final class TimelineSchemaCreator {
o.setRequired(false);
options.addOption(o);
+ o = new Option(APP_METRICS_TTL_OPTION_SHORT, "applicationMetricsTTL", true,
+ "TTL for metrics column family");
+ o.setArgName("applicationMetricsTTL");
+ o.setRequired(false);
+ options.addOption(o);
+
// Options without an argument
// No need to set arg name since we do not need an argument here
o = new Option(SKIP_EXISTING_TABLE_OPTION_SHORT, "skipExistingTable",
@@ -193,12 +207,14 @@ public final class TimelineSchemaCreator {
usage.append("The Optional options for creating tables include: \n");
usage.append("[-entityTableName <Entity Table Name>] " +
"The name of the Entity table\n");
- usage.append("[-metricsTTL <Entity Table Metrics TTL>]" +
+ usage.append("[-entityMetricsTTL <Entity Table Metrics TTL>]" +
" TTL for metrics in the Entity table\n");
usage.append("[-appToflowTableName <AppToflow Table Name>]" +
" The name of the AppToFlow table\n");
usage.append("[-applicationTableName <Application Table Name>]" +
" The name of the Application table\n");
+ usage.append("[-applicationMetricsTTL <Application Table Metrics TTL>]" +
+ " TTL for metrics in the Application table\n");
usage.append("[-skipExistingTable] Whether to skip existing" +
" hbase tables\n");
System.out.println(usage.toString());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/29] hadoop git commit: YARN-6547. Enhance SLS-based tests
leveraging invariant checker.
Posted by jh...@apache.org.
YARN-6547. Enhance SLS-based tests leveraging invariant checker.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b65100c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b65100c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b65100c1
Branch: refs/heads/YARN-5734
Commit: b65100c14bf9134de2bd8248dc62735682bee26c
Parents: c31cb87
Author: Carlo Curino <cu...@apache.org>
Authored: Tue Jun 6 17:19:15 2017 -0700
Committer: Carlo Curino <cu...@apache.org>
Committed: Tue Jun 6 17:19:15 2017 -0700
----------------------------------------------------------------------
.../hadoop/metrics2/source/JvmMetrics.java | 10 ++++
hadoop-tools/hadoop-sls/pom.xml | 2 +
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 8 +++
.../hadoop/yarn/sls/BaseSLSRunnerTest.java | 41 +++++++++++++--
.../sls/TestReservationSystemInvariants.java | 4 ++
.../apache/hadoop/yarn/sls/TestSLSRunner.java | 17 +++---
.../src/test/resources/exit-invariants.txt | 8 +++
.../src/test/resources/log4j.properties | 19 +++++++
.../src/test/resources/ongoing-invariants.txt | 54 ++++++++++++++++++++
9 files changed, 152 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b65100c1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
index caba170..c6369cd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
@@ -27,6 +27,7 @@ import java.lang.management.GarbageCollectorMXBean;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.log.metrics.EventCounter;
import org.apache.hadoop.metrics2.MetricsCollector;
@@ -59,6 +60,15 @@ public class JvmMetrics implements MetricsSource {
}
}
+ @VisibleForTesting
+ public synchronized void registerIfNeeded(){
+ // during tests impl might exist, but is not registered
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ if (ms.getSource("JvmMetrics") == null) {
+ ms.register(JvmMetrics.name(), JvmMetrics.description(), this);
+ }
+ }
+
static final float M = 1024*1024;
static public final float MEMORY_MAX_UNLIMITED_MB = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b65100c1/hadoop-tools/hadoop-sls/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml
index d70021c..4089473 100644
--- a/hadoop-tools/hadoop-sls/pom.xml
+++ b/hadoop-tools/hadoop-sls/pom.xml
@@ -135,6 +135,8 @@
<exclude>src/test/resources/syn.json</exclude>
<exclude>src/test/resources/inputsls.json</exclude>
<exclude>src/test/resources/nodes.json</exclude>
+ <exclude>src/test/resources/exit-invariants.txt</exclude>
+ <exclude>src/test/resources/ongoing-invariants.txt</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b65100c1/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index e679c9d..a77d401 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.tools.rumen.JobTraceReader;
import org.apache.hadoop.tools.rumen.LoggedJob;
import org.apache.hadoop.tools.rumen.LoggedTask;
@@ -243,6 +244,13 @@ public class SLSRunner extends Configured implements Tool {
return new MockAMLauncher(se, this.rmContext, amMap);
}
};
+
+ // Across runs of parametrized tests, the JvmMetrics objects is retained,
+ // but is not registered correctly
+ JvmMetrics jvmMetrics = JvmMetrics.initSingleton("ResourceManager", null);
+ jvmMetrics.registerIfNeeded();
+
+ // Init and start the actual ResourceManager
rm.init(rmConf);
rm.start();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b65100c1/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
index 8ef72ab..6b369f2 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
@@ -21,8 +21,10 @@ import net.jcip.annotations.NotThreadSafe;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.monitor.invariants.MetricsInvariantChecker;
import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
@@ -39,7 +41,7 @@ import java.util.UUID;
@RunWith(value = Parameterized.class)
@NotThreadSafe
@SuppressWarnings("VisibilityModifier")
-public class BaseSLSRunnerTest {
+public abstract class BaseSLSRunnerTest {
@Parameter(value = 0)
public String schedulerType;
@@ -54,6 +56,11 @@ public class BaseSLSRunnerTest {
public String nodeFile;
protected SLSRunner sls;
+ protected String ongoingInvariantFile;
+ protected String exitInvariantFile;
+
+ @Before
+ public abstract void setup();
@After
public void tearDown() throws InterruptedException {
@@ -82,22 +89,30 @@ public class BaseSLSRunnerTest {
switch (traceType) {
case "OLD_SLS":
args = new String[] {"-inputsls", traceLocation, "-output",
- slsOutputDir.getAbsolutePath()};
+ slsOutputDir.getAbsolutePath() };
break;
case "OLD_RUMEN":
args = new String[] {"-inputrumen", traceLocation, "-output",
- slsOutputDir.getAbsolutePath()};
+ slsOutputDir.getAbsolutePath() };
break;
default:
args = new String[] {"-tracetype", traceType, "-tracelocation",
- traceLocation, "-output", slsOutputDir.getAbsolutePath()};
+ traceLocation, "-output", slsOutputDir.getAbsolutePath() };
}
if (nodeFile != null) {
- args = ArrayUtils.addAll(args, new String[] {"-nodes", nodeFile});
+ args = ArrayUtils.addAll(args, new String[] {"-nodes", nodeFile });
}
+ // enable continuous invariant checks
conf.set(YarnConfiguration.RM_SCHEDULER, schedulerType);
+ if (ongoingInvariantFile != null) {
+ conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
+ MetricsInvariantChecker.class.getCanonicalName());
+ conf.set(MetricsInvariantChecker.INVARIANTS_FILE, ongoingInvariantFile);
+ conf.setBoolean(MetricsInvariantChecker.THROW_ON_VIOLATION, true);
+ }
+
sls = new SLSRunner(conf);
sls.run(args);
@@ -115,6 +130,22 @@ public class BaseSLSRunnerTest {
}
timeout--;
}
+ shutdownHookInvariantCheck();
+ }
+
+ /**
+ * Checks exit invariants (e.g., number of apps submitted, completed, etc.).
+ */
+ private void shutdownHookInvariantCheck() {
+
+ if(exitInvariantFile!=null) {
+ MetricsInvariantChecker ic = new MetricsInvariantChecker();
+ Configuration conf = new Configuration();
+ conf.set(MetricsInvariantChecker.INVARIANTS_FILE, exitInvariantFile);
+ conf.setBoolean(MetricsInvariantChecker.THROW_ON_VIOLATION, true);
+ ic.init(conf, null, null);
+ ic.editSchedule();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b65100c1/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestReservationSystemInvariants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestReservationSystemInvariants.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestReservationSystemInvariants.java
index b3a79cb..22e1e2e 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestReservationSystemInvariants.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestReservationSystemInvariants.java
@@ -70,4 +70,8 @@ public class TestReservationSystemInvariants extends BaseSLSRunnerTest {
}
+ @Override
+ public void setup() {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b65100c1/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
index b2bc8d5..567f0d9 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.sls;
import net.jcip.annotations.NotThreadSafe;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -38,12 +41,8 @@ public class TestSLSRunner extends BaseSLSRunnerTest {
@Parameters(name = "Testing with: {1}, {0}, (nodeFile {3})")
public static Collection<Object[]> data() {
- String capScheduler =
- "org.apache.hadoop.yarn.server.resourcemanager.scheduler."
- + "capacity.CapacityScheduler";
- String fairScheduler =
- "org.apache.hadoop.yarn.server.resourcemanager.scheduler."
- + "fair.FairScheduler";
+ String capScheduler = CapacityScheduler.class.getCanonicalName();
+ String fairScheduler = FairScheduler.class.getCanonicalName();
String slsTraceFile = "src/test/resources/inputsls.json";
String rumenTraceFile = "src/main/data/2jobs2min-rumen-jh.json";
String synthTraceFile = "src/test/resources/syn.json";
@@ -73,6 +72,12 @@ public class TestSLSRunner extends BaseSLSRunnerTest {
});
}
+ @Before
+ public void setup() {
+ ongoingInvariantFile = "src/test/resources/ongoing-invariants.txt";
+ exitInvariantFile = "src/test/resources/exit-invariants.txt";
+ }
+
@Test(timeout = 60000)
@SuppressWarnings("all")
public void testSimulatorRunning() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b65100c1/hadoop-tools/hadoop-sls/src/test/resources/exit-invariants.txt
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/resources/exit-invariants.txt b/hadoop-tools/hadoop-sls/src/test/resources/exit-invariants.txt
new file mode 100644
index 0000000..b4a3228
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/test/resources/exit-invariants.txt
@@ -0,0 +1,8 @@
+ActiveApplications >= 0
+AppsCompleted >= 0
+AppsFailed >= 0
+AppsKilled >= 0
+AppsPending >= 0
+AppsRunning >= 0
+AppsSubmitted >= 0
+PendingContainers >= 0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b65100c1/hadoop-tools/hadoop-sls/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/resources/log4j.properties b/hadoop-tools/hadoop-sls/src/test/resources/log4j.properties
new file mode 100644
index 0000000..81a3f6a
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b65100c1/hadoop-tools/hadoop-sls/src/test/resources/ongoing-invariants.txt
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/resources/ongoing-invariants.txt b/hadoop-tools/hadoop-sls/src/test/resources/ongoing-invariants.txt
new file mode 100644
index 0000000..363ed0d
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/test/resources/ongoing-invariants.txt
@@ -0,0 +1,54 @@
+running_0 >= 0
+running_60 >= 0
+running_300 >= 0
+running_1440 >= 0
+AppsSubmitted >= 0
+AppsRunning >= 0
+AppsPending >= 0
+AppsCompleted >= 0
+AppsKilled >= 0
+AppsFailed >= 0
+AllocatedMB >= 0
+AllocatedVCores >= 0
+AllocatedContainers >= 0
+AggregateContainersAllocated >= 0
+AggregateNodeLocalContainersAllocated >= 0
+AggregateRackLocalContainersAllocated >= 0
+AggregateOffSwitchContainersAllocated >= 0
+AggregateContainersReleased >= 0
+AggregateContainersPreempted >= 0
+AvailableMB >= 0
+AvailableVCores >= 0
+PendingMB >= 0
+PendingVCores >= 0
+PendingContainers >= 0
+ReservedMB >= 0
+ReservedVCores >= 0
+ReservedContainers >= 0
+ActiveUsers >= 0
+ActiveApplications >= 0
+AppAttemptFirstContainerAllocationDelayNumOps >= 0
+AppAttemptFirstContainerAllocationDelayAvgTime >= 0
+MemNonHeapUsedM >= 0
+MemNonHeapCommittedM >= 0
+MemNonHeapMaxM >= 0 || MemNonHeapMaxM == -1
+MemHeapUsedM >= 0
+MemHeapCommittedM >= 0
+MemHeapMaxM >= 0
+MemMaxM >= 0
+GcCountPS_Scavenge >= 0
+GcTimeMillisPS_Scavenge >= 0
+GcCountPS_MarkSweep >= 0
+GcTimeMillisPS_MarkSweep >= 0
+GcCount >= 0
+GcTimeMillis >= 0
+ThreadsNew >= 0
+ThreadsRunnable >= 0
+ThreadsBlocked >= 0
+ThreadsWaiting >= 0
+ThreadsTimedWaiting >= 0
+ThreadsTerminated >= 0
+LogFatal >= 0
+LogError >= 0
+LogWarn >= 0
+LogInfo >= 0
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/29] hadoop git commit: HDFS-11711. DN should not delete the block
On "Too many open files" Exception. Contributed by Brahma Reddy Battula.
Posted by jh...@apache.org.
HDFS-11711. DN should not delete the block On "Too many open files" Exception. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1869e177
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1869e177
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1869e177
Branch: refs/heads/YARN-5734
Commit: 1869e1771c7eeea46ccb822ce6f7081d994bb12c
Parents: b65100c
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Wed Jun 7 13:05:03 2017 +0800
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Wed Jun 7 13:05:33 2017 +0800
----------------------------------------------------------------------
.../hdfs/server/datanode/BlockSender.java | 13 ++--
.../server/datanode/DataNodeFaultInjector.java | 4 ++
.../server/datanode/TestDataNodeMetrics.java | 62 ++++++++++++++++++++
3 files changed, 75 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1869e177/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index 203ee35..3ff5c75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -302,6 +302,7 @@ class BlockSender implements java.io.Closeable {
LengthInputStream metaIn = null;
boolean keepMetaInOpen = false;
try {
+ DataNodeFaultInjector.get().throwTooManyOpenFiles();
metaIn = datanode.data.getMetaDataInputStream(block);
if (!corruptChecksumOk || metaIn != null) {
if (metaIn == null) {
@@ -331,10 +332,14 @@ class BlockSender implements java.io.Closeable {
LOG.warn("Could not find metadata file for " + block);
}
} catch (FileNotFoundException e) {
- // The replica is on its volume map but not on disk
- datanode.notifyNamenodeDeletedBlock(block, replica.getStorageUuid());
- datanode.data.invalidate(block.getBlockPoolId(),
- new Block[]{block.getLocalBlock()});
+ if ((e.getMessage() != null) && !(e.getMessage()
+ .contains("Too many open files"))) {
+ // The replica is on its volume map but not on disk
+ datanode
+ .notifyNamenodeDeletedBlock(block, replica.getStorageUuid());
+ datanode.data.invalidate(block.getBlockPoolId(),
+ new Block[] {block.getLocalBlock()});
+ }
throw e;
} finally {
if (!keepMetaInOpen) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1869e177/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
index d2d557f..0a2a60b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
+import java.io.FileNotFoundException;
import java.io.IOException;
/**
@@ -85,4 +86,7 @@ public class DataNodeFaultInjector {
public void startOfferService() throws Exception {}
public void endOfferService() throws Exception {}
+
+ public void throwTooManyOpenFiles() throws FileNotFoundException {
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1869e177/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
index 9abc19d..7b3dea7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
@@ -24,8 +24,10 @@ import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.*;
import java.io.Closeable;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.management.ManagementFactory;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.List;
@@ -45,8 +47,10 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -365,4 +369,62 @@ public class TestDataNodeMetrics {
}
}
}
+
+ @Test
+ public void testDNShouldNotDeleteBlockONTooManyOpenFiles()
+ throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ conf.setLong(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 1);
+ DataNodeFaultInjector oldInjector = DataNodeFaultInjector.get();
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ final DataNodeFaultInjector injector =
+ Mockito.mock(DataNodeFaultInjector.class);
+ try {
+ // wait until the cluster is up
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ Path p = new Path("/testShouldThrowTMP");
+ DFSTestUtil.writeFile(fs, p, new String("testdata"));
+ //Before DN throws too many open files
+ verifyBlockLocations(fs, p, 1);
+ Mockito.doThrow(new FileNotFoundException("Too many open files")).
+ when(injector).
+ throwTooManyOpenFiles();
+ DataNodeFaultInjector.set(injector);
+ ExtendedBlock b =
+ fs.getClient().getLocatedBlocks(p.toString(), 0).get(0).getBlock();
+ try {
+ new BlockSender(b, 0, -1, false, true, true,
+ cluster.getDataNodes().get(0), null,
+ CachingStrategy.newDefaultStrategy());
+ fail("Must throw FileNotFoundException");
+ } catch (FileNotFoundException fe) {
+ assertTrue("Should throw too many open files",
+ fe.getMessage().contains("Too many open files"));
+ }
+ cluster.triggerHeartbeats(); // IBR delete ack
+ //After DN throws too many open files
+ assertTrue(cluster.getDataNodes().get(0).getFSDataset().isValidBlock(b));
+ verifyBlockLocations(fs, p, 1);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ DataNodeFaultInjector.set(oldInjector);
+ }
+ }
+
+ private void verifyBlockLocations(DistributedFileSystem fs, Path p,
+ int expected) throws IOException, TimeoutException, InterruptedException {
+ final LocatedBlock lb =
+ fs.getClient().getLocatedBlocks(p.toString(), 0).get(0);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return lb.getLocations().length == expected;
+ }
+ }, 1000, 6000);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/29] hadoop git commit: YARN-5952. Create REST API for changing
YARN scheduler configurations. (Jonathan Hung via wangda)
Posted by jh...@apache.org.
YARN-5952. Create REST API for changing YARN scheduler configurations. (Jonathan Hung via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e160fc1c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e160fc1c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e160fc1c
Branch: refs/heads/YARN-5734
Commit: e160fc1c36d9379e5990b4b28b43c1fe6bac0087
Parents: 556a5e5
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Apr 3 10:12:01 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Thu Jun 8 20:03:59 2017 -0700
----------------------------------------------------------------------
.../scheduler/MutableConfScheduler.java | 40 ++
.../scheduler/MutableConfigurationProvider.java | 5 +-
.../scheduler/capacity/CapacityScheduler.java | 16 +-
.../conf/InMemoryConfigurationStore.java | 6 +-
.../conf/MutableCSConfigurationProvider.java | 24 +-
.../resourcemanager/webapp/RMWebServices.java | 169 +++++++
.../webapp/dao/QueueConfigInfo.java | 57 +++
.../webapp/dao/QueueConfigsUpdateInfo.java | 60 +++
.../TestMutableCSConfigurationProvider.java | 6 +-
.../TestRMWebServicesConfigurationMutation.java | 477 +++++++++++++++++++
10 files changed, 849 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e160fc1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
new file mode 100644
index 0000000..35e36e1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.security.UserGroupInformation;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Interface for a scheduler that supports changing configuration at runtime.
+ *
+ */
+public interface MutableConfScheduler extends ResourceScheduler {
+
+ /**
+ * Update the scheduler's configuration.
+ * @param user Caller of this update
+ * @param confUpdate key-value map of the configuration update
+ * @throws IOException if update is invalid
+ */
+ void updateConfiguration(UserGroupInformation user,
+ Map<String, String> confUpdate) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e160fc1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index da30a2b..889c3bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import java.io.IOException;
import java.util.Map;
/**
@@ -29,7 +30,9 @@ public interface MutableConfigurationProvider {
* Update the scheduler configuration with the provided key value pairs.
* @param user User issuing the request
* @param confUpdate Key-value pairs for configurations to be updated.
+ * @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(String user, Map<String, String> confUpdate);
+ void mutateConfiguration(String user, Map<String, String> confUpdate)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e160fc1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 9353df2..85138d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -85,6 +85,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnSched
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
@@ -149,7 +151,7 @@ import com.google.common.util.concurrent.SettableFuture;
public class CapacityScheduler extends
AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> implements
PreemptableResourceScheduler, CapacitySchedulerContext, Configurable,
- ResourceAllocationCommitter {
+ ResourceAllocationCommitter, MutableConfScheduler {
private static final Log LOG = LogFactory.getLog(CapacityScheduler.class);
@@ -2486,4 +2488,16 @@ public class CapacityScheduler extends
writeLock.unlock();
}
}
+
+ @Override
+ public void updateConfiguration(UserGroupInformation user,
+ Map<String, String> confUpdate) throws IOException {
+ if (csConfProvider instanceof MutableConfigurationProvider) {
+ ((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
+ user.getShortUserName(), confUpdate);
+ } else {
+ throw new UnsupportedOperationException("Configured CS configuration " +
+ "provider does not support updating configuration.");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e160fc1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index a208fb9..b97be1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -58,7 +58,11 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore {
if (isValid) {
Map<String, String> mutations = mutation.getUpdates();
for (Map.Entry<String, String> kv : mutations.entrySet()) {
- schedConf.set(kv.getKey(), kv.getValue());
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
}
}
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e160fc1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 267ab6a..ea1b3c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -60,34 +60,44 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
Configuration initialSchedConf = new Configuration(false);
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
- this.schedConf = initialSchedConf;
- confStore.initialize(config, initialSchedConf);
+ this.schedConf = new Configuration(false);
+ // We need to explicitly set the key-values in schedConf, otherwise
+ // these configuration keys cannot be deleted when
+ // configuration is reloaded.
+ for (Map.Entry<String, String> kv : initialSchedConf) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ confStore.initialize(config, schedConf);
this.conf = config;
}
@Override
public CapacitySchedulerConfiguration loadConfiguration(Configuration
configuration) throws IOException {
- Configuration loadedConf = new Configuration(configuration);
- loadedConf.addResource(schedConf);
+ Configuration loadedConf = new Configuration(schedConf);
+ loadedConf.addResource(configuration);
return new CapacitySchedulerConfiguration(loadedConf, false);
}
@Override
public void mutateConfiguration(String user,
- Map<String, String> confUpdate) {
+ Map<String, String> confUpdate) throws IOException {
Configuration oldConf = new Configuration(schedConf);
LogMutation log = new LogMutation(confUpdate, user);
long id = confStore.logMutation(log);
for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
- schedConf.set(kv.getKey(), kv.getValue());
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
}
try {
rmContext.getScheduler().reinitialize(conf, rmContext);
} catch (IOException e) {
schedConf = oldConf;
confStore.confirmMutation(id, false);
- return;
+ throw e;
}
confStore.confirmMutation(id, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e160fc1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index acfb2b8..f7bb73d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -57,6 +57,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
+import com.google.common.base.Joiner;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -136,11 +137,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
@@ -2561,4 +2564,170 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
app.getApplicationTimeouts().get(appTimeout.getTimeoutType()));
return Response.status(Status.OK).entity(timeout).build();
}
+
+ @PUT
+ @Path("/queues")
+ @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response updateSchedulerConfiguration(QueueConfigsUpdateInfo
+ mutationInfo, @Context HttpServletRequest hsr)
+ throws AuthorizationException, InterruptedException {
+ init();
+
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ ApplicationACLsManager aclsManager = rm.getApplicationACLsManager();
+ if (aclsManager.areACLsEnabled()) {
+ if (callerUGI == null || !aclsManager.isAdmin(callerUGI)) {
+ String msg = "Only admins can carry out this operation.";
+ throw new ForbiddenException(msg);
+ }
+ }
+
+ ResourceScheduler scheduler = rm.getResourceScheduler();
+ if (scheduler instanceof MutableConfScheduler) {
+ try {
+ callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException, YarnException {
+ Map<String, String> confUpdate =
+ constructKeyValueConfUpdate(mutationInfo);
+ ((CapacityScheduler) scheduler).updateConfiguration(callerUGI,
+ confUpdate);
+ return null;
+ }
+ });
+ } catch (IOException e) {
+ return Response.status(Status.BAD_REQUEST).entity(e.getMessage())
+ .build();
+ }
+ return Response.status(Status.OK).entity("Configuration change " +
+ "successfully applied.").build();
+ } else {
+ return Response.status(Status.BAD_REQUEST)
+ .entity("Configuration change only supported by CapacityScheduler.")
+ .build();
+ }
+ }
+
+ private Map<String, String> constructKeyValueConfUpdate(
+ QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ CapacitySchedulerConfiguration currentConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ CapacitySchedulerConfiguration proposedConf =
+ new CapacitySchedulerConfiguration(currentConf, false);
+ Map<String, String> confUpdate = new HashMap<>();
+ for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
+ removeQueue(queueToRemove, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
+ addQueue(addQueueInfo, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
+ updateQueue(updateQueueInfo, proposedConf, confUpdate);
+ }
+ return confUpdate;
+ }
+
+ private void removeQueue(
+ String queueToRemove, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (queueToRemove == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ String queueName = queueToRemove.substring(
+ queueToRemove.lastIndexOf('.') + 1);
+ CSQueue queue = cs.getQueue(queueName);
+ if (queue == null ||
+ !queue.getQueuePath().equals(queueToRemove)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+ } else if (queueToRemove.lastIndexOf('.') == -1) {
+ throw new IOException("Can't remove queue " + queueToRemove);
+ }
+ String parentQueuePath = queueToRemove.substring(0, queueToRemove
+ .lastIndexOf('.'));
+ String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
+ List<String> newSiblingQueues = new ArrayList<>();
+ for (String siblingQueue : siblingQueues) {
+ if (!siblingQueue.equals(queueName)) {
+ newSiblingQueues.add(siblingQueue);
+ }
+ }
+ proposedConf.setQueues(parentQueuePath, newSiblingQueues
+ .toArray(new String[0]));
+ String queuesConfig = CapacitySchedulerConfiguration.PREFIX +
+ parentQueuePath + CapacitySchedulerConfiguration.DOT +
+ CapacitySchedulerConfiguration.QUEUES;
+ if (newSiblingQueues.size() == 0) {
+ confUpdate.put(queuesConfig, null);
+ } else {
+ confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
+ }
+ for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+ ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
+ .entrySet()) {
+ proposedConf.unset(confRemove.getKey());
+ confUpdate.put(confRemove.getKey(), null);
+ }
+ }
+ }
+
+ private void addQueue(
+ QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (addInfo == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ String queuePath = addInfo.getQueue();
+ String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
+ if (cs.getQueue(queueName) != null) {
+ throw new IOException("Can't add existing queue " + queuePath);
+ } else if (queuePath.lastIndexOf('.') == -1) {
+ throw new IOException("Can't add invalid queue " + queuePath);
+ }
+ String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+ String[] siblings = proposedConf.getQueues(parentQueue);
+ List<String> siblingQueues = siblings == null ? new ArrayList<>() :
+ new ArrayList<>(Arrays.<String>asList(siblings));
+ siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
+ proposedConf.setQueues(parentQueue,
+ siblingQueues.toArray(new String[0]));
+ confUpdate.put(CapacitySchedulerConfiguration.PREFIX +
+ parentQueue + CapacitySchedulerConfiguration.DOT +
+ CapacitySchedulerConfiguration.QUEUES,
+ Joiner.on(',').join(siblingQueues));
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
+ queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
+
+ private void updateQueue(QueueConfigInfo updateInfo,
+ CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) {
+ if (updateInfo == null) {
+ return;
+ } else {
+ String queuePath = updateInfo.getQueue();
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
+ queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e160fc1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
new file mode 100644
index 0000000..b20eda6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for adding or updating a queue to scheduler configuration
+ * for this queue.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueConfigInfo {
+
+ @XmlElement(name = "queueName")
+ private String queue;
+
+ private HashMap<String, String> params = new HashMap<>();
+
+ public QueueConfigInfo() { }
+
+ public QueueConfigInfo(String queue, Map<String, String> params) {
+ this.queue = queue;
+ this.params = new HashMap<>(params);
+ }
+
+ public String getQueue() {
+ return this.queue;
+ }
+
+ public HashMap<String, String> getParams() {
+ return this.params;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e160fc1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
new file mode 100644
index 0000000..644ec90
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for making scheduler configuration changes (supports adding,
+ * removing, or updating a queue).
+ */
+@XmlRootElement(name = "schedConf")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueConfigsUpdateInfo {
+
+ @XmlElement(name = "add")
+ private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "remove")
+ private ArrayList<String> removeQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "update")
+ private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
+
+ public QueueConfigsUpdateInfo() {
+ // JAXB needs this
+ }
+
+ public ArrayList<QueueConfigInfo> getAddQueueInfo() {
+ return addQueueInfo;
+ }
+
+ public ArrayList<String> getRemoveQueueInfo() {
+ return removeQueueInfo;
+ }
+
+ public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
+ return updateQueueInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e160fc1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 3f103b1..254da31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -77,7 +77,11 @@ public class TestMutableCSConfigurationProvider {
assertNull(confProvider.loadConfiguration(conf).get("badKey"));
doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
- confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ try {
+ confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ } catch (IOException e) {
+ // Expected exception.
+ }
assertNull(confProvider.loadConfiguration(conf).get("badKey"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e160fc1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
new file mode 100644
index 0000000..d149055
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -0,0 +1,477 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.Status;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Test scheduler configuration mutation via REST API.
+ */
+public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
+
+ private static final File CONF_FILE = new File(new File("target",
+ "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
+ private static final File OLD_CONF_FILE = new File(new File("target",
+ "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE + ".tmp");
+
+ private static MockRM rm;
+ private static String userName;
+ private static CapacitySchedulerConfiguration csConf;
+ private static YarnConfiguration conf;
+
+ private static class WebServletModule extends ServletModule {
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ try {
+ userName = UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException ioe) {
+ throw new RuntimeException("Unable to get current user name "
+ + ioe.getMessage(), ioe);
+ }
+ csConf = new CapacitySchedulerConfiguration(new Configuration(false),
+ false);
+ setupQueueConfiguration(csConf);
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName);
+ try {
+ if (CONF_FILE.exists()) {
+ if (!CONF_FILE.renameTo(OLD_CONF_FILE)) {
+ throw new RuntimeException("Failed to rename conf file");
+ }
+ }
+ FileOutputStream out = new FileOutputStream(CONF_FILE);
+ csConf.writeXml(out);
+ out.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to write XML file", e);
+ }
+ rm = new MockRM(conf);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ filter("/*").through(TestRMWebServicesAppsModification
+ .TestRMCustomAuthFilter.class);
+ }
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ GuiceServletConfig.setInjector(
+ Guice.createInjector(new WebServletModule()));
+ }
+
+ private static void setupQueueConfiguration(
+ CapacitySchedulerConfiguration config) {
+ config.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{"a", "b", "c"});
+
+ final String a = CapacitySchedulerConfiguration.ROOT + ".a";
+ config.setCapacity(a, 25f);
+ config.setMaximumCapacity(a, 50f);
+
+ final String a1 = a + ".a1";
+ final String a2 = a + ".a2";
+ config.setQueues(a, new String[]{"a1", "a2"});
+ config.setCapacity(a1, 100f);
+ config.setCapacity(a2, 0f);
+
+ final String b = CapacitySchedulerConfiguration.ROOT + ".b";
+ config.setCapacity(b, 75f);
+
+ final String c = CapacitySchedulerConfiguration.ROOT + ".c";
+ config.setCapacity(c, 0f);
+
+ final String c1 = c + ".c1";
+ config.setQueues(c, new String[] {"c1"});
+ config.setCapacity(c1, 0f);
+ }
+
+ public TestRMWebServicesConfigurationMutation() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testAddNestedQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Add parent queue root.d with two children d1 and d2.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> d1Capacity = new HashMap<>();
+ d1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
+ d1Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "25");
+ Map<String, String> nearEmptyCapacity = new HashMap<>();
+ nearEmptyCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "1E-4");
+ nearEmptyCapacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY,
+ "1E-4");
+ Map<String, String> d2Capacity = new HashMap<>();
+ d2Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "75");
+ d2Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "75");
+ QueueConfigInfo d1 = new QueueConfigInfo("root.d.d1", d1Capacity);
+ QueueConfigInfo d2 = new QueueConfigInfo("root.d.d2", d2Capacity);
+ QueueConfigInfo d = new QueueConfigInfo("root.d", nearEmptyCapacity);
+ updateInfo.getAddQueueInfo().add(d1);
+ updateInfo.getAddQueueInfo().add(d2);
+ updateInfo.getAddQueueInfo().add(d);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(4, newCSConf.getQueues("root").length);
+ assertEquals(2, newCSConf.getQueues("root.d").length);
+ assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d1"),
+ 0.01f);
+ assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"),
+ 0.01f);
+ }
+
+ @Test
+ public void testAddWithUpdate() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Add root.d with capacity 25, reducing root.b capacity from 75 to 50.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> dCapacity = new HashMap<>();
+ dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
+ Map<String, String> bCapacity = new HashMap<>();
+ bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "50");
+ QueueConfigInfo d = new QueueConfigInfo("root.d", dCapacity);
+ QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity);
+ updateInfo.getAddQueueInfo().add(d);
+ updateInfo.getUpdateQueueInfo().add(b);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(4, newCSConf.getQueues("root").length);
+ assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d"), 0.01f);
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
+ }
+
+ @Test
+ public void testRemoveQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.a.a2");
+ // Remove root.a.a2
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.a.a2");
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(1, newCSConf.getQueues("root.a").length);
+ assertEquals("a1", newCSConf.getQueues("root.a")[0]);
+ }
+
+ @Test
+ public void testRemoveParentQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.c", "root.c.c1");
+ // Remove root.c (parent queue)
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.c");
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(2, newCSConf.getQueues("root").length);
+ assertNull(newCSConf.getQueues("root.c"));
+ }
+
+ @Test
+ public void testRemoveParentQueueWithCapacity() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.a", "root.a.a1", "root.a.a2");
+ // Remove root.a (parent queue) with capacity 25
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.a");
+
+ // Set root.b capacity to 100
+ Map<String, String> bCapacity = new HashMap<>();
+ bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100");
+ QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity);
+ updateInfo.getUpdateQueueInfo().add(b);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(2, newCSConf.getQueues("root").length);
+ assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity("root.b"),
+ 0.01f);
+ }
+
+ @Test
+ public void testRemoveMultipleQueues() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.b", "root.c", "root.c.c1");
+ // Remove root.b and root.c
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.b");
+ updateInfo.getRemoveQueueInfo().add("root.c");
+ Map<String, String> aCapacity = new HashMap<>();
+ aCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100");
+ aCapacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "100");
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.a", aCapacity);
+ updateInfo.getUpdateQueueInfo().add(configInfo);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(1, newCSConf.getQueues("root").length);
+ }
+
+ private void stopQueue(String... queuePaths) throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Set state of queues to STOPPED.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> stoppedParam = new HashMap<>();
+ stoppedParam.put(CapacitySchedulerConfiguration.STATE,
+ QueueState.STOPPED.toString());
+ for (String queue : queuePaths) {
+ QueueConfigInfo stoppedInfo = new QueueConfigInfo(queue, stoppedParam);
+ updateInfo.getUpdateQueueInfo().add(stoppedInfo);
+ }
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ for (String queue : queuePaths) {
+ assertEquals(QueueState.STOPPED, newCSConf.getState(queue));
+ }
+ }
+
+ @Test
+ public void testUpdateQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Update config value.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
+ "0.2");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
+ cs.getConfiguration()
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"),
+ 0.001f);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
+ assertEquals(0.2f, newCSConf
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"), 0.001f);
+
+ // Remove config. Config value should be reverted to default.
+ updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
+ null);
+ aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ updateInfo.getUpdateQueueInfo().clear();
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ newCSConf = cs.getConfiguration();
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, newCSConf
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"),
+ 0.001f);
+ }
+
+ @Test
+ public void testUpdateQueueCapacity() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Update root.a and root.b capacity to 50.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ QueueConfigInfo bUpdateInfo = new QueueConfigInfo("root.b", updateParam);
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ updateInfo.getUpdateQueueInfo().add(bUpdateInfo);
+
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.a"), 0.01f);
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ if (rm != null) {
+ rm.stop();
+ }
+ CONF_FILE.delete();
+ if (!OLD_CONF_FILE.renameTo(CONF_FILE)) {
+ throw new RuntimeException("Failed to re-copy old configuration file");
+ }
+ super.tearDown();
+ }
+
+ @SuppressWarnings("rawtypes")
+ private String toJson(Object nsli, Class klass) throws Exception {
+ StringWriter sw = new StringWriter();
+ JSONJAXBContext ctx = new JSONJAXBContext(klass);
+ JSONMarshaller jm = ctx.createJSONMarshaller();
+ jm.marshallToJSON(nsli, sw);
+ return sw.toString();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/29] hadoop git commit: YARN-6679. Reduce Resource instance
overhead via non-PBImpl (Contributed by Daryn Sharp via Daniel Templeton)
Posted by jh...@apache.org.
YARN-6679. Reduce Resource instance overhead via non-PBImpl
(Contributed by Daryn Sharp via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a062374c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a062374c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a062374c
Branch: refs/heads/YARN-5734
Commit: a062374c396ba2bddbbe4d3806cc2896293b4e0b
Parents: 5672ae7
Author: Daniel Templeton <te...@apache.org>
Authored: Thu Jun 8 10:08:08 2017 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Thu Jun 8 10:08:08 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/api/records/Resource.java | 53 ++++++++++++++++----
.../impl/pb/AllocateResponsePBImpl.java | 2 +-
.../pb/GetNewApplicationResponsePBImpl.java | 9 ++--
...RegisterApplicationMasterResponsePBImpl.java | 2 +-
.../ApplicationResourceUsageReportPBImpl.java | 14 ++----
.../pb/ApplicationSubmissionContextPBImpl.java | 6 +--
.../api/records/impl/pb/ContainerPBImpl.java | 6 +--
.../records/impl/pb/ContainerReportPBImpl.java | 6 +--
.../records/impl/pb/ContainerStatusPBImpl.java | 2 +-
.../api/records/impl/pb/NodeReportPBImpl.java | 10 ++--
.../yarn/api/records/impl/pb/ProtoUtils.java | 2 +-
.../impl/pb/ReservationRequestPBImpl.java | 2 +-
.../pb/ResourceAllocationRequestPBImpl.java | 2 +-
.../records/impl/pb/ResourceOptionPBImpl.java | 2 +-
.../api/records/impl/pb/ResourcePBImpl.java | 26 +++++-----
.../records/impl/pb/ResourceRequestPBImpl.java | 2 +-
.../yarn/security/ContainerTokenIdentifier.java | 2 +-
.../hadoop/yarn/util/resource/Resources.java | 11 +---
.../impl/pb/ContainerStartDataPBImpl.java | 7 ++-
.../impl/pb/NMContainerStatusPBImpl.java | 6 +--
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 2 +-
.../pb/RegisterNodeManagerRequestPBImpl.java | 3 +-
.../pb/RegisterNodeManagerResponsePBImpl.java | 3 +-
.../protocolrecords/TestProtocolRecords.java | 20 ++++++++
.../recovery/NMLeveldbStateStoreService.java | 3 +-
.../server/ContainerTokenIdentifierForTest.java | 5 +-
26 files changed, 122 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 0d31b6f..0fd41a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
-import org.apache.hadoop.yarn.util.Records;
/**
@@ -53,22 +52,49 @@ import org.apache.hadoop.yarn.util.Records;
@Stable
public abstract class Resource implements Comparable<Resource> {
+ private static class SimpleResource extends Resource {
+ private long memory;
+ private long vcores;
+ SimpleResource(long memory, long vcores) {
+ this.memory = memory;
+ this.vcores = vcores;
+ }
+ @Override
+ public int getMemory() {
+ return (int)memory;
+ }
+ @Override
+ public void setMemory(int memory) {
+ this.memory = memory;
+ }
+ @Override
+ public long getMemorySize() {
+ return memory;
+ }
+ @Override
+ public void setMemorySize(long memory) {
+ this.memory = memory;
+ }
+ @Override
+ public int getVirtualCores() {
+ return (int)vcores;
+ }
+ @Override
+ public void setVirtualCores(int vcores) {
+ this.vcores = vcores;
+ }
+ }
+
@Public
@Stable
public static Resource newInstance(int memory, int vCores) {
- Resource resource = Records.newRecord(Resource.class);
- resource.setMemorySize(memory);
- resource.setVirtualCores(vCores);
- return resource;
+ return new SimpleResource(memory, vCores);
}
@Public
@Stable
public static Resource newInstance(long memory, int vCores) {
- Resource resource = Records.newRecord(Resource.class);
- resource.setMemorySize(memory);
- resource.setVirtualCores(vCores);
- return resource;
+ return new SimpleResource(memory, vCores);
}
/**
@@ -168,6 +194,15 @@ public abstract class Resource implements Comparable<Resource> {
}
@Override
+ public int compareTo(Resource other) {
+ long diff = this.getMemorySize() - other.getMemorySize();
+ if (diff == 0) {
+ diff = this.getVirtualCores() - other.getVirtualCores();
+ }
+ return diff == 0 ? 0 : (diff > 0 ? 1 : -1);
+ }
+
+ @Override
public String toString() {
return "<memory:" + getMemorySize() + ", vCores:" + getVirtualCores() + ">";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
index de3fc47..1455d6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
@@ -753,7 +753,7 @@ public class AllocateResponsePBImpl extends AllocateResponse {
}
private synchronized ResourceProto convertToProtoFormat(Resource r) {
- return ((ResourcePBImpl) r).getProto();
+ return ProtoUtils.convertToProtoFormat(r);
}
private synchronized PreemptionMessagePBImpl convertFromProtoFormat(PreemptionMessageProto p) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewApplicationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewApplicationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewApplicationResponsePBImpl.java
index eb8ca2c..155e756 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewApplicationResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewApplicationResponsePBImpl.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
@@ -158,13 +159,13 @@ public class GetNewApplicationResponsePBImpl extends GetNewApplicationResponse {
private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
return ((ApplicationIdPBImpl)t).getProto();
}
-
+
private Resource convertFromProtoFormat(ResourceProto resource) {
- return new ResourcePBImpl(resource);
+ return new ResourcePBImpl(resource);
}
private ResourceProto convertToProtoFormat(Resource resource) {
- return ((ResourcePBImpl)resource).getProto();
+ return ProtoUtils.convertToProtoFormat(resource);
}
-}
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
index a95aadf..1a70933 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
@@ -438,7 +438,7 @@ public class RegisterApplicationMasterResponsePBImpl extends
}
private ResourceProto convertToProtoFormat(Resource resource) {
- return ((ResourcePBImpl)resource).getProto();
+ return ProtoUtils.convertToProtoFormat(resource);
}
private ContainerPBImpl convertFromProtoFormat(ContainerProto p) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
index b6a38e7..1c85e28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
@@ -79,20 +79,14 @@ extends ApplicationResourceUsageReport {
}
private void mergeLocalToBuilder() {
- if (this.usedResources != null
- && !((ResourcePBImpl) this.usedResources).getProto().equals(
- builder.getUsedResources())) {
+ if (this.usedResources != null) {
builder.setUsedResources(convertToProtoFormat(this.usedResources));
}
- if (this.reservedResources != null
- && !((ResourcePBImpl) this.reservedResources).getProto().equals(
- builder.getReservedResources())) {
+ if (this.reservedResources != null) {
builder.setReservedResources(
convertToProtoFormat(this.reservedResources));
}
- if (this.neededResources != null
- && !((ResourcePBImpl) this.neededResources).getProto().equals(
- builder.getNeededResources())) {
+ if (this.neededResources != null) {
builder.setNeededResources(convertToProtoFormat(this.neededResources));
}
}
@@ -257,7 +251,7 @@ extends ApplicationResourceUsageReport {
}
private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl)t).getProto();
+ return ProtoUtils.convertToProtoFormat(t);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
index 6e46eb6..e3dbf40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
@@ -119,9 +119,7 @@ extends ApplicationSubmissionContext {
if (this.amContainer != null) {
builder.setAmContainerSpec(convertToProtoFormat(this.amContainer));
}
- if (this.resource != null &&
- !((ResourcePBImpl) this.resource).getProto().equals(
- builder.getResource())) {
+ if (this.resource != null) {
builder.setResource(convertToProtoFormat(this.resource));
}
if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
@@ -474,7 +472,7 @@ extends ApplicationSubmissionContext {
}
private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl)t).getProto();
+ return ProtoUtils.convertToProtoFormat(t);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
index 6bf653d..b6e22d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
@@ -93,9 +93,7 @@ public class ContainerPBImpl extends Container {
builder.getNodeId())) {
builder.setNodeId(convertToProtoFormat(this.nodeId));
}
- if (this.resource != null
- && !((ResourcePBImpl) this.resource).getProto().equals(
- builder.getResource())) {
+ if (this.resource != null) {
builder.setResource(convertToProtoFormat(this.resource));
}
if (this.priority != null &&
@@ -307,7 +305,7 @@ public class ContainerPBImpl extends Container {
}
private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl)t).getProto();
+ return ProtoUtils.convertToProtoFormat(t);
}
private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
index 1f0405f..5d435da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
@@ -270,9 +270,7 @@ public class ContainerReportPBImpl extends ContainerReport {
&& !((NodeIdPBImpl) nodeId).getProto().equals(builder.getNodeId())) {
builder.setNodeId(convertToProtoFormat(this.nodeId));
}
- if (this.resource != null
- && !((ResourcePBImpl) this.resource).getProto().equals(
- builder.getResource())) {
+ if (this.resource != null) {
builder.setResource(convertToProtoFormat(this.resource));
}
if (this.priority != null
@@ -318,7 +316,7 @@ public class ContainerReportPBImpl extends ContainerReport {
}
private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl) t).getProto();
+ return ProtoUtils.convertToProtoFormat(t);
}
private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
index 219cf02..11a3d86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
@@ -339,7 +339,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
}
private ResourceProto convertToProtoFormat(Resource e) {
- return ((ResourcePBImpl)e).getProto();
+ return ProtoUtils.convertToProtoFormat(e);
}
private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
index 1b88ab0..0d205e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
@@ -253,14 +253,10 @@ public class NodeReportPBImpl extends NodeReport {
builder.getNodeId())) {
builder.setNodeId(convertToProtoFormat(this.nodeId));
}
- if (this.used != null
- && !((ResourcePBImpl) this.used).getProto().equals(
- builder.getUsed())) {
+ if (this.used != null) {
builder.setUsed(convertToProtoFormat(this.used));
}
- if (this.capability != null
- && !((ResourcePBImpl) this.capability).getProto().equals(
- builder.getCapability())) {
+ if (this.capability != null) {
builder.setCapability(convertToProtoFormat(this.capability));
}
if (this.labels != null) {
@@ -310,7 +306,7 @@ public class NodeReportPBImpl extends NodeReport {
}
private ResourceProto convertToProtoFormat(Resource r) {
- return ((ResourcePBImpl) r).getProto();
+ return ProtoUtils.convertToProtoFormat(r);
}
private ResourceUtilizationPBImpl convertFromProtoFormat(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 926c757..fa9c430 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -346,7 +346,7 @@ public class ProtoUtils {
* Resource
*/
public static synchronized ResourceProto convertToProtoFormat(Resource r) {
- return ((ResourcePBImpl) r).getProto();
+ return ResourcePBImpl.getProto(r);
}
public static Resource convertFromProtoFormat(ResourceProto resource) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestPBImpl.java
index e36b9fb..e5a9433 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationRequestPBImpl.java
@@ -140,7 +140,7 @@ public class ReservationRequestPBImpl extends ReservationRequest {
}
private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl) t).getProto();
+ return ProtoUtils.convertToProtoFormat(t);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceAllocationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceAllocationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceAllocationRequestPBImpl.java
index 737ae44..805c628 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceAllocationRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceAllocationRequestPBImpl.java
@@ -146,7 +146,7 @@ public class ResourceAllocationRequestPBImpl extends
}
private ResourceProto convertToProtoFormat(Resource p) {
- return ((ResourcePBImpl)p).getProto();
+ return ProtoUtils.convertToProtoFormat(p);
}
private void mergeLocalToBuilder() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
index 5a4a44e..f8e24ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
@@ -80,7 +80,7 @@ public class ResourceOptionPBImpl extends ResourceOption {
private ResourceProto convertToProtoFormat(
Resource resource) {
- return ((ResourcePBImpl)resource).getProto();
+ return ProtoUtils.convertToProtoFormat(resource);
}
private ResourcePBImpl convertFromProtoFormat(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 6686696..34109be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -31,7 +31,20 @@ public class ResourcePBImpl extends Resource {
ResourceProto proto = ResourceProto.getDefaultInstance();
ResourceProto.Builder builder = null;
boolean viaProto = false;
-
+
+ // call via ProtoUtils.convertToProtoFormat(Resource)
+ static ResourceProto getProto(Resource r) {
+ final ResourcePBImpl pb;
+ if (r instanceof ResourcePBImpl) {
+ pb = (ResourcePBImpl)r;
+ } else {
+ pb = new ResourcePBImpl();
+ pb.setMemorySize(r.getMemorySize());
+ pb.setVirtualCores(r.getVirtualCores());
+ }
+ return pb.getProto();
+ }
+
public ResourcePBImpl() {
builder = ResourceProto.newBuilder();
}
@@ -89,15 +102,4 @@ public class ResourcePBImpl extends Resource {
maybeInitBuilder();
builder.setVirtualCores(vCores);
}
-
- @Override
- public int compareTo(Resource other) {
- long diff = this.getMemorySize() - other.getMemorySize();
- if (diff == 0) {
- diff = this.getVirtualCores() - other.getVirtualCores();
- }
- return diff == 0 ? 0 : (diff > 0 ? 1 : -1);
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index 9890296..eba5395 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -217,7 +217,7 @@ public class ResourceRequestPBImpl extends ResourceRequest {
}
private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl)t).getProto();
+ return ProtoUtils.convertToProtoFormat(t);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index 4b34998..07917ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -132,7 +132,7 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
builder.setNmHostAddr(hostName);
builder.setAppSubmitter(appSubmitter);
if (r != null) {
- builder.setResource(((ResourcePBImpl)r).getProto());
+ builder.setResource(ProtoUtils.convertToProtoFormat(r));
}
builder.setExpiryTimeStamp(expiryTimeStamp);
builder.setMasterKeyId(masterKeyId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 68db194..b2bb099 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.util.resource;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.Records;
@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
@Unstable
@@ -123,10 +122,7 @@ public class Resources {
}
public static Resource createResource(int memory, int cores) {
- Resource resource = Records.newRecord(Resource.class);
- resource.setMemorySize(memory);
- resource.setVirtualCores(cores);
- return resource;
+ return Resource.newInstance(memory, cores);
}
public static Resource createResource(long memory) {
@@ -134,10 +130,7 @@ public class Resources {
}
public static Resource createResource(long memory, int cores) {
- Resource resource = Records.newRecord(Resource.class);
- resource.setMemorySize(memory);
- resource.setVirtualCores(cores);
- return resource;
+ return Resource.newInstance(memory, cores);
}
public static Resource none() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
index 6d248b2..7b7e965 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProto;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProtoOrBuilder;
@@ -189,9 +190,7 @@ public class ContainerStartDataPBImpl extends ContainerStartData {
builder.getContainerId())) {
builder.setContainerId(convertToProtoFormat(this.containerId));
}
- if (this.resource != null
- && !((ResourcePBImpl) this.resource).getProto().equals(
- builder.getAllocatedResource())) {
+ if (this.resource != null) {
builder.setAllocatedResource(convertToProtoFormat(this.resource));
}
if (this.nodeId != null
@@ -232,7 +231,7 @@ public class ContainerStartDataPBImpl extends ContainerStartData {
}
private ResourceProto convertToProtoFormat(Resource resource) {
- return ((ResourcePBImpl) resource).getProto();
+ return ProtoUtils.convertToProtoFormat(resource);
}
private ResourcePBImpl convertFromProtoFormat(ResourceProto resource) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
index 2d1046f..2380391 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
@@ -256,9 +256,7 @@ public class NMContainerStatusPBImpl extends NMContainerStatus {
builder.setContainerId(convertToProtoFormat(this.containerId));
}
- if (this.resource != null
- && !((ResourcePBImpl) this.resource).getProto().equals(
- builder.getResource())) {
+ if (this.resource != null) {
builder.setResource(convertToProtoFormat(this.resource));
}
@@ -295,7 +293,7 @@ public class NMContainerStatusPBImpl extends NMContainerStatus {
}
private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl) t).getProto();
+ return ProtoUtils.convertToProtoFormat(t);
}
private ContainerStateProto
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index 37bf570..51c1a78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -646,7 +646,7 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse {
}
private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl)t).getProto();
+ return ProtoUtils.convertToProtoFormat(t);
}
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
index 0291e0b..eda06d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
@@ -387,7 +388,7 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
}
private static ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl)t).getProto();
+ return ProtoUtils.convertToProtoFormat(t);
}
private static NMContainerStatusPBImpl convertFromProtoFormat(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
index 2121fd1..6321309 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
@@ -253,7 +254,7 @@ public class RegisterNodeManagerResponsePBImpl
}
private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl)t).getProto();
+ return ProtoUtils.convertToProtoFormat(t);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
index b9cde24..74f19e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
@@ -40,6 +40,9 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NMContainerStatusPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
@@ -56,6 +59,23 @@ import org.junit.Test;
public class TestProtocolRecords {
@Test
+ public void testResource() {
+ final long mem = 123;
+ final int vcores = 456;
+ final Resource r = Resource.newInstance(mem, vcores);
+ // should be a lightweight SimpleResource which is a private inner class
+ // so just verify it's not the heavyweight pb impl.
+ Assert.assertFalse(r instanceof ResourcePBImpl);
+ Assert.assertEquals(mem, r.getMemorySize());
+ Assert.assertEquals(vcores, r.getVirtualCores());
+
+ ResourceProto proto = ProtoUtils.convertToProtoFormat(r);
+ Assert.assertEquals(mem, proto.getMemory());
+ Assert.assertEquals(vcores, proto.getVirtualCores());
+ Assert.assertEquals(r, ProtoUtils.convertFromProtoFormat(proto));
+ }
+
+ @Test
public void testNMContainerStatus() {
ApplicationId appId = ApplicationId.newInstance(123456789, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index ab23456..d1b9d65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
@@ -386,7 +387,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
try {
// New value will overwrite old values for the same key
batch.put(bytes(keyResChng),
- ((ResourcePBImpl) capability).getProto().toByteArray());
+ ProtoUtils.convertToProtoFormat(capability).toByteArray());
batch.put(bytes(keyVersion), bytes(Integer.toString(containerVersion)));
db.write(batch);
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a062374c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
index 7e8004d..99b3be5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.LogAggregationContextPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationContextProto;
@@ -60,7 +61,7 @@ public class ContainerTokenIdentifierForTest extends ContainerTokenIdentifier {
builder.setNmHostAddr(hostName);
builder.setAppSubmitter(appSubmitter);
if (r != null) {
- builder.setResource(((ResourcePBImpl)r).getProto());
+ builder.setResource(ProtoUtils.convertToProtoFormat(r));
}
builder.setExpiryTimeStamp(expiryTimeStamp);
builder.setMasterKeyId(masterKeyId);
@@ -91,7 +92,7 @@ public class ContainerTokenIdentifierForTest extends ContainerTokenIdentifier {
ResourcePBImpl resource = (ResourcePBImpl)identifier.getResource();
if (resource != null) {
- builder.setResource(resource.getProto());
+ builder.setResource(ProtoUtils.convertToProtoFormat(resource));
}
builder.setExpiryTimeStamp(identifier.getExpiryTimeStamp());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/29] hadoop git commit: HDFS-11945. Internal lease recovery may
not be retried for a long time. Contributed by Kihwal Lee
Posted by jh...@apache.org.
HDFS-11945. Internal lease recovery may not be retried for a long time. Contributed by Kihwal Lee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a33c9d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a33c9d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a33c9d5
Branch: refs/heads/YARN-5734
Commit: 1a33c9d58927186c2f219a5ecb5f1573801823ad
Parents: 164c222
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Jun 8 14:36:47 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Jun 8 14:36:47 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/LeaseManager.java | 27 +++++++++++++++++---
.../apache/hadoop/hdfs/TestLeaseRecovery2.java | 12 +++++----
.../hdfs/server/namenode/TestFileTruncate.java | 2 +-
.../hdfs/server/namenode/TestLeaseManager.java | 14 ++++++++++
4 files changed, 46 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a33c9d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index dcdb958..8695d63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -79,6 +79,8 @@ public class LeaseManager {
private long hardLimit = HdfsConstants.LEASE_HARDLIMIT_PERIOD;
static final int INODE_FILTER_WORKER_COUNT_MAX = 4;
static final int INODE_FILTER_WORKER_TASK_MIN = 512;
+ private long lastHolderUpdateTime;
+ private String internalLeaseHolder;
// Used for handling lock-leases
// Mapping: leaseHolder -> Lease
@@ -97,7 +99,26 @@ public class LeaseManager {
private Daemon lmthread;
private volatile boolean shouldRunMonitor;
- LeaseManager(FSNamesystem fsnamesystem) {this.fsnamesystem = fsnamesystem;}
+ LeaseManager(FSNamesystem fsnamesystem) {
+ this.fsnamesystem = fsnamesystem;
+ updateInternalLeaseHolder();
+ }
+
+ // Update the internal lease holder with the current time stamp.
+ private void updateInternalLeaseHolder() {
+ this.lastHolderUpdateTime = Time.monotonicNow();
+ this.internalLeaseHolder = HdfsServerConstants.NAMENODE_LEASE_HOLDER +
+ "-" + Time.formatTime(Time.now());
+ }
+
+ // Get the current internal lease holder name.
+ String getInternalLeaseHolder() {
+ long elapsed = Time.monotonicNow() - lastHolderUpdateTime;
+ if (elapsed > hardLimit) {
+ updateInternalLeaseHolder();
+ }
+ return internalLeaseHolder;
+ }
Lease getLease(String holder) {
return leases.get(holder);
@@ -467,6 +488,7 @@ public class LeaseManager {
Long[] leaseINodeIds = files.toArray(new Long[files.size()]);
FSDirectory fsd = fsnamesystem.getFSDirectory();
String p = null;
+ String newHolder = getInternalLeaseHolder();
for(Long id : leaseINodeIds) {
try {
INodesInPath iip = INodesInPath.fromINode(fsd.getInode(id));
@@ -478,8 +500,7 @@ public class LeaseManager {
boolean completed = false;
try {
completed = fsnamesystem.internalReleaseLease(
- leaseToCheck, p, iip,
- HdfsServerConstants.NAMENODE_LEASE_HOLDER);
+ leaseToCheck, p, iip, newHolder);
} catch (IOException e) {
LOG.warn("Cannot release the path " + p + " in the lease "
+ leaseToCheck + ". It will be retried.", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a33c9d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
index e8cd476..a807bbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
@@ -465,7 +465,8 @@ public class TestLeaseRecovery2 {
cluster.getNameNode(), fileStr);
assertFalse("original lease holder should not be the NN",
- originalLeaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER));
+ originalLeaseHolder.startsWith(
+ HdfsServerConstants.NAMENODE_LEASE_HOLDER));
// hflush file
AppendTestUtil.LOG.info("hflush");
@@ -501,8 +502,9 @@ public class TestLeaseRecovery2 {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
- return HdfsServerConstants.NAMENODE_LEASE_HOLDER.equals(
- NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), path));
+ String holder =
+ NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), path);
+ return holder.startsWith(HdfsServerConstants.NAMENODE_LEASE_HOLDER);
}
}, (int)SHORT_LEASE_PERIOD, (int)SHORT_LEASE_PERIOD * 10);
@@ -563,8 +565,8 @@ public class TestLeaseRecovery2 {
if (size == 0) {
assertEquals("lease holder should null, file is closed", null, holder);
} else {
- assertEquals("lease holder should now be the NN",
- HdfsServerConstants.NAMENODE_LEASE_HOLDER, holder);
+ assertTrue("lease holder should now be the NN",
+ holder.startsWith(HdfsServerConstants.NAMENODE_LEASE_HOLDER));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a33c9d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index 0ea587c..d4215e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -642,7 +642,7 @@ public class TestFileTruncate {
String leaseHolder =
NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(),
p.toUri().getPath());
- if(leaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER)) {
+ if(leaseHolder.startsWith(HdfsServerConstants.NAMENODE_LEASE_HOLDER)) {
recoveryTriggered = true;
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a33c9d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
index 9adb071..74752f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import com.google.common.collect.Lists;
@@ -99,6 +100,19 @@ public class TestLeaseManager {
assertTrue(lm.countLease() < numLease);
}
+ /**
+ * Test whether the internal lease holder name is updated properly.
+ */
+ @Test
+ public void testInternalLeaseHolder() throws Exception {
+ LeaseManager lm = new LeaseManager(makeMockFsNameSystem());
+ // Set the hard lease limit to 500ms.
+ lm.setLeasePeriod(100L, 500L);
+ String holder = lm.getInternalLeaseHolder();
+ Thread.sleep(1000);
+ assertNotEquals(holder, lm.getInternalLeaseHolder());
+ }
+
@Test
public void testCountPath() {
LeaseManager lm = new LeaseManager(makeMockFsNameSystem());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/29] hadoop git commit: YARN-5951. Changes to allow
CapacityScheduler to use configuration store
Posted by jh...@apache.org.
YARN-5951. Changes to allow CapacityScheduler to use configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c3cedd7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c3cedd7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c3cedd7c
Branch: refs/heads/YARN-5734
Commit: c3cedd7cc097597fa4b3a4a081a82adb2ab5a8bc
Parents: 7c9694c
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Mon Jan 30 19:03:48 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Thu Jun 8 20:03:58 2017 -0700
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 36 +++++------
.../CapacitySchedulerConfiguration.java | 10 +++
.../capacity/conf/CSConfigurationProvider.java | 46 ++++++++++++++
.../conf/FileBasedCSConfigurationProvider.java | 67 ++++++++++++++++++++
.../scheduler/capacity/conf/package-info.java | 29 +++++++++
.../capacity/TestCapacityScheduler.java | 4 +-
6 files changed, 170 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3cedd7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index d3186da..a925904 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
@@ -104,6 +103,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation;
@@ -162,6 +163,7 @@ public class CapacityScheduler extends
private int offswitchPerHeartbeatLimit;
+ private CSConfigurationProvider csConfProvider;
@Override
public void setConf(Configuration conf) {
@@ -285,7 +287,18 @@ public class CapacityScheduler extends
IOException {
try {
writeLock.lock();
- this.conf = loadCapacitySchedulerConfiguration(configuration);
+ String confProviderStr = configuration.get(
+ CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
+ if (confProviderStr.equals(
+ CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) {
+ this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext);
+ } else {
+ throw new IOException("Invalid CS configuration provider: " +
+ confProviderStr);
+ }
+ this.csConfProvider.init(configuration);
+ this.conf = this.csConfProvider.loadConfiguration(configuration);
validateConf(this.conf);
this.minimumAllocation = this.conf.getMinimumAllocation();
initMaximumResourceCapability(this.conf.getMaximumAllocation());
@@ -392,7 +405,7 @@ public class CapacityScheduler extends
writeLock.lock();
Configuration configuration = new Configuration(newConf);
CapacitySchedulerConfiguration oldConf = this.conf;
- this.conf = loadCapacitySchedulerConfiguration(configuration);
+ this.conf = csConfProvider.loadConfiguration(configuration);
validateConf(this.conf);
try {
LOG.info("Re-initializing queues...");
@@ -1754,23 +1767,6 @@ public class CapacityScheduler extends
return true;
}
- private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration(
- Configuration configuration) throws IOException {
- try {
- InputStream CSInputStream =
- this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(configuration,
- YarnConfiguration.CS_CONFIGURATION_FILE);
- if (CSInputStream != null) {
- configuration.addResource(CSInputStream);
- return new CapacitySchedulerConfiguration(configuration, false);
- }
- return new CapacitySchedulerConfiguration(configuration, true);
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
private String getDefaultReservationQueueName(String planQueueName) {
return planQueueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3cedd7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index c3c9585..6dd7aef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -292,6 +292,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final boolean DEFAULT_LAZY_PREEMPTION_ENABLED = false;
+ @Private
+ public static final String CS_CONF_PROVIDER = PREFIX
+ + "configuration.provider";
+
+ @Private
+ public static final String FILE_CS_CONF_PROVIDER = "file";
+
+ @Private
+ public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
+
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
public CapacitySchedulerConfiguration() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3cedd7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
new file mode 100644
index 0000000..c9984ac
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import java.io.IOException;
+
+/**
+ * Configuration provider for {@link CapacityScheduler}.
+ */
+public interface CSConfigurationProvider {
+
+ /**
+ * Initialize the configuration provider with given conf.
+ * @param conf configuration to initialize with
+ */
+ void init(Configuration conf);
+
+ /**
+ * Loads capacity scheduler configuration object.
+ * @param conf initial bootstrap configuration
+ * @return CS configuration
+ * @throws IOException if fail to retrieve configuration
+ */
+ CapacitySchedulerConfiguration loadConfiguration(Configuration conf)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3cedd7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
new file mode 100644
index 0000000..51c64fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * {@link CapacityScheduler} configuration provider based on local
+ * {@code capacity-scheduler.xml} file.
+ */
+public class FileBasedCSConfigurationProvider implements
+ CSConfigurationProvider {
+
+ private RMContext rmContext;
+
+ /**
+ * Construct file based CS configuration provider with given context.
+ * @param rmContext the RM context
+ */
+ public FileBasedCSConfigurationProvider(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void init(Configuration conf) {}
+
+ @Override
+ public CapacitySchedulerConfiguration loadConfiguration(Configuration conf)
+ throws IOException {
+ try {
+ InputStream csInputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.CS_CONFIGURATION_FILE);
+ if (csInputStream != null) {
+ conf.addResource(csInputStream);
+ return new CapacitySchedulerConfiguration(conf, false);
+ }
+ return new CapacitySchedulerConfiguration(conf, true);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3cedd7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
new file mode 100644
index 0000000..08d0522
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package
+ * org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf
+ * contains classes related to capacity scheduler configuration management.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3cedd7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index bf1f6eb..5de6c6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -231,13 +231,13 @@ public class TestCapacityScheduler {
@Test (timeout = 30000)
public void testConfValidation() throws Exception {
- ResourceScheduler scheduler = new CapacityScheduler();
+ CapacityScheduler scheduler = new CapacityScheduler();
scheduler.setRMContext(resourceManager.getRMContext());
Configuration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
try {
- scheduler.reinitialize(conf, mockContext);
+ scheduler.init(conf);
fail("Exception is expected because the min memory allocation is" +
" larger than the max memory allocation.");
} catch (YarnRuntimeException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/29] hadoop git commit: HDFS-11861.
ipc.Client.Connection#sendRpcRequest should log request name. Contributed by
John Zhuge.
Posted by jh...@apache.org.
HDFS-11861. ipc.Client.Connection#sendRpcRequest should log request name. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5672ae7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5672ae7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5672ae7b
Branch: refs/heads/YARN-5734
Commit: 5672ae7b37ce75086a1cb5bb9a388288fc913eb7
Parents: df35e4c
Author: John Zhuge <jz...@apache.org>
Authored: Sun May 21 00:18:35 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Wed Jun 7 15:52:52 2017 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5672ae7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index c0a5be9..6b21c75 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -1109,7 +1109,8 @@ public class Client implements AutoCloseable {
return;
}
if (LOG.isDebugEnabled()) {
- LOG.debug(getName() + " sending #" + call.id);
+ LOG.debug(getName() + " sending #" + call.id
+ + " " + call.rpcRequest);
}
// RpcRequestHeader + RpcRequest
ipcStreams.sendRequest(buf.toByteArray());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/29] hadoop git commit: HDFS-11914. Add more diagnosis info for
fsimage transfer failure. Contributed by Yongjun Zhang.
Posted by jh...@apache.org.
HDFS-11914. Add more diagnosis info for fsimage transfer failure. Contributed by Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a28a2b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a28a2b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a28a2b9
Branch: refs/heads/YARN-5734
Commit: 6a28a2b914664fbbb5267f13e82d42b3bb49610c
Parents: 5fd9742
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Mon Jun 5 16:31:03 2017 -0700
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Mon Jun 5 16:31:03 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/common/Util.java | 6 ++---
.../hdfs/server/namenode/TransferFsImage.java | 23 +++++++++++++++++---
2 files changed, 23 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a28a2b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
index e9ceeb0..5dee16a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
@@ -225,6 +225,7 @@ public final class Util {
stream = new DigestInputStream(stream, digester);
}
boolean finishedReceiving = false;
+ int num = 1;
List<FileOutputStream> outputStreams = Lists.newArrayList();
@@ -256,7 +257,6 @@ public final class Util {
}
}
- int num = 1;
byte[] buf = new byte[IO_FILE_BUFFER_SIZE];
while (num > 0) {
num = stream.read(buf);
@@ -305,8 +305,8 @@ public final class Util {
// exception that makes it look like a server-side problem!
deleteTmpFiles(localPaths);
throw new IOException("File " + url + " received length " + received +
- " is not of the advertised size " +
- advertisedSize);
+ " is not of the advertised size " + advertisedSize +
+ ". Fsimage name: " + fsImageName + " lastReceived: " + num);
}
}
xferStats.insert(0, String.format("Combined time for file download and" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a28a2b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index 7316414..771a43e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -339,6 +339,11 @@ public class TransferFsImage {
FileInputStream infile, DataTransferThrottler throttler,
Canceler canceler) throws IOException {
byte buf[] = new byte[IO_FILE_BUFFER_SIZE];
+ long total = 0;
+ int num = 1;
+ IOException ioe = null;
+ String reportStr = "Sending fileName: " + localfile.getAbsolutePath()
+ + ", fileSize: " + localfile.length() + ".";
try {
CheckpointFaultInjector.getInstance()
.aboutToSendFile(localfile);
@@ -352,7 +357,6 @@ public class TransferFsImage {
// and the rest of the image will be sent over the wire
infile.read(buf);
}
- int num = 1;
while (num > 0) {
if (canceler != null && canceler.isCancelled()) {
throw new SaveNamespaceCancelledException(
@@ -368,16 +372,29 @@ public class TransferFsImage {
LOG.warn("SIMULATING A CORRUPT BYTE IN IMAGE TRANSFER!");
buf[0]++;
}
-
+
out.write(buf, 0, num);
+ total += num;
if (throttler != null) {
throttler.throttle(num, canceler);
}
}
} catch (EofException e) {
- LOG.info("Connection closed by client");
+ reportStr += " Connection closed by client.";
+ ioe = e;
out = null; // so we don't close in the finally
+ } catch (IOException ie) {
+ ioe = ie;
+ throw ie;
} finally {
+ reportStr += " Sent total: " + total +
+ " bytes. Size of last segment intended to send: " + num
+ + " bytes.";
+ if (ioe != null) {
+ LOG.info(reportStr, ioe);
+ } else {
+ LOG.info(reportStr);
+ }
if (out != null) {
out.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/29] hadoop git commit: YARN-5949. Add pluggable configuration ACL
policy interface and implementation. (Jonathan Hung via wangda)
Posted by jh...@apache.org.
YARN-5949. Add pluggable configuration ACL policy interface and implementation. (Jonathan Hung via wangda)
Change-Id: Ib98e82ff753bede21fcab2e6ca9ec1e7a5a2008f
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bf3548f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bf3548f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bf3548f
Branch: refs/heads/YARN-5734
Commit: 8bf3548f16da0024b6309ca3e1f83881ece8ad1f
Parents: e160fc1
Author: Wangda Tan <wa...@apache.org>
Authored: Mon May 22 13:38:31 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Thu Jun 8 20:03:59 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 3 +
.../src/main/resources/yarn-default.xml | 11 ++
.../ConfigurationMutationACLPolicy.java | 47 ++++++
.../ConfigurationMutationACLPolicyFactory.java | 49 ++++++
.../DefaultConfigurationMutationACLPolicy.java | 45 ++++++
.../scheduler/MutableConfScheduler.java | 19 ++-
.../scheduler/MutableConfigurationProvider.java | 8 +-
.../scheduler/capacity/CapacityScheduler.java | 6 +-
.../conf/MutableCSConfigurationProvider.java | 151 +++++++++++++++++-
...ueueAdminConfigurationMutationACLPolicy.java | 96 ++++++++++++
.../resourcemanager/webapp/RMWebServices.java | 132 +---------------
.../TestConfigurationMutationACLPolicies.java | 154 +++++++++++++++++++
.../TestMutableCSConfigurationProvider.java | 40 +++--
13 files changed, 610 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index fa9e79d..6e0cd32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -613,6 +613,9 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_CONFIGURATION_STORE =
MEMORY_CONFIGURATION_STORE;
+ public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
+ YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
+
public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX
+ "authorization-provider";
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 02284e6..5f2769e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3120,4 +3120,15 @@
<value>memory</value>
</property>
+ <property>
+ <description>
+ The class to use for configuration mutation ACL policy if using a mutable
+ configuration provider. Controls whether a mutation request is allowed.
+ The DefaultConfigurationMutationACLPolicy checks if the requestor is a
+ YARN admin.
+ </description>
+ <name>yarn.scheduler.configuration.mutation.acl-policy.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..724487b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+/**
+ * Interface for determining whether configuration mutations are allowed.
+ */
+public interface ConfigurationMutationACLPolicy {
+
+ /**
+ * Initialize ACL policy with configuration and RMContext.
+ * @param conf Configuration to initialize with.
+ * @param rmContext rmContext
+ */
+ void init(Configuration conf, RMContext rmContext);
+
+ /**
+ * Check if mutation is allowed.
+ * @param user User issuing the request
+ * @param confUpdate configurations to be updated
+ * @return whether provided mutation is allowed or not
+ */
+ boolean isMutationAllowed(UserGroupInformation user, QueueConfigsUpdateInfo
+ confUpdate);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
new file mode 100644
index 0000000..2898785
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Factory class for creating instances of
+ * {@link ConfigurationMutationACLPolicy}.
+ */
+public final class ConfigurationMutationACLPolicyFactory {
+
+ private static final Log LOG = LogFactory.getLog(
+ ConfigurationMutationACLPolicyFactory.class);
+
+ private ConfigurationMutationACLPolicyFactory() {
+ // Unused.
+ }
+
+ public static ConfigurationMutationACLPolicy getPolicy(Configuration conf) {
+ Class<? extends ConfigurationMutationACLPolicy> policyClass =
+ conf.getClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ DefaultConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ LOG.info("Using ConfigurationMutationACLPolicy implementation - " +
+ policyClass);
+ return ReflectionUtils.newInstance(policyClass, conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..680c3b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+/**
+ * Default configuration mutation ACL policy. Checks if user is YARN admin.
+ */
+public class DefaultConfigurationMutationACLPolicy implements
+ ConfigurationMutationACLPolicy {
+
+ private YarnAuthorizationProvider authorizer;
+
+ @Override
+ public void init(Configuration conf, RMContext rmContext) {
+ authorizer = YarnAuthorizationProvider.getInstance(conf);
+ }
+
+ @Override
+ public boolean isMutationAllowed(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) {
+ return authorizer.isAdmin(user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 35e36e1..93a935e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -17,10 +17,11 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import java.io.IOException;
-import java.util.Map;
/**
* Interface for a scheduler that supports changing configuration at runtime.
@@ -31,10 +32,22 @@ public interface MutableConfScheduler extends ResourceScheduler {
/**
* Update the scheduler's configuration.
* @param user Caller of this update
- * @param confUpdate key-value map of the configuration update
+ * @param confUpdate configuration update
* @throws IOException if update is invalid
*/
void updateConfiguration(UserGroupInformation user,
- Map<String, String> confUpdate) throws IOException;
+ QueueConfigsUpdateInfo confUpdate) throws IOException;
+ /**
+ * Get the scheduler configuration.
+ * @return the scheduler configuration
+ */
+ Configuration getConfiguration();
+
+ /**
+ * Get queue object based on queue name.
+ * @param queueName the queue name
+ * @return the queue object
+ */
+ Queue getQueue(String queueName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 889c3bc..f04c128 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
import java.io.IOException;
-import java.util.Map;
/**
* Interface for allowing changing scheduler configurations.
@@ -32,7 +34,7 @@ public interface MutableConfigurationProvider {
* @param confUpdate Key-value pairs for configurations to be updated.
* @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(String user, Map<String, String> confUpdate)
- throws IOException;
+ void mutateConfiguration(UserGroupInformation user, QueueConfigsUpdateInfo
+ confUpdate) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 85138d5..a0e7391 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -136,6 +136,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -619,6 +620,7 @@ public class CapacityScheduler extends
preemptionManager.refreshQueues(null, this.getRootQueue());
}
+ @Override
public CSQueue getQueue(String queueName) {
if (queueName == null) {
return null;
@@ -2491,10 +2493,10 @@ public class CapacityScheduler extends
@Override
public void updateConfiguration(UserGroupInformation user,
- Map<String, String> confUpdate) throws IOException {
+ QueueConfigsUpdateInfo confUpdate) throws IOException {
if (csConfProvider instanceof MutableConfigurationProvider) {
((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
- user.getShortUserName(), confUpdate);
+ user, confUpdate);
} else {
throw new UnsupportedOperationException("Configured CS configuration " +
"provider does not support updating configuration.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index ea1b3c0..8b879b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -18,14 +18,27 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import com.google.common.base.Joiner;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
/**
@@ -38,6 +51,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
private Configuration schedConf;
private YarnConfigurationStore confStore;
+ private ConfigurationMutationACLPolicy aclMutationPolicy;
private RMContext rmContext;
private Configuration conf;
@@ -68,6 +82,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
schedConf.set(kv.getKey(), kv.getValue());
}
confStore.initialize(config, schedConf);
+ this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory
+ .getPolicy(config);
+ aclMutationPolicy.init(config, rmContext);
this.conf = config;
}
@@ -80,12 +97,17 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
@Override
- public void mutateConfiguration(String user,
- Map<String, String> confUpdate) throws IOException {
+ public void mutateConfiguration(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) throws IOException {
+ if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
+ throw new AccessControlException("User is not admin of all modified" +
+ " queues.");
+ }
Configuration oldConf = new Configuration(schedConf);
- LogMutation log = new LogMutation(confUpdate, user);
+ Map<String, String> kvUpdate = constructKeyValueConfUpdate(confUpdate);
+ LogMutation log = new LogMutation(kvUpdate, user.getShortUserName());
long id = confStore.logMutation(log);
- for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
+ for (Map.Entry<String, String> kv : kvUpdate.entrySet()) {
if (kv.getValue() == null) {
schedConf.unset(kv.getKey());
} else {
@@ -101,4 +123,125 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
confStore.confirmMutation(id, true);
}
+
+
+ private Map<String, String> constructKeyValueConfUpdate(
+ QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ CapacitySchedulerConfiguration proposedConf =
+ new CapacitySchedulerConfiguration(cs.getConfiguration(), false);
+ Map<String, String> confUpdate = new HashMap<>();
+ for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
+ removeQueue(queueToRemove, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
+ addQueue(addQueueInfo, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
+ updateQueue(updateQueueInfo, proposedConf, confUpdate);
+ }
+ return confUpdate;
+ }
+
+ private void removeQueue(
+ String queueToRemove, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (queueToRemove == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ String queueName = queueToRemove.substring(
+ queueToRemove.lastIndexOf('.') + 1);
+ CSQueue queue = cs.getQueue(queueName);
+ if (queue == null ||
+ !queue.getQueuePath().equals(queueToRemove)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+ } else if (queueToRemove.lastIndexOf('.') == -1) {
+ throw new IOException("Can't remove queue " + queueToRemove);
+ }
+ String parentQueuePath = queueToRemove.substring(0, queueToRemove
+ .lastIndexOf('.'));
+ String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
+ List<String> newSiblingQueues = new ArrayList<>();
+ for (String siblingQueue : siblingQueues) {
+ if (!siblingQueue.equals(queueName)) {
+ newSiblingQueues.add(siblingQueue);
+ }
+ }
+ proposedConf.setQueues(parentQueuePath, newSiblingQueues
+ .toArray(new String[0]));
+ String queuesConfig = CapacitySchedulerConfiguration.PREFIX
+ + parentQueuePath + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.QUEUES;
+ if (newSiblingQueues.size() == 0) {
+ confUpdate.put(queuesConfig, null);
+ } else {
+ confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
+ }
+ for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+ ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
+ .entrySet()) {
+ proposedConf.unset(confRemove.getKey());
+ confUpdate.put(confRemove.getKey(), null);
+ }
+ }
+ }
+
+ private void addQueue(
+ QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (addInfo == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ String queuePath = addInfo.getQueue();
+ String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
+ if (cs.getQueue(queueName) != null) {
+ throw new IOException("Can't add existing queue " + queuePath);
+ } else if (queuePath.lastIndexOf('.') == -1) {
+ throw new IOException("Can't add invalid queue " + queuePath);
+ }
+ String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+ String[] siblings = proposedConf.getQueues(parentQueue);
+ List<String> siblingQueues = siblings == null ? new ArrayList<>() :
+ new ArrayList<>(Arrays.<String>asList(siblings));
+ siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
+ proposedConf.setQueues(parentQueue,
+ siblingQueues.toArray(new String[0]));
+ confUpdate.put(CapacitySchedulerConfiguration.PREFIX
+ + parentQueue + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.QUEUES,
+ Joiner.on(',').join(siblingQueues));
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX
+ + queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
+
+ private void updateQueue(QueueConfigInfo updateInfo,
+ CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) {
+ if (updateInfo == null) {
+ return;
+ } else {
+ String queuePath = updateInfo.getQueue();
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX
+ + queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..1f94c1c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A configuration mutation ACL policy which checks that user has admin
+ * privileges on all queues they are changing.
+ */
+public class QueueAdminConfigurationMutationACLPolicy implements
+ ConfigurationMutationACLPolicy {
+
+ private RMContext rmContext;
+
+ @Override
+ public void init(Configuration conf, RMContext context) {
+ this.rmContext = context;
+ }
+
+ @Override
+ public boolean isMutationAllowed(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) {
+ Set<String> queues = new HashSet<>();
+ for (QueueConfigInfo addQueueInfo : confUpdate.getAddQueueInfo()) {
+ queues.add(addQueueInfo.getQueue());
+ }
+ for (String removeQueue : confUpdate.getRemoveQueueInfo()) {
+ queues.add(removeQueue);
+ }
+ for (QueueConfigInfo updateQueueInfo : confUpdate.getUpdateQueueInfo()) {
+ queues.add(updateQueueInfo.getQueue());
+ }
+ for (String queuePath : queues) {
+ String queueName = queuePath.lastIndexOf('.') != -1 ?
+ queuePath.substring(queuePath.lastIndexOf('.') + 1) : queuePath;
+ QueueInfo queueInfo = null;
+ try {
+ queueInfo = rmContext.getScheduler()
+ .getQueueInfo(queueName, false, false);
+ } catch (IOException e) {
+ // Queue is not found, do nothing.
+ }
+ String parentPath = queuePath;
+ // TODO: handle global config change.
+ while (queueInfo == null) {
+ // We are adding a queue (whose parent we are possibly also adding).
+ // Check ACL of lowest parent queue which already exists.
+ parentPath = parentPath.substring(0, parentPath.lastIndexOf('.'));
+ String parentName = parentPath.lastIndexOf('.') != -1 ?
+ parentPath.substring(parentPath.lastIndexOf('.') + 1) : parentPath;
+ try {
+ queueInfo = rmContext.getScheduler()
+ .getQueueInfo(parentName, false, false);
+ } catch (IOException e) {
+ // Queue is not found, do nothing.
+ }
+ }
+ Queue queue = ((MutableConfScheduler) rmContext.getScheduler())
+ .getQueue(queueInfo.getQueueName());
+ if (queue != null && !queue.hasAccess(QueueACL.ADMINISTER_QUEUE, user)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index f7bb73d..6d2df76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -57,7 +57,6 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
-import com.google.common.base.Joiner;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -143,7 +142,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
@@ -2590,10 +2588,8 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws IOException, YarnException {
- Map<String, String> confUpdate =
- constructKeyValueConfUpdate(mutationInfo);
- ((CapacityScheduler) scheduler).updateConfiguration(callerUGI,
- confUpdate);
+ ((MutableConfScheduler) scheduler).updateConfiguration(callerUGI,
+ mutationInfo);
return null;
}
});
@@ -2605,129 +2601,9 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
"successfully applied.").build();
} else {
return Response.status(Status.BAD_REQUEST)
- .entity("Configuration change only supported by CapacityScheduler.")
+ .entity("Configuration change only supported by " +
+ "MutableConfScheduler.")
.build();
}
}
-
- private Map<String, String> constructKeyValueConfUpdate(
- QueueConfigsUpdateInfo mutationInfo) throws IOException {
- CapacitySchedulerConfiguration currentConf =
- ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
- CapacitySchedulerConfiguration proposedConf =
- new CapacitySchedulerConfiguration(currentConf, false);
- Map<String, String> confUpdate = new HashMap<>();
- for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
- removeQueue(queueToRemove, proposedConf, confUpdate);
- }
- for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
- addQueue(addQueueInfo, proposedConf, confUpdate);
- }
- for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
- updateQueue(updateQueueInfo, proposedConf, confUpdate);
- }
- return confUpdate;
- }
-
- private void removeQueue(
- String queueToRemove, CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) throws IOException {
- if (queueToRemove == null) {
- return;
- } else {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- String queueName = queueToRemove.substring(
- queueToRemove.lastIndexOf('.') + 1);
- CSQueue queue = cs.getQueue(queueName);
- if (queue == null ||
- !queue.getQueuePath().equals(queueToRemove)) {
- throw new IOException("Queue " + queueToRemove + " not found");
- } else if (queueToRemove.lastIndexOf('.') == -1) {
- throw new IOException("Can't remove queue " + queueToRemove);
- }
- String parentQueuePath = queueToRemove.substring(0, queueToRemove
- .lastIndexOf('.'));
- String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
- List<String> newSiblingQueues = new ArrayList<>();
- for (String siblingQueue : siblingQueues) {
- if (!siblingQueue.equals(queueName)) {
- newSiblingQueues.add(siblingQueue);
- }
- }
- proposedConf.setQueues(parentQueuePath, newSiblingQueues
- .toArray(new String[0]));
- String queuesConfig = CapacitySchedulerConfiguration.PREFIX +
- parentQueuePath + CapacitySchedulerConfiguration.DOT +
- CapacitySchedulerConfiguration.QUEUES;
- if (newSiblingQueues.size() == 0) {
- confUpdate.put(queuesConfig, null);
- } else {
- confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
- }
- for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
- ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
- .entrySet()) {
- proposedConf.unset(confRemove.getKey());
- confUpdate.put(confRemove.getKey(), null);
- }
- }
- }
-
- private void addQueue(
- QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) throws IOException {
- if (addInfo == null) {
- return;
- } else {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- String queuePath = addInfo.getQueue();
- String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
- if (cs.getQueue(queueName) != null) {
- throw new IOException("Can't add existing queue " + queuePath);
- } else if (queuePath.lastIndexOf('.') == -1) {
- throw new IOException("Can't add invalid queue " + queuePath);
- }
- String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
- String[] siblings = proposedConf.getQueues(parentQueue);
- List<String> siblingQueues = siblings == null ? new ArrayList<>() :
- new ArrayList<>(Arrays.<String>asList(siblings));
- siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
- proposedConf.setQueues(parentQueue,
- siblingQueues.toArray(new String[0]));
- confUpdate.put(CapacitySchedulerConfiguration.PREFIX +
- parentQueue + CapacitySchedulerConfiguration.DOT +
- CapacitySchedulerConfiguration.QUEUES,
- Joiner.on(',').join(siblingQueues));
- String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
- queuePath + CapacitySchedulerConfiguration.DOT;
- for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
- if (kv.getValue() == null) {
- proposedConf.unset(keyPrefix + kv.getKey());
- } else {
- proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
- }
- confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
- }
- }
- }
-
- private void updateQueue(QueueConfigInfo updateInfo,
- CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) {
- if (updateInfo == null) {
- return;
- } else {
- String queuePath = updateInfo.getQueue();
- String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
- queuePath + CapacitySchedulerConfiguration.DOT;
- for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
- if (kv.getValue() == null) {
- proposedConf.unset(keyPrefix + kv.getKey());
- } else {
- proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
- }
- confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
new file mode 100644
index 0000000..4016dcf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestConfigurationMutationACLPolicies {
+
+ private ConfigurationMutationACLPolicy policy;
+ private RMContext rmContext;
+ private MutableConfScheduler scheduler;
+
+ private static final UserGroupInformation GOOD_USER = UserGroupInformation
+ .createUserForTesting("goodUser", new String[] {});
+ private static final UserGroupInformation BAD_USER = UserGroupInformation
+ .createUserForTesting("badUser", new String[] {});
+ private static final Map<String, String> EMPTY_MAP =
+ Collections.<String, String>emptyMap();
+
+ @Before
+ public void setUp() throws IOException {
+ rmContext = mock(RMContext.class);
+ scheduler = mock(MutableConfScheduler.class);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
+ mockQueue("a", scheduler);
+ mockQueue("b", scheduler);
+ mockQueue("b1", scheduler);
+ }
+
+ private void mockQueue(String queueName, MutableConfScheduler scheduler)
+ throws IOException {
+ QueueInfo queueInfo = QueueInfo.newInstance(queueName, 0, 0, 0, null, null,
+ null, null, null, null, false);
+ when(scheduler.getQueueInfo(eq(queueName), anyBoolean(), anyBoolean()))
+ .thenReturn(queueInfo);
+ Queue queue = mock(Queue.class);
+ when(queue.hasAccess(eq(QueueACL.ADMINISTER_QUEUE), eq(GOOD_USER)))
+ .thenReturn(true);
+ when(queue.hasAccess(eq(QueueACL.ADMINISTER_QUEUE), eq(BAD_USER)))
+ .thenReturn(false);
+ when(scheduler.getQueue(eq(queueName))).thenReturn(queue);
+ }
+ @Test
+ public void testDefaultPolicy() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, GOOD_USER.getShortUserName());
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ DefaultConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, null));
+ assertFalse(policy.isMutationAllowed(BAD_USER, null));
+ }
+
+ @Test
+ public void testQueueAdminBasedPolicy() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.a", EMPTY_MAP);
+ updateInfo.getUpdateQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyAddQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Add root.b.b1. Should check ACL of root.b queue.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2", EMPTY_MAP);
+ updateInfo.getAddQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyAddNestedQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Add root.b.b1.b11. Should check ACL of root.b queue.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2.b21", EMPTY_MAP);
+ updateInfo.getAddQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyRemoveQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Remove root.b.b1.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.b.b1");
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf3548f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 254da31..13229b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -19,8 +19,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import org.junit.Before;
import org.junit.Test;
@@ -43,22 +47,34 @@ public class TestMutableCSConfigurationProvider {
private MutableCSConfigurationProvider confProvider;
private RMContext rmContext;
- private Map<String, String> goodUpdate;
- private Map<String, String> badUpdate;
+ private QueueConfigsUpdateInfo goodUpdate;
+ private QueueConfigsUpdateInfo badUpdate;
private CapacityScheduler cs;
- private static final String TEST_USER = "testUser";
+ private static final UserGroupInformation TEST_USER = UserGroupInformation
+ .createUserForTesting("testUser", new String[] {});
@Before
public void setUp() {
cs = mock(CapacityScheduler.class);
rmContext = mock(RMContext.class);
when(rmContext.getScheduler()).thenReturn(cs);
+ when(cs.getConfiguration()).thenReturn(
+ new CapacitySchedulerConfiguration());
confProvider = new MutableCSConfigurationProvider(rmContext);
- goodUpdate = new HashMap<>();
- goodUpdate.put("goodKey", "goodVal");
- badUpdate = new HashMap<>();
- badUpdate.put("badKey", "badVal");
+ goodUpdate = new QueueConfigsUpdateInfo();
+ Map<String, String> goodUpdateMap = new HashMap<>();
+ goodUpdateMap.put("goodKey", "goodVal");
+ QueueConfigInfo goodUpdateInfo = new
+ QueueConfigInfo("root.a", goodUpdateMap);
+ goodUpdate.getUpdateQueueInfo().add(goodUpdateInfo);
+
+ badUpdate = new QueueConfigsUpdateInfo();
+ Map<String, String> badUpdateMap = new HashMap<>();
+ badUpdateMap.put("badKey", "badVal");
+ QueueConfigInfo badUpdateInfo = new
+ QueueConfigInfo("root.a", badUpdateMap);
+ badUpdate.getUpdateQueueInfo().add(badUpdateInfo);
}
@Test
@@ -66,15 +82,16 @@ public class TestMutableCSConfigurationProvider {
Configuration conf = new Configuration();
confProvider.init(conf);
assertNull(confProvider.loadConfiguration(conf)
- .get("goodKey"));
+ .get("yarn.scheduler.capacity.root.a.goodKey"));
doNothing().when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
confProvider.mutateConfiguration(TEST_USER, goodUpdate);
assertEquals("goodVal", confProvider.loadConfiguration(conf)
- .get("goodKey"));
+ .get("yarn.scheduler.capacity.root.a.goodKey"));
- assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ assertNull(confProvider.loadConfiguration(conf).get(
+ "yarn.scheduler.capacity.root.a.badKey"));
doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
try {
@@ -82,6 +99,7 @@ public class TestMutableCSConfigurationProvider {
} catch (IOException e) {
// Expected exception.
}
- assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ assertNull(confProvider.loadConfiguration(conf).get(
+ "yarn.scheduler.capacity.root.a.badKey"));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/29] hadoop git commit: MAPREDUCE-6676. NNBench should Throw
IOException when rename and delete fails. Contributed by Brahma Reddy
Battula.
Posted by jh...@apache.org.
MAPREDUCE-6676. NNBench should Throw IOException when rename and delete fails. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24181f55
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24181f55
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24181f55
Branch: refs/heads/YARN-5734
Commit: 24181f559f008fc7945702c0fad7d1028f75bfc6
Parents: 0b7bff7
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Wed Jun 7 16:34:47 2017 +0800
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Wed Jun 7 16:34:47 2017 +0800
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/hdfs/NNBench.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24181f55/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
index ee3cc00..29eac43 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
@@ -868,7 +868,10 @@ public class NNBench extends Configured implements Tool {
try {
// Set up timer for measuring AL
startTimeAL = System.currentTimeMillis();
- filesystem.rename(filePath, filePathR);
+ boolean result = filesystem.rename(filePath, filePathR);
+ if (!result) {
+ throw new IOException("rename failed for " + filePath);
+ }
totalTimeAL1 += (System.currentTimeMillis() - startTimeAL);
successfulOp = true;
@@ -901,7 +904,10 @@ public class NNBench extends Configured implements Tool {
try {
// Set up timer for measuring AL
startTimeAL = System.currentTimeMillis();
- filesystem.delete(filePath, true);
+ boolean result = filesystem.delete(filePath, true);
+ if (!result) {
+ throw new IOException("delete failed for " + filePath);
+ }
totalTimeAL1 += (System.currentTimeMillis() - startTimeAL);
successfulOp = true;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/29] hadoop git commit: HDFS-11708. Positional read will fail if
replicas moved to different DNs after stream is opened. Contributed by
Vinayakumar B.
Posted by jh...@apache.org.
HDFS-11708. Positional read will fail if replicas moved to different DNs after stream is opened. Contributed by Vinayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/70fc6746
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/70fc6746
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/70fc6746
Branch: refs/heads/YARN-5734
Commit: 70fc6746b326b9a913e8bebca5f5afaf01ab9e11
Parents: 1869e17
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jun 7 10:11:23 2017 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jun 7 10:55:53 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSInputStream.java | 38 ++---
.../hadoop/hdfs/DFSStripedInputStream.java | 2 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 46 ++++++
.../java/org/apache/hadoop/hdfs/TestPread.java | 146 +++++++++++++++++++
.../server/datanode/TestBlockReplacement.java | 40 +----
5 files changed, 221 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70fc6746/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index d388d00..77f5a92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -559,6 +559,8 @@ public class DFSInputStream extends FSInputStream
chosenNode = retval.info;
InetSocketAddress targetAddr = retval.addr;
StorageType storageType = retval.storageType;
+ // Latest block if refreshed by chooseDatanode()
+ targetBlock = retval.block;
try {
blockReader = getBlockReader(targetBlock, offsetIntoBlock,
@@ -915,7 +917,7 @@ public class DFSInputStream extends FSInputStream
chosenNode.getXferAddr(dfsClient.getConf().isConnectToDnViaHostname());
DFSClient.LOG.debug("Connecting to datanode {}", dnAddr);
InetSocketAddress targetAddr = NetUtils.createSocketAddr(dnAddr);
- return new DNAddrPair(chosenNode, targetAddr, storageType);
+ return new DNAddrPair(chosenNode, targetAddr, storageType, block);
}
/**
@@ -957,12 +959,13 @@ public class DFSInputStream extends FSInputStream
protected void fetchBlockByteRange(LocatedBlock block, long start, long end,
ByteBuffer buf, CorruptedBlocks corruptedBlocks)
throws IOException {
- block = refreshLocatedBlock(block);
while (true) {
DNAddrPair addressPair = chooseDataNode(block, null);
+ // Latest block, if refreshed internally
+ block = addressPair.block;
try {
- actualGetFromOneDataNode(addressPair, block, start, end,
- buf, corruptedBlocks);
+ actualGetFromOneDataNode(addressPair, start, end, buf,
+ corruptedBlocks);
return;
} catch (IOException e) {
checkInterrupted(e); // check if the read has been interrupted
@@ -983,8 +986,7 @@ public class DFSInputStream extends FSInputStream
public ByteBuffer call() throws Exception {
try (TraceScope ignored = dfsClient.getTracer().
newScope("hedgedRead" + hedgedReadId, parentSpanId)) {
- actualGetFromOneDataNode(datanode, block, start, end, bb,
- corruptedBlocks);
+ actualGetFromOneDataNode(datanode, start, end, bb, corruptedBlocks);
return bb;
}
}
@@ -995,27 +997,21 @@ public class DFSInputStream extends FSInputStream
* Read data from one DataNode.
*
* @param datanode the datanode from which to read data
- * @param block the located block containing the requested data
* @param startInBlk the startInBlk offset of the block
* @param endInBlk the endInBlk offset of the block
* @param buf the given byte buffer into which the data is read
* @param corruptedBlocks map recording list of datanodes with corrupted
* block replica
*/
- void actualGetFromOneDataNode(final DNAddrPair datanode, LocatedBlock block,
- final long startInBlk, final long endInBlk, ByteBuffer buf,
- CorruptedBlocks corruptedBlocks)
+ void actualGetFromOneDataNode(final DNAddrPair datanode, final long startInBlk,
+ final long endInBlk, ByteBuffer buf, CorruptedBlocks corruptedBlocks)
throws IOException {
DFSClientFaultInjector.get().startFetchFromDatanode();
int refetchToken = 1; // only need to get a new access token once
int refetchEncryptionKey = 1; // only need to get a new encryption key once
final int len = (int) (endInBlk - startInBlk + 1);
-
+ LocatedBlock block = datanode.block;
while (true) {
- // cached block locations may have been updated by chooseDataNode()
- // or fetchBlockAt(). Always get the latest list of locations at the
- // start of the loop.
- block = refreshLocatedBlock(block);
BlockReader reader = null;
try {
DFSClientFaultInjector.get().fetchFromDatanodeException();
@@ -1078,6 +1074,9 @@ public class DFSInputStream extends FSInputStream
addToDeadNodes(datanode.info);
throw new IOException(msg);
}
+ // Refresh the block for updated tokens in case of token failures or
+ // encryption key failures.
+ block = refreshLocatedBlock(block);
} finally {
if (reader != null) {
reader.close();
@@ -1113,7 +1112,6 @@ public class DFSInputStream extends FSInputStream
ByteBuffer bb;
int len = (int) (end - start + 1);
int hedgedReadId = 0;
- block = refreshLocatedBlock(block);
while (true) {
// see HDFS-6591, this metric is used to verify/catch unnecessary loops
hedgedReadOpsLoopNumForTesting++;
@@ -1123,6 +1121,8 @@ public class DFSInputStream extends FSInputStream
// chooseDataNode is a commitment. If no node, we go to
// the NN to reget block locations. Only go here on first read.
chosenNode = chooseDataNode(block, ignored);
+ // Latest block, if refreshed internally
+ block = chosenNode.block;
bb = ByteBuffer.allocate(len);
Callable<ByteBuffer> getFromDataNodeCallable = getFromOneDataNode(
chosenNode, block, start, end, bb,
@@ -1160,6 +1160,8 @@ public class DFSInputStream extends FSInputStream
if (chosenNode == null) {
chosenNode = chooseDataNode(block, ignored);
}
+ // Latest block, if refreshed internally
+ block = chosenNode.block;
bb = ByteBuffer.allocate(len);
Callable<ByteBuffer> getFromDataNodeCallable = getFromOneDataNode(
chosenNode, block, start, end, bb,
@@ -1530,12 +1532,14 @@ public class DFSInputStream extends FSInputStream
final DatanodeInfo info;
final InetSocketAddress addr;
final StorageType storageType;
+ final LocatedBlock block;
DNAddrPair(DatanodeInfo info, InetSocketAddress addr,
- StorageType storageType) {
+ StorageType storageType, LocatedBlock block) {
this.info = info;
this.addr = addr;
this.storageType = storageType;
+ this.block = block;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70fc6746/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 75ad022..d4d0646 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -236,7 +236,7 @@ public class DFSStripedInputStream extends DFSInputStream {
BlockReader reader = null;
final ReaderRetryPolicy retry = new ReaderRetryPolicy();
DFSInputStream.DNAddrPair dnInfo =
- new DFSInputStream.DNAddrPair(null, null, null);
+ new DFSInputStream.DNAddrPair(null, null, null, null);
while (true) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70fc6746/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 2cfcc2b..038b6ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -50,6 +50,7 @@ import java.lang.reflect.Modifier;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Socket;
+import java.net.SocketException;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
@@ -122,7 +123,9 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
@@ -2055,6 +2058,49 @@ public class DFSTestUtil {
return lastBlock;
}
+ /*
+ * Copy a block from sourceProxy to destination. If the block becomes
+ * over-replicated, preferably remove it from source.
+ * Return true if a block is successfully copied; otherwise false.
+ */
+ public static boolean replaceBlock(ExtendedBlock block, DatanodeInfo source,
+ DatanodeInfo sourceProxy, DatanodeInfo destination) throws IOException {
+ return replaceBlock(block, source, sourceProxy, destination,
+ StorageType.DEFAULT, Status.SUCCESS);
+ }
+
+ /*
+ * Replace block
+ */
+ public static boolean replaceBlock(ExtendedBlock block, DatanodeInfo source,
+ DatanodeInfo sourceProxy, DatanodeInfo destination,
+ StorageType targetStorageType, Status opStatus) throws IOException,
+ SocketException {
+ Socket sock = new Socket();
+ try {
+ sock.connect(NetUtils.createSocketAddr(destination.getXferAddr()),
+ HdfsConstants.READ_TIMEOUT);
+ sock.setKeepAlive(true);
+ // sendRequest
+ DataOutputStream out = new DataOutputStream(sock.getOutputStream());
+ new Sender(out).replaceBlock(block, targetStorageType,
+ BlockTokenSecretManager.DUMMY_TOKEN, source.getDatanodeUuid(),
+ sourceProxy, null);
+ out.flush();
+ // receiveResponse
+ DataInputStream reply = new DataInputStream(sock.getInputStream());
+
+ BlockOpResponseProto proto = BlockOpResponseProto.parseDelimitedFrom(
+ reply);
+ while (proto.getStatus() == Status.IN_PROGRESS) {
+ proto = BlockOpResponseProto.parseDelimitedFrom(reply);
+ }
+ return proto.getStatus() == opStatus;
+ } finally {
+ sock.close();
+ }
+ }
+
/**
* Because currently DFSStripedOutputStream does not support hflush/hsync,
* tests can use this method to flush all the buffered data to DataNodes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70fc6746/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
index 637f2df..85fc97b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
@@ -23,6 +23,8 @@ import static org.junit.Assert.assertTrue;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -30,6 +32,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ChecksumException;
@@ -38,6 +42,9 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
@@ -51,6 +58,8 @@ import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import com.google.common.base.Supplier;
+
/**
* This class tests the DFS positional read functionality in a single node
* mini-cluster.
@@ -542,6 +551,143 @@ public class TestPread {
}
}
+ /**
+ * Scenario: 1. Write a file with RF=2, DN1 and DN2<br>
+ * 2. Open the stream, Consider Locations are [DN1, DN2] in LocatedBlock.<br>
+ * 3. Move block from DN2 to DN3.<br>
+ * 4. Let block gets replicated to another DN3<br>
+ * 5. Stop DN1 also.<br>
+ * 6. Current valid Block locations in NameNode [DN1, DN3]<br>
+ * 7. Consider next calls to getBlockLocations() always returns DN3 as last
+ * location.<br>
+ */
+ @Test
+ public void testPreadFailureWithChangedBlockLocations() throws Exception {
+ doPreadTestWithChangedLocations();
+ }
+
+ /**
+ * Scenario: 1. Write a file with RF=2, DN1 and DN2<br>
+ * 2. Open the stream, Consider Locations are [DN1, DN2] in LocatedBlock.<br>
+ * 3. Move block from DN2 to DN3.<br>
+ * 4. Let block gets replicated to another DN3<br>
+ * 5. Stop DN1 also.<br>
+ * 6. Current valid Block locations in NameNode [DN1, DN3]<br>
+ * 7. Consider next calls to getBlockLocations() always returns DN3 as last
+ * location.<br>
+ */
+ @Test
+ public void testPreadHedgedFailureWithChangedBlockLocations()
+ throws Exception {
+ isHedgedRead = true;
+ doPreadTestWithChangedLocations();
+ }
+
+ private void doPreadTestWithChangedLocations()
+ throws IOException, TimeoutException, InterruptedException {
+ GenericTestUtils.setLogLevel(DFSClient.LOG, Level.DEBUG);
+ Configuration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ if (isHedgedRead) {
+ conf.setInt(HdfsClientConfigKeys.HedgedRead.THREADPOOL_SIZE_KEY, 2);
+ }
+ try (MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(3).build()) {
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ Path p = new Path("/test");
+ String data = "testingmissingblock";
+ DFSTestUtil.writeFile(dfs, p, data);
+
+ FSDataInputStream in = dfs.open(p);
+ List<LocatedBlock> blocks = DFSTestUtil.getAllBlocks(in);
+ LocatedBlock lb = blocks.get(0);
+ DFSTestUtil.waitForReplication(cluster, lb.getBlock(), 1, 2, 0);
+ blocks = DFSTestUtil.getAllBlocks(in);
+ DatanodeInfo[] locations = null;
+ for (LocatedBlock locatedBlock : blocks) {
+ locations = locatedBlock.getLocations();
+ DFSClient.LOG
+ .info(locatedBlock.getBlock() + " " + Arrays.toString(locations));
+ }
+ final DatanodeInfo validDownLocation = locations[0];
+ final DFSClient client = dfs.getClient();
+ DFSClient dfsClient = Mockito.spy(client);
+ // Keep the valid location as last in the locations list for second
+ // requests
+ // onwards.
+ final AtomicInteger count = new AtomicInteger(0);
+ Mockito.doAnswer(new Answer<LocatedBlocks>() {
+ @Override
+ public LocatedBlocks answer(InvocationOnMock invocation)
+ throws Throwable {
+ if (count.compareAndSet(0, 1)) {
+ return (LocatedBlocks) invocation.callRealMethod();
+ }
+ Object obj = invocation.callRealMethod();
+ LocatedBlocks locatedBlocks = (LocatedBlocks) obj;
+ LocatedBlock lb = locatedBlocks.get(0);
+ DatanodeInfo[] locations = lb.getLocations();
+ if (!(locations[0].getName().equals(validDownLocation.getName()))) {
+ // Latest location which is currently down, should be first
+ DatanodeInfo l = locations[0];
+ locations[0] = locations[locations.length - 1];
+ locations[locations.length - 1] = l;
+ }
+ return locatedBlocks;
+ }
+ }).when(dfsClient).getLocatedBlocks(p.toString(), 0);
+
+ // Findout target node to move the block to.
+ DatanodeInfo[] nodes =
+ cluster.getNameNodeRpc().getDatanodeReport(DatanodeReportType.LIVE);
+ DatanodeInfo toMove = null;
+ List<DatanodeInfo> locationsList = Arrays.asList(locations);
+ for (DatanodeInfo node : nodes) {
+ if (locationsList.contains(node)) {
+ continue;
+ }
+ toMove = node;
+ break;
+ }
+ // STEP 2: Open stream
+ DFSInputStream din = dfsClient.open(p.toString());
+ // STEP 3: Move replica
+ final DatanodeInfo source = locations[1];
+ final DatanodeInfo destination = toMove;
+ DFSTestUtil.replaceBlock(lb.getBlock(), source, locations[1], toMove);
+ // Wait for replica to get deleted
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+
+ @Override
+ public Boolean get() {
+ try {
+ LocatedBlocks lbs = dfsClient.getLocatedBlocks(p.toString(), 0);
+ LocatedBlock lb = lbs.get(0);
+ List<DatanodeInfo> locations = Arrays.asList(lb.getLocations());
+ DFSClient.LOG
+ .info("Source :" + source + ", destination: " + destination);
+ DFSClient.LOG.info("Got updated locations :" + locations);
+ return locations.contains(destination)
+ && !locations.contains(source);
+ } catch (IOException e) {
+ DFSClient.LOG.error("Problem in getting block locations", e);
+ }
+ return null;
+ }
+ }, 1000, 10000);
+ DFSTestUtil.waitForReplication(cluster, lb.getBlock(), 1, 2, 0);
+ // STEP 4: Stop first node in new locations
+ cluster.stopDataNode(validDownLocation.getName());
+ DFSClient.LOG.info("Starting read");
+ byte[] buf = new byte[1024];
+ int n = din.read(0, buf, 0, data.length());
+ assertEquals(data.length(), n);
+ assertEquals("Data should be read", data, new String(buf, 0, n));
+ DFSClient.LOG.info("Read completed");
+ }
+ }
+
public static void main(String[] args) throws Exception {
new TestPread().testPreadDFS();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70fc6746/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
index 8992d47..97255ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
@@ -17,13 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.net.Socket;
import java.net.SocketException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -48,19 +48,14 @@ import org.apache.hadoop.hdfs.client.BlockReportOptions;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
-import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Time;
import org.junit.Test;
@@ -371,7 +366,7 @@ public class TestBlockReplacement {
*/
private boolean replaceBlock( ExtendedBlock block, DatanodeInfo source,
DatanodeInfo sourceProxy, DatanodeInfo destination) throws IOException {
- return replaceBlock(block, source, sourceProxy, destination,
+ return DFSTestUtil.replaceBlock(block, source, sourceProxy, destination,
StorageType.DEFAULT, Status.SUCCESS);
}
@@ -385,29 +380,8 @@ public class TestBlockReplacement {
DatanodeInfo destination,
StorageType targetStorageType,
Status opStatus) throws IOException, SocketException {
- Socket sock = new Socket();
- try {
- sock.connect(NetUtils.createSocketAddr(destination.getXferAddr()),
- HdfsConstants.READ_TIMEOUT);
- sock.setKeepAlive(true);
- // sendRequest
- DataOutputStream out = new DataOutputStream(sock.getOutputStream());
- new Sender(out).replaceBlock(block, targetStorageType,
- BlockTokenSecretManager.DUMMY_TOKEN, source.getDatanodeUuid(),
- sourceProxy, null);
- out.flush();
- // receiveResponse
- DataInputStream reply = new DataInputStream(sock.getInputStream());
-
- BlockOpResponseProto proto =
- BlockOpResponseProto.parseDelimitedFrom(reply);
- while (proto.getStatus() == Status.IN_PROGRESS) {
- proto = BlockOpResponseProto.parseDelimitedFrom(reply);
- }
- return proto.getStatus() == opStatus;
- } finally {
- sock.close();
- }
+ return DFSTestUtil.replaceBlock(block, source, sourceProxy, destination,
+ targetStorageType, opStatus);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/29] hadoop git commit: YARN-6634. [API] Refactor ResourceManager
WebServices to make API explicit. (Giovanni Matteo Fumarola via curino)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547cb53e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index bd0602b..acfb2b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -204,18 +204,26 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
@Singleton
-@Path("/ws/v1/cluster")
-public class RMWebServices extends WebServices {
+@Path(RMWSConsts.RM_WEB_SERVICE_PATH)
+public class RMWebServices extends WebServices implements RMWebServiceProtocol {
+
private static final Log LOG =
LogFactory.getLog(RMWebServices.class.getName());
- private static final String EMPTY = "";
- private static final String ANY = "*";
+
private final ResourceManager rm;
- private static RecordFactory recordFactory = RecordFactoryProvider
- .getRecordFactory(null);
+ private static RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
private final Configuration conf;
private @Context HttpServletResponse response;
+ // -------Default values of QueryParams for RMWebServiceProtocol--------
+
+ public static final String DEFAULT_QUEUE = "default";
+ public static final String DEFAULT_RESERVATION_ID = "";
+ public static final String DEFAULT_START_TIME = "0";
+ public static final String DEFAULT_END_TIME = "-1";
+ public static final String DEFAULT_INCLUDE_RESOURCE = "false";
+
@VisibleForTesting
boolean isCentralizedNodeLabelConfiguration = true;
@@ -241,15 +249,15 @@ public class RMWebServices extends WebServices {
// Check for the authorization.
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
List<String> forwardedAddresses = null;
- String forwardedFor = hsr.getHeader("X-Forwarded-For");
+ String forwardedFor = hsr.getHeader(RMWSConsts.FORWARDED_FOR);
if (forwardedFor != null) {
forwardedAddresses = Arrays.asList(forwardedFor.split(","));
}
if (callerUGI != null
&& !(this.rm.getApplicationACLsManager().checkAccess(callerUGI,
- ApplicationAccessType.VIEW_APP, app.getUser(),
- app.getApplicationId()) ||
- this.rm.getQueueACLsManager().checkAccess(callerUGI,
+ ApplicationAccessType.VIEW_APP, app.getUser(),
+ app.getApplicationId())
+ || this.rm.getQueueACLsManager().checkAccess(callerUGI,
QueueACL.ADMINISTER_QUEUE, app, hsr.getRemoteAddr(),
forwardedAddresses))) {
return false;
@@ -258,39 +266,43 @@ public class RMWebServices extends WebServices {
}
private void init() {
- //clear content type
+ // clear content type
response.setContentType(null);
}
@GET
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public ClusterInfo get() {
return getClusterInfo();
}
@GET
- @Path("/info")
+ @Path(RMWSConsts.INFO)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public ClusterInfo getClusterInfo() {
init();
return new ClusterInfo(this.rm);
}
@GET
- @Path("/metrics")
+ @Path(RMWSConsts.METRICS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public ClusterMetricsInfo getClusterMetricsInfo() {
init();
return new ClusterMetricsInfo(this.rm);
}
@GET
- @Path("/scheduler")
+ @Path(RMWSConsts.SCHEDULER)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public SchedulerTypeInfo getSchedulerInfo() {
init();
ResourceScheduler rs = rm.getResourceScheduler();
@@ -298,8 +310,7 @@ public class RMWebServices extends WebServices {
if (rs instanceof CapacityScheduler) {
CapacityScheduler cs = (CapacityScheduler) rs;
CSQueue root = cs.getRootQueue();
- sinfo =
- new CapacitySchedulerInfo(root, cs);
+ sinfo = new CapacitySchedulerInfo(root, cs);
} else if (rs instanceof FairScheduler) {
FairScheduler fs = (FairScheduler) rs;
sinfo = new FairSchedulerInfo(fs);
@@ -312,10 +323,11 @@ public class RMWebServices extends WebServices {
}
@POST
- @Path("/scheduler/logs")
+ @Path(RMWSConsts.SCHEDULER_LOGS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
- public String dumpSchedulerLogs(@FormParam("time") String time,
+ @Override
+ public String dumpSchedulerLogs(@FormParam(RMWSConsts.TIME) String time,
@Context HttpServletRequest hsr) throws IOException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
@@ -345,52 +357,51 @@ public class RMWebServices extends WebServices {
return "Capacity scheduler logs are being created.";
}
- /**
- * Returns all nodes in the cluster. If the states param is given, returns
- * all nodes that are in the comma-separated list of states.
- */
@GET
- @Path("/nodes")
+ @Path(RMWSConsts.NODES)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
- public NodesInfo getNodes(@QueryParam("states") String states) {
+ @Override
+ public NodesInfo getNodes(@QueryParam(RMWSConsts.STATES) String states) {
init();
ResourceScheduler sched = this.rm.getResourceScheduler();
if (sched == null) {
throw new NotFoundException("Null ResourceScheduler instance");
}
-
+
EnumSet<NodeState> acceptedStates;
if (states == null) {
acceptedStates = EnumSet.allOf(NodeState.class);
} else {
acceptedStates = EnumSet.noneOf(NodeState.class);
for (String stateStr : states.split(",")) {
- acceptedStates.add(
- NodeState.valueOf(StringUtils.toUpperCase(stateStr)));
+ acceptedStates
+ .add(NodeState.valueOf(StringUtils.toUpperCase(stateStr)));
}
}
-
- Collection<RMNode> rmNodes = RMServerUtils.queryRMNodes(
- this.rm.getRMContext(), acceptedStates);
+
+ Collection<RMNode> rmNodes =
+ RMServerUtils.queryRMNodes(this.rm.getRMContext(), acceptedStates);
NodesInfo nodesInfo = new NodesInfo();
for (RMNode rmNode : rmNodes) {
NodeInfo nodeInfo = new NodeInfo(rmNode, sched);
- if (EnumSet.of(NodeState.LOST, NodeState.DECOMMISSIONED, NodeState.REBOOTED)
+ if (EnumSet
+ .of(NodeState.LOST, NodeState.DECOMMISSIONED, NodeState.REBOOTED)
.contains(rmNode.getState())) {
- nodeInfo.setNodeHTTPAddress(EMPTY);
+ nodeInfo.setNodeHTTPAddress(RMWSConsts.EMPTY);
}
nodesInfo.add(nodeInfo);
}
-
+
return nodesInfo;
}
@GET
- @Path("/nodes/{nodeId}")
+ @Path(RMWSConsts.NODES_NODEID)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
- public NodeInfo getNode(@PathParam("nodeId") String nodeId) {
+ @Override
+ public NodeInfo getNode(@PathParam(RMWSConsts.NODEID) String nodeId) {
init();
if (nodeId == null || nodeId.isEmpty()) {
throw new NotFoundException("nodeId, " + nodeId + ", is empty or null");
@@ -411,28 +422,29 @@ public class RMWebServices extends WebServices {
}
NodeInfo nodeInfo = new NodeInfo(ni, sched);
if (isInactive) {
- nodeInfo.setNodeHTTPAddress(EMPTY);
+ nodeInfo.setNodeHTTPAddress(RMWSConsts.EMPTY);
}
return nodeInfo;
}
@GET
- @Path("/apps")
+ @Path(RMWSConsts.APPS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public AppsInfo getApps(@Context HttpServletRequest hsr,
- @QueryParam("state") String stateQuery,
- @QueryParam("states") Set<String> statesQuery,
- @QueryParam("finalStatus") String finalStatusQuery,
- @QueryParam("user") String userQuery,
- @QueryParam("queue") String queueQuery,
- @QueryParam("limit") String count,
- @QueryParam("startedTimeBegin") String startedBegin,
- @QueryParam("startedTimeEnd") String startedEnd,
- @QueryParam("finishedTimeBegin") String finishBegin,
- @QueryParam("finishedTimeEnd") String finishEnd,
- @QueryParam("applicationTypes") Set<String> applicationTypes,
- @QueryParam("applicationTags") Set<String> applicationTags) {
+ @QueryParam(RMWSConsts.STATE) String stateQuery,
+ @QueryParam(RMWSConsts.STATES) Set<String> statesQuery,
+ @QueryParam(RMWSConsts.FINAL_STATUS) String finalStatusQuery,
+ @QueryParam(RMWSConsts.USER) String userQuery,
+ @QueryParam(RMWSConsts.QUEUE) String queueQuery,
+ @QueryParam(RMWSConsts.LIMIT) String count,
+ @QueryParam(RMWSConsts.STARTED_TIME_BEGIN) String startedBegin,
+ @QueryParam(RMWSConsts.STARTED_TIME_END) String startedEnd,
+ @QueryParam(RMWSConsts.FINISHED_TIME_BEGIN) String finishBegin,
+ @QueryParam(RMWSConsts.FINISHED_TIME_END) String finishEnd,
+ @QueryParam(RMWSConsts.APPLICATION_TYPES) Set<String> applicationTypes,
+ @QueryParam(RMWSConsts.APPLICATION_TAGS) Set<String> applicationTags) {
boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
@@ -460,7 +472,8 @@ public class RMWebServices extends WebServices {
checkStart = true;
sBegin = Long.parseLong(startedBegin);
if (sBegin < 0) {
- throw new BadRequestException("startedTimeBegin must be greater than 0");
+ throw new BadRequestException(
+ "startedTimeBegin must be greater than 0");
}
}
if (startedEnd != null && !startedEnd.isEmpty()) {
@@ -563,8 +576,8 @@ public class RMWebServices extends WebServices {
List<ApplicationReport> appReports = null;
try {
- appReports = rm.getClientRMService()
- .getApplications(request, false).getApplicationList();
+ appReports = rm.getClientRMService().getApplications(request, false)
+ .getApplicationList();
} catch (YarnException e) {
LOG.error("Unable to retrieve apps from ClientRMService", e);
throw new YarnRuntimeException(
@@ -588,19 +601,20 @@ public class RMWebServices extends WebServices {
}
}
- AppInfo app = new AppInfo(rm, rmapp,
- hasAccess(rmapp, hsr), WebAppUtils.getHttpSchemePrefix(conf));
+ AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr),
+ WebAppUtils.getHttpSchemePrefix(conf));
allApps.add(app);
}
return allApps;
}
@GET
- @Path("/scheduler/activities")
+ @Path(RMWSConsts.SCHEDULER_ACTIVITIES)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public ActivitiesInfo getActivities(@Context HttpServletRequest hsr,
- @QueryParam("nodeId") String nodeId) {
+ @QueryParam(RMWSConsts.NODEID) String nodeId) {
YarnScheduler scheduler = rm.getRMContext().getScheduler();
if (scheduler instanceof AbstractYarnScheduler) {
@@ -636,10 +650,12 @@ public class RMWebServices extends WebServices {
boolean correctNodeId = false;
for (FiCaSchedulerNode node : nodeList) {
- if ((portName.equals("") && node.getRMNode().getHostName().equals(
- hostName)) || (!portName.equals("") && node.getRMNode()
- .getHostName().equals(hostName) && String.valueOf(
- node.getRMNode().getCommandPort()).equals(portName))) {
+ if ((portName.equals("")
+ && node.getRMNode().getHostName().equals(hostName))
+ || (!portName.equals("")
+ && node.getRMNode().getHostName().equals(hostName)
+ && String.valueOf(node.getRMNode().getCommandPort())
+ .equals(portName))) {
correctNodeId = true;
nodeId = node.getNodeID().toString();
break;
@@ -665,11 +681,13 @@ public class RMWebServices extends WebServices {
}
@GET
- @Path("/scheduler/app-activities")
+ @Path(RMWSConsts.SCHEDULER_APP_ACTIVITIES)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public AppActivitiesInfo getAppActivities(@Context HttpServletRequest hsr,
- @QueryParam("appId") String appId, @QueryParam("maxTime") String time) {
+ @QueryParam(RMWSConsts.APP_ID) String appId,
+ @QueryParam(RMWSConsts.MAX_TIME) String time) {
YarnScheduler scheduler = rm.getRMContext().getScheduler();
if (scheduler instanceof AbstractYarnScheduler) {
@@ -683,7 +701,7 @@ public class RMWebServices extends WebServices {
return new AppActivitiesInfo(errMessage, appId);
}
- if(appId == null) {
+ if (appId == null) {
String errMessage = "Must provide an application Id";
return new AppActivitiesInfo(errMessage, null);
}
@@ -716,13 +734,14 @@ public class RMWebServices extends WebServices {
}
@GET
- @Path("/appstatistics")
+ @Path(RMWSConsts.APP_STATISTICS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public ApplicationStatisticsInfo getAppStatistics(
@Context HttpServletRequest hsr,
- @QueryParam("states") Set<String> stateQueries,
- @QueryParam("applicationTypes") Set<String> typeQueries) {
+ @QueryParam(RMWSConsts.STATES) Set<String> stateQueries,
+ @QueryParam(RMWSConsts.APPLICATION_TYPES) Set<String> typeQueries) {
init();
// parse the params and build the scoreboard
@@ -731,7 +750,7 @@ public class RMWebServices extends WebServices {
Set<String> types = parseQueries(typeQueries, false);
// if no types, counts the applications of any types
if (types.size() == 0) {
- types.add(ANY);
+ types.add(RMWSConsts.ANY);
} else if (types.size() != 1) {
throw new BadRequestException("# of applicationTypes = " + types.size()
+ ", we temporarily support at most one applicationType");
@@ -752,10 +771,9 @@ public class RMWebServices extends WebServices {
for (RMApp rmapp : apps.values()) {
YarnApplicationState state = rmapp.createApplicationState();
String type = StringUtils.toLowerCase(rmapp.getApplicationType().trim());
- if (states.contains(
- StringUtils.toLowerCase(state.toString()))) {
- if (types.contains(ANY)) {
- countApp(scoreboard, state, ANY);
+ if (states.contains(StringUtils.toLowerCase(state.toString()))) {
+ if (types.contains(RMWSConsts.ANY)) {
+ countApp(scoreboard, state, RMWSConsts.ANY);
} else if (types.contains(type)) {
countApp(scoreboard, state, type);
}
@@ -764,10 +782,10 @@ public class RMWebServices extends WebServices {
// fill the response object
ApplicationStatisticsInfo appStatInfo = new ApplicationStatisticsInfo();
- for (Map.Entry<YarnApplicationState, Map<String, Long>> partScoreboard
- : scoreboard.entrySet()) {
- for (Map.Entry<String, Long> statEntry
- : partScoreboard.getValue().entrySet()) {
+ for (Map.Entry<YarnApplicationState, Map<String, Long>> partScoreboard : scoreboard
+ .entrySet()) {
+ for (Map.Entry<String, Long> statEntry : partScoreboard.getValue()
+ .entrySet()) {
StatisticsItemInfo statItem = new StatisticsItemInfo(
partScoreboard.getKey(), statEntry.getKey(), statEntry.getValue());
appStatInfo.add(statItem);
@@ -777,9 +795,9 @@ public class RMWebServices extends WebServices {
}
private static Map<YarnApplicationState, Map<String, Long>> buildScoreboard(
- Set<String> states, Set<String> types) {
- Map<YarnApplicationState, Map<String, Long>> scoreboard
- = new HashMap<YarnApplicationState, Map<String, Long>>();
+ Set<String> states, Set<String> types) {
+ Map<YarnApplicationState, Map<String, Long>> scoreboard =
+ new HashMap<YarnApplicationState, Map<String, Long>>();
// default states will result in enumerating all YarnApplicationStates
assert !states.isEmpty();
for (String state : states) {
@@ -804,11 +822,12 @@ public class RMWebServices extends WebServices {
}
@GET
- @Path("/apps/{appid}")
+ @Path(RMWSConsts.APPS_APPID)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public AppInfo getApp(@Context HttpServletRequest hsr,
- @PathParam("appid") String appId) {
+ @PathParam(RMWSConsts.APPID) String appId) {
init();
ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId);
RMApp app = rm.getRMContext().getRMApps().get(id);
@@ -819,11 +838,12 @@ public class RMWebServices extends WebServices {
}
@GET
- @Path("/apps/{appid}/appattempts")
+ @Path(RMWSConsts.APPS_APPID_APPATTEMPTS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public AppAttemptsInfo getAppAttempts(@Context HttpServletRequest hsr,
- @PathParam("appid") String appId) {
+ @PathParam(RMWSConsts.APPID) String appId) {
init();
ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId);
@@ -843,25 +863,27 @@ public class RMWebServices extends WebServices {
}
@GET
- @Path("/apps/{appid}/appattempts/{appattemptid}")
+ @Path(RMWSConsts.APPS_APPID_APPATTEMPTS_APPATTEMPTID)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Override
- public org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo getAppAttempt(@Context HttpServletRequest req,
- @Context HttpServletResponse res, @PathParam("appid") String appId,
- @PathParam("appattemptid") String appAttemptId) {
+ public org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo getAppAttempt(
+ @Context HttpServletRequest req, @Context HttpServletResponse res,
+ @PathParam(RMWSConsts.APPID) String appId,
+ @PathParam(RMWSConsts.APPATTEMPTID) String appAttemptId) {
init(res);
return super.getAppAttempt(req, res, appId, appAttemptId);
}
@GET
- @Path("/apps/{appid}/appattempts/{appattemptid}/containers")
+ @Path(RMWSConsts.APPS_APPID_APPATTEMPTS_APPATTEMPTID_CONTAINERS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Override
public ContainersInfo getContainers(@Context HttpServletRequest req,
- @Context HttpServletResponse res, @PathParam("appid") String appId,
- @PathParam("appattemptid") String appAttemptId) {
+ @Context HttpServletResponse res,
+ @PathParam(RMWSConsts.APPID) String appId,
+ @PathParam(RMWSConsts.APPATTEMPTID) String appAttemptId) {
init(res);
return super.getContainers(req, res, appId, appAttemptId);
}
@@ -872,8 +894,9 @@ public class RMWebServices extends WebServices {
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Override
public ContainerInfo getContainer(@Context HttpServletRequest req,
- @Context HttpServletResponse res, @PathParam("appid") String appId,
- @PathParam("appattemptid") String appAttemptId,
+ @Context HttpServletResponse res,
+ @PathParam(RMWSConsts.APPID) String appId,
+ @PathParam(RMWSConsts.APPATTEMPTID) String appAttemptId,
@PathParam("containerid") String containerId) {
init(res);
return super.getContainer(req, res, appId, appAttemptId, containerId);
@@ -883,8 +906,9 @@ public class RMWebServices extends WebServices {
@Path("/apps/{appid}/state")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public AppState getAppState(@Context HttpServletRequest hsr,
- @PathParam("appid") String appId) throws AuthorizationException {
+ @PathParam(RMWSConsts.APPID) String appId) throws AuthorizationException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
String userName = "";
@@ -896,8 +920,8 @@ public class RMWebServices extends WebServices {
app = getRMAppForAppId(appId);
} catch (NotFoundException e) {
RMAuditLogger.logFailure(userName, AuditConstants.GET_APP_STATE,
- "UNKNOWN", "RMWebService",
- "Trying to get state of an absent application " + appId);
+ "UNKNOWN", "RMWebService",
+ "Trying to get state of an absent application " + appId);
throw e;
}
@@ -912,14 +936,15 @@ public class RMWebServices extends WebServices {
// to 202
@PUT
- @Path("/apps/{appid}/state")
+ @Path(RMWSConsts.APPS_APPID_STATE)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Override
public Response updateAppState(AppState targetState,
- @Context HttpServletRequest hsr, @PathParam("appid") String appId)
- throws AuthorizationException, YarnException, InterruptedException,
- IOException {
+ @Context HttpServletRequest hsr,
+ @PathParam(RMWSConsts.APPID) String appId) throws AuthorizationException,
+ YarnException, InterruptedException, IOException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
@@ -939,8 +964,8 @@ public class RMWebServices extends WebServices {
app = getRMAppForAppId(appId);
} catch (NotFoundException e) {
RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST,
- "UNKNOWN", "RMWebService", "Trying to kill an absent application "
- + appId);
+ "UNKNOWN", "RMWebService",
+ "Trying to kill an absent application " + appId);
throw e;
}
@@ -948,12 +973,13 @@ public class RMWebServices extends WebServices {
// user is attempting to change state. right we only
// allow users to kill the app
- if (targetState.getState().equals(YarnApplicationState.KILLED.toString())) {
+ if (targetState.getState()
+ .equals(YarnApplicationState.KILLED.toString())) {
return killApp(app, callerUGI, hsr, targetState.getDiagnostics());
}
- throw new BadRequestException("Only '"
- + YarnApplicationState.KILLED.toString()
- + "' is allowed as a target state.");
+ throw new BadRequestException(
+ "Only '" + YarnApplicationState.KILLED.toString()
+ + "' is allowed as a target state.");
}
AppState ret = new AppState();
@@ -961,19 +987,20 @@ public class RMWebServices extends WebServices {
return Response.status(Status.OK).entity(ret).build();
}
-
+
@GET
- @Path("/get-node-to-labels")
+ @Path(RMWSConsts.GET_NODE_TO_LABELS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public NodeToLabelsInfo getNodeToLabels(@Context HttpServletRequest hsr)
throws IOException {
init();
NodeToLabelsInfo ntl = new NodeToLabelsInfo();
HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels();
- Map<NodeId, Set<NodeLabel>> nodeIdToLabels = rm.getRMContext()
- .getNodeLabelManager().getNodeLabelsInfo();
+ Map<NodeId, Set<NodeLabel>> nodeIdToLabels =
+ rm.getRMContext().getNodeLabelManager().getNodeLabelsInfo();
for (Map.Entry<NodeId, Set<NodeLabel>> nitle : nodeIdToLabels.entrySet()) {
List<NodeLabel> labels = new ArrayList<NodeLabel>(nitle.getValue());
@@ -984,11 +1011,12 @@ public class RMWebServices extends WebServices {
}
@GET
- @Path("/label-mappings")
+ @Path(RMWSConsts.LABEL_MAPPINGS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public LabelsToNodesInfo getLabelsToNodes(
- @QueryParam("labels") Set<String> labels) throws IOException {
+ @QueryParam(RMWSConsts.LABELS) Set<String> labels) throws IOException {
init();
LabelsToNodesInfo lts = new LabelsToNodesInfo();
@@ -1007,17 +1035,19 @@ public class RMWebServices extends WebServices {
for (NodeId nodeId : entry.getValue()) {
nodeIdStrList.add(nodeId.toString());
}
- ltsMap.put(new NodeLabelInfo(entry.getKey()), new NodeIDsInfo(
- nodeIdStrList));
+ ltsMap.put(new NodeLabelInfo(entry.getKey()),
+ new NodeIDsInfo(nodeIdStrList));
}
return lts;
}
@POST
- @Path("/replace-node-to-labels")
+ @Path(RMWSConsts.REPLACE_NODE_TO_LABELS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
- public Response replaceLabelsOnNodes(final NodeToLabelsEntryList newNodeToLabels,
+ @Override
+ public Response replaceLabelsOnNodes(
+ final NodeToLabelsEntryList newNodeToLabels,
@Context HttpServletRequest hsr) throws IOException {
Map<NodeId, Set<String>> nodeIdToLabels =
new HashMap<NodeId, Set<String>>();
@@ -1032,9 +1062,10 @@ public class RMWebServices extends WebServices {
}
@POST
- @Path("/nodes/{nodeId}/replace-labels")
+ @Path(RMWSConsts.NODES_NODEID_REPLACE_LABELS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public Response replaceLabelsOnNode(
@QueryParam("labels") Set<String> newNodeLabelsName,
@Context HttpServletRequest hsr, @PathParam("nodeId") String nodeId)
@@ -1053,21 +1084,19 @@ public class RMWebServices extends WebServices {
String operation) throws IOException {
init();
- NodeLabelsUtils.verifyCentralizedNodeLabelConfEnabled(
- "replaceLabelsOnNode", isCentralizedNodeLabelConfiguration);
+ NodeLabelsUtils.verifyCentralizedNodeLabelConfEnabled("replaceLabelsOnNode",
+ isCentralizedNodeLabelConfiguration);
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
- String msg =
- "Unable to obtain user name, user not authenticated for"
- + " post to ..." + operation;
+ String msg = "Unable to obtain user name, user not authenticated for"
+ + " post to ..." + operation;
throw new AuthorizationException(msg);
}
if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
- String msg =
- "User " + callerUGI.getShortUserName() + " not authorized"
- + " for post to ..." + operation;
+ String msg = "User " + callerUGI.getShortUserName() + " not authorized"
+ + " for post to ..." + operation;
throw new AuthorizationException(msg);
}
try {
@@ -1081,58 +1110,60 @@ public class RMWebServices extends WebServices {
}
@GET
- @Path("/get-node-labels")
+ @Path(RMWSConsts.GET_NODE_LABELS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
- public NodeLabelsInfo getClusterNodeLabels(@Context HttpServletRequest hsr)
- throws IOException {
+ @Override
+ public NodeLabelsInfo getClusterNodeLabels(@Context HttpServletRequest hsr)
+ throws IOException {
init();
- List<NodeLabel> nodeLabels = rm.getRMContext().getNodeLabelManager()
- .getClusterNodeLabels();
+ List<NodeLabel> nodeLabels =
+ rm.getRMContext().getNodeLabelManager().getClusterNodeLabels();
NodeLabelsInfo ret = new NodeLabelsInfo(nodeLabels);
return ret;
}
-
+
@POST
- @Path("/add-node-labels")
+ @Path(RMWSConsts.ADD_NODE_LABELS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public Response addToClusterNodeLabels(final NodeLabelsInfo newNodeLabels,
- @Context HttpServletRequest hsr)
- throws Exception {
+ @Context HttpServletRequest hsr) throws Exception {
init();
-
+
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
String msg = "Unable to obtain user name, user not authenticated for"
- + " post to .../add-node-labels";
+ + " post to .../add-node-labels";
throw new AuthorizationException(msg);
}
if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
String msg = "User " + callerUGI.getShortUserName() + " not authorized"
- + " for post to .../add-node-labels ";
+ + " for post to .../add-node-labels ";
throw new AuthorizationException(msg);
}
-
+
try {
rm.getRMContext().getNodeLabelManager()
.addToCluserNodeLabels(newNodeLabels.getNodeLabels());
} catch (IOException e) {
throw new BadRequestException(e);
}
-
+
return Response.status(Status.OK).build();
}
-
+
@POST
- @Path("/remove-node-labels")
+ @Path(RMWSConsts.REMOVE_NODE_LABELS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public Response removeFromCluserNodeLabels(
- @QueryParam("labels") Set<String> oldNodeLabels,
+ @QueryParam(RMWSConsts.LABELS) Set<String> oldNodeLabels,
@Context HttpServletRequest hsr) throws Exception {
init();
@@ -1157,18 +1188,19 @@ public class RMWebServices extends WebServices {
return Response.status(Status.OK).build();
}
-
+
@GET
- @Path("/nodes/{nodeId}/get-labels")
+ @Path(RMWSConsts.NODES_NODEID_GETLABELS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr,
- @PathParam("nodeId") String nodeId) throws IOException {
+ @PathParam(RMWSConsts.NODEID) String nodeId) throws IOException {
init();
NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
- List<NodeLabel> labels = new ArrayList<NodeLabel>(rm.getRMContext()
- .getNodeLabelManager().getLabelsInfoByNode(nid));
+ List<NodeLabel> labels = new ArrayList<NodeLabel>(
+ rm.getRMContext().getNodeLabelManager().getLabelsInfoByNode(nid));
return new NodeLabelsInfo(labels);
}
@@ -1183,20 +1215,19 @@ public class RMWebServices extends WebServices {
final ApplicationId appid = app.getApplicationId();
KillApplicationResponse resp = null;
try {
- resp =
- callerUGI
- .doAs(new PrivilegedExceptionAction<KillApplicationResponse>() {
- @Override
- public KillApplicationResponse run() throws IOException,
- YarnException {
- KillApplicationRequest req =
- KillApplicationRequest.newInstance(appid);
- if (diagnostic != null) {
- req.setDiagnostics(diagnostic);
- }
- return rm.getClientRMService().forceKillApplication(req);
+ resp = callerUGI
+ .doAs(new PrivilegedExceptionAction<KillApplicationResponse>() {
+ @Override
+ public KillApplicationResponse run()
+ throws IOException, YarnException {
+ KillApplicationRequest req =
+ KillApplicationRequest.newInstance(appid);
+ if (diagnostic != null) {
+ req.setDiagnostics(diagnostic);
}
- });
+ return rm.getClientRMService().forceKillApplication(req);
+ }
+ });
} catch (UndeclaredThrowableException ue) {
// if the root cause is a permissions issue
// bubble that up to the user
@@ -1204,9 +1235,8 @@ public class RMWebServices extends WebServices {
YarnException ye = (YarnException) ue.getCause();
if (ye.getCause() instanceof AccessControlException) {
String appId = app.getApplicationId().toString();
- String msg =
- "Unauthorized attempt to kill appid " + appId
- + " by remote user " + userName;
+ String msg = "Unauthorized attempt to kill appid " + appId
+ + " by remote user " + userName;
return Response.status(Status.FORBIDDEN).entity(msg).build();
} else {
throw ue;
@@ -1221,20 +1251,21 @@ public class RMWebServices extends WebServices {
if (resp.getIsKillCompleted()) {
RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST,
- "RMWebService", app.getApplicationId());
+ "RMWebService", app.getApplicationId());
} else {
return Response.status(Status.ACCEPTED).entity(ret)
- .header(HttpHeaders.LOCATION, hsr.getRequestURL()).build();
+ .header(HttpHeaders.LOCATION, hsr.getRequestURL()).build();
}
return Response.status(Status.OK).entity(ret).build();
}
@GET
- @Path("/apps/{appid}/priority")
+ @Path(RMWSConsts.APPS_APPID_PRIORITY)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public AppPriority getAppPriority(@Context HttpServletRequest hsr,
- @PathParam("appid") String appId) throws AuthorizationException {
+ @PathParam(RMWSConsts.APPID) String appId) throws AuthorizationException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
String userName = "UNKNOWN-USER";
@@ -1252,21 +1283,21 @@ public class RMWebServices extends WebServices {
}
AppPriority ret = new AppPriority();
- ret.setPriority(
- app.getApplicationPriority().getPriority());
+ ret.setPriority(app.getApplicationPriority().getPriority());
return ret;
}
@PUT
- @Path("/apps/{appid}/priority")
+ @Path(RMWSConsts.APPS_APPID_PRIORITY)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Override
public Response updateApplicationPriority(AppPriority targetPriority,
- @Context HttpServletRequest hsr, @PathParam("appid") String appId)
- throws AuthorizationException, YarnException, InterruptedException,
- IOException {
+ @Context HttpServletRequest hsr,
+ @PathParam(RMWSConsts.APPID) String appId) throws AuthorizationException,
+ YarnException, InterruptedException, IOException {
init();
if (targetPriority == null) {
throw new YarnException("Target Priority cannot be null");
@@ -1305,7 +1336,7 @@ public class RMWebServices extends WebServices {
private Response modifyApplicationPriority(final RMApp app,
UserGroupInformation callerUGI, final int appPriority)
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException {
String userName = callerUGI.getUserName();
try {
callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
@@ -1340,17 +1371,18 @@ public class RMWebServices extends WebServices {
throw ue;
}
}
- AppPriority ret = new AppPriority(
- app.getApplicationPriority().getPriority());
+ AppPriority ret =
+ new AppPriority(app.getApplicationPriority().getPriority());
return Response.status(Status.OK).entity(ret).build();
}
@GET
- @Path("/apps/{appid}/queue")
+ @Path(RMWSConsts.APPS_APPID_QUEUE)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public AppQueue getAppQueue(@Context HttpServletRequest hsr,
- @PathParam("appid") String appId) throws AuthorizationException {
+ @PathParam(RMWSConsts.APPID) String appId) throws AuthorizationException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
String userName = "UNKNOWN-USER";
@@ -1362,8 +1394,8 @@ public class RMWebServices extends WebServices {
app = getRMAppForAppId(appId);
} catch (NotFoundException e) {
RMAuditLogger.logFailure(userName, AuditConstants.GET_APP_QUEUE,
- "UNKNOWN", "RMWebService",
- "Trying to get queue of an absent application " + appId);
+ "UNKNOWN", "RMWebService",
+ "Trying to get queue of an absent application " + appId);
throw e;
}
@@ -1374,14 +1406,15 @@ public class RMWebServices extends WebServices {
}
@PUT
- @Path("/apps/{appid}/queue")
+ @Path(RMWSConsts.APPS_APPID_QUEUE)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Override
public Response updateAppQueue(AppQueue targetQueue,
- @Context HttpServletRequest hsr, @PathParam("appid") String appId)
- throws AuthorizationException, YarnException, InterruptedException,
- IOException {
+ @Context HttpServletRequest hsr,
+ @PathParam(RMWSConsts.APPID) String appId) throws AuthorizationException,
+ YarnException, InterruptedException, IOException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
@@ -1401,8 +1434,8 @@ public class RMWebServices extends WebServices {
app = getRMAppForAppId(appId);
} catch (NotFoundException e) {
RMAuditLogger.logFailure(userName, AuditConstants.MOVE_APP_REQUEST,
- "UNKNOWN", "RMWebService", "Trying to move an absent application "
- + appId);
+ "UNKNOWN", "RMWebService",
+ "Trying to move an absent application " + appId);
throw e;
}
@@ -1427,18 +1460,16 @@ public class RMWebServices extends WebServices {
final ApplicationId appid = app.getApplicationId();
final String reqTargetQueue = targetQueue;
try {
- callerUGI
- .doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws IOException,
- YarnException {
- MoveApplicationAcrossQueuesRequest req =
- MoveApplicationAcrossQueuesRequest.newInstance(appid,
+ callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException, YarnException {
+ MoveApplicationAcrossQueuesRequest req =
+ MoveApplicationAcrossQueuesRequest.newInstance(appid,
reqTargetQueue);
- rm.getClientRMService().moveApplicationAcrossQueues(req);
- return null;
- }
- });
+ rm.getClientRMService().moveApplicationAcrossQueues(req);
+ return null;
+ }
+ });
} catch (UndeclaredThrowableException ue) {
// if the root cause is a permissions issue
// bubble that up to the user
@@ -1446,14 +1477,13 @@ public class RMWebServices extends WebServices {
YarnException ye = (YarnException) ue.getCause();
if (ye.getCause() instanceof AccessControlException) {
String appId = app.getApplicationId().toString();
- String msg =
- "Unauthorized attempt to move appid " + appId
- + " by remote user " + userName;
+ String msg = "Unauthorized attempt to move appid " + appId
+ + " by remote user " + userName;
return Response.status(Status.FORBIDDEN).entity(msg).build();
} else if (ye.getMessage().startsWith("App in")
&& ye.getMessage().endsWith("state cannot be moved.")) {
return Response.status(Status.BAD_REQUEST).entity(ye.getMessage())
- .build();
+ .build();
} else {
throw ue;
}
@@ -1496,32 +1526,22 @@ public class RMWebServices extends WebServices {
private boolean isStaticUser(UserGroupInformation callerUGI) {
String staticUser =
conf.get(CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER,
- CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER);
+ CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER);
return staticUser.equals(callerUGI.getUserName());
}
- /**
- * Generates a new ApplicationId which is then sent to the client
- *
- * @param hsr
- * the servlet request
- * @return Response containing the app id and the maximum resource
- * capabilities
- * @throws AuthorizationException
- * @throws IOException
- * @throws InterruptedException
- */
@POST
- @Path("/apps/new-application")
+ @Path(RMWSConsts.APPS_NEW_APPLICATION)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public Response createNewApplication(@Context HttpServletRequest hsr)
throws AuthorizationException, IOException, InterruptedException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
- throw new AuthorizationException("Unable to obtain user name, "
- + "user not authenticated");
+ throw new AuthorizationException(
+ "Unable to obtain user name, " + "user not authenticated");
}
if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
String msg = "The default static user cannot carry out this operation.";
@@ -1536,33 +1556,21 @@ public class RMWebServices extends WebServices {
// reuse the code in ClientRMService to create new app
// get the new app id and submit app
// set location header with new app location
- /**
- * Function to submit an app to the RM
- *
- * @param newApp
- * structure containing information to construct the
- * ApplicationSubmissionContext
- * @param hsr
- * the servlet request
- * @return Response containing the status code
- * @throws AuthorizationException
- * @throws IOException
- * @throws InterruptedException
- */
@POST
- @Path("/apps")
+ @Path(RMWSConsts.APPS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Override
public Response submitApplication(ApplicationSubmissionContextInfo newApp,
- @Context HttpServletRequest hsr) throws AuthorizationException,
- IOException, InterruptedException {
+ @Context HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
- throw new AuthorizationException("Unable to obtain user name, "
- + "user not authenticated");
+ throw new AuthorizationException(
+ "Unable to obtain user name, " + "user not authenticated");
}
if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
@@ -1578,13 +1586,13 @@ public class RMWebServices extends WebServices {
try {
callerUGI
- .doAs(new PrivilegedExceptionAction<SubmitApplicationResponse>() {
- @Override
- public SubmitApplicationResponse run() throws IOException,
- YarnException {
- return rm.getClientRMService().submitApplication(req);
- }
- });
+ .doAs(new PrivilegedExceptionAction<SubmitApplicationResponse>() {
+ @Override
+ public SubmitApplicationResponse run()
+ throws IOException, YarnException {
+ return rm.getClientRMService().submitApplication(req);
+ }
+ });
} catch (UndeclaredThrowableException ue) {
if (ue.getCause() instanceof YarnException) {
throw new BadRequestException(ue.getCause().getMessage());
@@ -1595,7 +1603,7 @@ public class RMWebServices extends WebServices {
String url = hsr.getRequestURL() + "/" + newApp.getApplicationId();
return Response.status(Status.ACCEPTED).header(HttpHeaders.LOCATION, url)
- .build();
+ .build();
}
/**
@@ -1618,7 +1626,7 @@ public class RMWebServices extends WebServices {
}
NewApplication appId =
new NewApplication(resp.getApplicationId().toString(),
- new ResourceInfo(resp.getMaximumResourceCapability()));
+ new ResourceInfo(resp.getMaximumResourceCapability()));
return appId;
}
@@ -1626,8 +1634,7 @@ public class RMWebServices extends WebServices {
* Create the actual ApplicationSubmissionContext to be submitted to the RM
* from the information provided by the user.
*
- * @param newApp
- * the information provided by the user
+ * @param newApp the information provided by the user
* @return returns the constructed ApplicationSubmissionContext
* @throws IOException
*/
@@ -1644,28 +1651,27 @@ public class RMWebServices extends WebServices {
} catch (Exception e) {
throw new BadRequestException(error);
}
- ApplicationSubmissionContext appContext =
- ApplicationSubmissionContext.newInstance(appid,
- newApp.getApplicationName(), newApp.getQueue(),
- Priority.newInstance(newApp.getPriority()),
- createContainerLaunchContext(newApp), newApp.getUnmanagedAM(),
- newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(),
- createAppSubmissionContextResource(newApp),
- newApp.getApplicationType(),
- newApp.getKeepContainersAcrossApplicationAttempts(),
- newApp.getAppNodeLabelExpression(),
- newApp.getAMContainerNodeLabelExpression());
+ ApplicationSubmissionContext appContext = ApplicationSubmissionContext
+ .newInstance(appid, newApp.getApplicationName(), newApp.getQueue(),
+ Priority.newInstance(newApp.getPriority()),
+ createContainerLaunchContext(newApp), newApp.getUnmanagedAM(),
+ newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(),
+ createAppSubmissionContextResource(newApp),
+ newApp.getApplicationType(),
+ newApp.getKeepContainersAcrossApplicationAttempts(),
+ newApp.getAppNodeLabelExpression(),
+ newApp.getAMContainerNodeLabelExpression());
appContext.setApplicationTags(newApp.getApplicationTags());
appContext.setAttemptFailuresValidityInterval(
newApp.getAttemptFailuresValidityInterval());
if (newApp.getLogAggregationContextInfo() != null) {
- appContext.setLogAggregationContext(createLogAggregationContext(
- newApp.getLogAggregationContextInfo()));
+ appContext.setLogAggregationContext(
+ createLogAggregationContext(newApp.getLogAggregationContextInfo()));
}
String reservationIdStr = newApp.getReservationId();
if (reservationIdStr != null && !reservationIdStr.isEmpty()) {
- ReservationId reservationId = ReservationId.parseReservationId(
- reservationIdStr);
+ ReservationId reservationId =
+ ReservationId.parseReservationId(reservationIdStr);
appContext.setReservationID(reservationId);
}
return appContext;
@@ -1674,20 +1680,19 @@ public class RMWebServices extends WebServices {
protected Resource createAppSubmissionContextResource(
ApplicationSubmissionContextInfo newApp) throws BadRequestException {
if (newApp.getResource().getvCores() > rm.getConfig().getInt(
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)) {
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)) {
String msg = "Requested more cores than configured max";
throw new BadRequestException(msg);
}
if (newApp.getResource().getMemorySize() > rm.getConfig().getInt(
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)) {
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)) {
String msg = "Requested more memory than configured max";
throw new BadRequestException(msg);
}
- Resource r =
- Resource.newInstance(newApp.getResource().getMemorySize(), newApp
- .getResource().getvCores());
+ Resource r = Resource.newInstance(newApp.getResource().getMemorySize(),
+ newApp.getResource().getvCores());
return r;
}
@@ -1696,21 +1701,20 @@ public class RMWebServices extends WebServices {
* ApplicationSubmissionContext. This function takes the user information and
* generates the ByteBuffer structures required by the ContainerLaunchContext
*
- * @param newApp
- * the information provided by the user
+ * @param newApp the information provided by the user
* @return created context
* @throws BadRequestException
* @throws IOException
*/
protected ContainerLaunchContext createContainerLaunchContext(
- ApplicationSubmissionContextInfo newApp) throws BadRequestException,
- IOException {
+ ApplicationSubmissionContextInfo newApp)
+ throws BadRequestException, IOException {
// create container launch context
HashMap<String, ByteBuffer> hmap = new HashMap<String, ByteBuffer>();
for (Map.Entry<String, String> entry : newApp
- .getContainerLaunchContextInfo().getAuxillaryServiceData().entrySet()) {
+ .getContainerLaunchContextInfo().getAuxillaryServiceData().entrySet()) {
if (entry.getValue().isEmpty() == false) {
Base64 decoder = new Base64(0, null, true);
byte[] data = decoder.decode(entry.getValue());
@@ -1720,27 +1724,23 @@ public class RMWebServices extends WebServices {
HashMap<String, LocalResource> hlr = new HashMap<String, LocalResource>();
for (Map.Entry<String, LocalResourceInfo> entry : newApp
- .getContainerLaunchContextInfo().getResources().entrySet()) {
+ .getContainerLaunchContextInfo().getResources().entrySet()) {
LocalResourceInfo l = entry.getValue();
- LocalResource lr =
- LocalResource.newInstance(
- URL.fromURI(l.getUrl()), l.getType(),
- l.getVisibility(), l.getSize(), l.getTimestamp());
+ LocalResource lr = LocalResource.newInstance(URL.fromURI(l.getUrl()),
+ l.getType(), l.getVisibility(), l.getSize(), l.getTimestamp());
hlr.put(entry.getKey(), lr);
}
DataOutputBuffer out = new DataOutputBuffer();
- Credentials cs =
- createCredentials(newApp.getContainerLaunchContextInfo()
- .getCredentials());
+ Credentials cs = createCredentials(
+ newApp.getContainerLaunchContextInfo().getCredentials());
cs.writeTokenStorageToStream(out);
ByteBuffer tokens = ByteBuffer.wrap(out.getData());
- ContainerLaunchContext ctx =
- ContainerLaunchContext.newInstance(hlr, newApp
- .getContainerLaunchContextInfo().getEnvironment(), newApp
- .getContainerLaunchContextInfo().getCommands(), hmap, tokens, newApp
- .getContainerLaunchContextInfo().getAcls());
+ ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(hlr,
+ newApp.getContainerLaunchContextInfo().getEnvironment(),
+ newApp.getContainerLaunchContextInfo().getCommands(), hmap, tokens,
+ newApp.getContainerLaunchContextInfo().getAcls());
return ctx;
}
@@ -1749,20 +1749,21 @@ public class RMWebServices extends WebServices {
* Generate a Credentials object from the information in the CredentialsInfo
* object.
*
- * @param credentials
- * the CredentialsInfo provided by the user.
+ * @param credentials the CredentialsInfo provided by the user.
* @return
*/
private Credentials createCredentials(CredentialsInfo credentials) {
Credentials ret = new Credentials();
try {
- for (Map.Entry<String, String> entry : credentials.getTokens().entrySet()) {
+ for (Map.Entry<String, String> entry : credentials.getTokens()
+ .entrySet()) {
Text alias = new Text(entry.getKey());
Token<TokenIdentifier> token = new Token<TokenIdentifier>();
token.decodeFromUrlString(entry.getValue());
ret.addToken(alias, token);
}
- for (Map.Entry<String, String> entry : credentials.getSecrets().entrySet()) {
+ for (Map.Entry<String, String> entry : credentials.getSecrets()
+ .entrySet()) {
Text alias = new Text(entry.getKey());
Base64 decoder = new Base64(0, null, true);
byte[] secret = decoder.decode(entry.getValue());
@@ -1770,8 +1771,8 @@ public class RMWebServices extends WebServices {
}
} catch (IOException ie) {
throw new BadRequestException(
- "Could not parse credentials data; exception message = "
- + ie.getMessage());
+ "Could not parse credentials data; exception message = "
+ + ie.getMessage());
}
return ret;
}
@@ -1787,14 +1788,13 @@ public class RMWebServices extends WebServices {
String authType = hsr.getAuthType();
if (!KerberosAuthenticationHandler.TYPE.equalsIgnoreCase(authType)) {
- String msg =
- "Delegation token operations can only be carried out on a "
- + "Kerberos authenticated channel. Expected auth type is "
- + KerberosAuthenticationHandler.TYPE + ", got type " + authType;
+ String msg = "Delegation token operations can only be carried out on a "
+ + "Kerberos authenticated channel. Expected auth type is "
+ + KerberosAuthenticationHandler.TYPE + ", got type " + authType;
throw new YarnException(msg);
}
- if (hsr
- .getAttribute(DelegationTokenAuthenticationHandler.DELEGATION_TOKEN_UGI_ATTRIBUTE) != null) {
+ if (hsr.getAttribute(
+ DelegationTokenAuthenticationHandler.DELEGATION_TOKEN_UGI_ATTRIBUTE) != null) {
String msg =
"Delegation token operations cannot be carried out using delegation"
+ " token authentication.";
@@ -1817,10 +1817,11 @@ public class RMWebServices extends WebServices {
}
@POST
- @Path("/delegation-token")
+ @Path(RMWSConsts.DELEGATION_TOKEN)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Override
public Response postDelegationToken(DelegationToken tokenData,
@Context HttpServletRequest hsr) throws AuthorizationException,
IOException, InterruptedException, Exception {
@@ -1836,14 +1837,14 @@ public class RMWebServices extends WebServices {
}
@POST
- @Path("/delegation-token/expiration")
+ @Path(RMWSConsts.DELEGATION_TOKEN_EXPIRATION)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public Response
- postDelegationTokenExpiration(@Context HttpServletRequest hsr)
- throws AuthorizationException, IOException, InterruptedException,
- Exception {
+ @Override
+ public Response postDelegationTokenExpiration(@Context HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException,
+ Exception {
init();
UserGroupInformation callerUGI;
@@ -1866,35 +1867,33 @@ public class RMWebServices extends WebServices {
final String renewer = tokenData.getRenewer();
GetDelegationTokenResponse resp;
try {
- resp =
- callerUGI
- .doAs(new PrivilegedExceptionAction<GetDelegationTokenResponse>() {
- @Override
- public GetDelegationTokenResponse run() throws IOException,
- YarnException {
- GetDelegationTokenRequest createReq =
- GetDelegationTokenRequest.newInstance(renewer);
- return rm.getClientRMService().getDelegationToken(createReq);
- }
- });
+ resp = callerUGI
+ .doAs(new PrivilegedExceptionAction<GetDelegationTokenResponse>() {
+ @Override
+ public GetDelegationTokenResponse run()
+ throws IOException, YarnException {
+ GetDelegationTokenRequest createReq =
+ GetDelegationTokenRequest.newInstance(renewer);
+ return rm.getClientRMService().getDelegationToken(createReq);
+ }
+ });
} catch (Exception e) {
LOG.info("Create delegation token request failed", e);
throw e;
}
Token<RMDelegationTokenIdentifier> tk =
- new Token<RMDelegationTokenIdentifier>(resp.getRMDelegationToken()
- .getIdentifier().array(), resp.getRMDelegationToken().getPassword()
- .array(), new Text(resp.getRMDelegationToken().getKind()), new Text(
- resp.getRMDelegationToken().getService()));
+ new Token<RMDelegationTokenIdentifier>(
+ resp.getRMDelegationToken().getIdentifier().array(),
+ resp.getRMDelegationToken().getPassword().array(),
+ new Text(resp.getRMDelegationToken().getKind()),
+ new Text(resp.getRMDelegationToken().getService()));
RMDelegationTokenIdentifier identifier = tk.decodeIdentifier();
- long currentExpiration =
- rm.getRMContext().getRMDelegationTokenSecretManager()
- .getRenewDate(identifier);
- DelegationToken respToken =
- new DelegationToken(tk.encodeToUrlString(), renewer, identifier
- .getOwner().toString(), tk.getKind().toString(), currentExpiration,
- identifier.getMaxDate());
+ long currentExpiration = rm.getRMContext()
+ .getRMDelegationTokenSecretManager().getRenewDate(identifier);
+ DelegationToken respToken = new DelegationToken(tk.encodeToUrlString(),
+ renewer, identifier.getOwner().toString(), tk.getKind().toString(),
+ currentExpiration, identifier.getMaxDate());
return Response.status(Status.OK).entity(respToken).build();
}
@@ -1906,30 +1905,29 @@ public class RMWebServices extends WebServices {
Token<RMDelegationTokenIdentifier> token =
extractToken(tokenData.getToken());
- org.apache.hadoop.yarn.api.records.Token dToken =
- BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
- .toString(), token.getPassword(), token.getService().toString());
+ org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils
+ .newDelegationToken(token.getIdentifier(), token.getKind().toString(),
+ token.getPassword(), token.getService().toString());
final RenewDelegationTokenRequest req =
RenewDelegationTokenRequest.newInstance(dToken);
RenewDelegationTokenResponse resp;
try {
- resp =
- callerUGI
- .doAs(new PrivilegedExceptionAction<RenewDelegationTokenResponse>() {
- @Override
- public RenewDelegationTokenResponse run() throws IOException,
- YarnException {
- return rm.getClientRMService().renewDelegationToken(req);
- }
- });
+ resp = callerUGI
+ .doAs(new PrivilegedExceptionAction<RenewDelegationTokenResponse>() {
+ @Override
+ public RenewDelegationTokenResponse run() throws YarnException {
+ return rm.getClientRMService().renewDelegationToken(req);
+ }
+ });
} catch (UndeclaredThrowableException ue) {
if (ue.getCause() instanceof YarnException) {
if (ue.getCause().getCause() instanceof InvalidToken) {
throw new BadRequestException(ue.getCause().getCause().getMessage());
- } else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) {
+ } else if (ue.getCause()
+ .getCause() instanceof org.apache.hadoop.security.AccessControlException) {
return Response.status(Status.FORBIDDEN)
- .entity(ue.getCause().getCause().getMessage()).build();
+ .entity(ue.getCause().getCause().getMessage()).build();
}
LOG.info("Renew delegation token request failed", ue);
throw ue;
@@ -1955,9 +1953,10 @@ public class RMWebServices extends WebServices {
// since urls tend to get logged and anyone with access to
// the logs can extract tokens which are meant to be secret
@DELETE
- @Path("/delegation-token")
+ @Path(RMWSConsts.DELEGATION_TOKEN)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public Response cancelDelegationToken(@Context HttpServletRequest hsr)
throws AuthorizationException, IOException, InterruptedException,
Exception {
@@ -1972,28 +1971,29 @@ public class RMWebServices extends WebServices {
Token<RMDelegationTokenIdentifier> token = extractToken(hsr);
- org.apache.hadoop.yarn.api.records.Token dToken =
- BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
- .toString(), token.getPassword(), token.getService().toString());
+ org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils
+ .newDelegationToken(token.getIdentifier(), token.getKind().toString(),
+ token.getPassword(), token.getService().toString());
final CancelDelegationTokenRequest req =
CancelDelegationTokenRequest.newInstance(dToken);
try {
callerUGI
- .doAs(new PrivilegedExceptionAction<CancelDelegationTokenResponse>() {
- @Override
- public CancelDelegationTokenResponse run() throws IOException,
- YarnException {
- return rm.getClientRMService().cancelDelegationToken(req);
- }
- });
+ .doAs(new PrivilegedExceptionAction<CancelDelegationTokenResponse>() {
+ @Override
+ public CancelDelegationTokenResponse run()
+ throws IOException, YarnException {
+ return rm.getClientRMService().cancelDelegationToken(req);
+ }
+ });
} catch (UndeclaredThrowableException ue) {
if (ue.getCause() instanceof YarnException) {
if (ue.getCause().getCause() instanceof InvalidToken) {
throw new BadRequestException(ue.getCause().getCause().getMessage());
- } else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) {
+ } else if (ue.getCause()
+ .getCause() instanceof org.apache.hadoop.security.AccessControlException) {
return Response.status(Status.FORBIDDEN)
- .entity(ue.getCause().getCause().getMessage()).build();
+ .entity(ue.getCause().getCause().getMessage()).build();
}
LOG.info("Renew delegation token request failed", ue);
throw ue;
@@ -2012,9 +2012,8 @@ public class RMWebServices extends WebServices {
HttpServletRequest request) {
String encodedToken = request.getHeader(DELEGATION_TOKEN_HEADER);
if (encodedToken == null) {
- String msg =
- "Header '" + DELEGATION_TOKEN_HEADER
- + "' containing encoded token not found";
+ String msg = "Header '" + DELEGATION_TOKEN_HEADER
+ + "' containing encoded token not found";
throw new BadRequestException(msg);
}
return extractToken(encodedToken);
@@ -2032,28 +2031,18 @@ public class RMWebServices extends WebServices {
return token;
}
- /**
- * Generates a new ReservationId which is then sent to the client.
- *
- * @param hsr the servlet request
- * @return Response containing the app id and the maximum resource
- * capabilities
- * @throws AuthorizationException if the user is not authorized
- * to invoke this method.
- * @throws IOException if creation fails.
- * @throws InterruptedException if interrupted.
- */
@POST
- @Path("/reservation/new-reservation")
+ @Path(RMWSConsts.RESERVATION_NEW)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public Response createNewReservation(@Context HttpServletRequest hsr)
- throws AuthorizationException, IOException, InterruptedException {
+ throws AuthorizationException, IOException, InterruptedException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
- throw new AuthorizationException("Unable to obtain user name, "
- + "user not authenticated");
+ throw new AuthorizationException(
+ "Unable to obtain user name, " + "user not authenticated");
}
if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
String msg = "The default static user cannot carry out this operation.";
@@ -2088,32 +2077,21 @@ public class RMWebServices extends WebServices {
return reservationId;
}
- /**
- * Function to submit a Reservation to the RM.
- *
- * @param resContext provides information to construct the
- * ReservationSubmissionRequest
- * @param hsr the servlet request
- * @return Response containing the status code
- * @throws AuthorizationException
- * @throws IOException
- * @throws InterruptedException
- */
@POST
- @Path("/reservation/submit")
+ @Path(RMWSConsts.RESERVATION_SUBMIT)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public Response submitReservation(
- ReservationSubmissionRequestInfo resContext,
- @Context HttpServletRequest hsr) throws AuthorizationException,
- IOException, InterruptedException {
+ @Override
+ public Response submitReservation(ReservationSubmissionRequestInfo resContext,
+ @Context HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
- throw new AuthorizationException("Unable to obtain user name, "
- + "user not authenticated");
+ throw new AuthorizationException(
+ "Unable to obtain user name, " + "user not authenticated");
}
if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
String msg = "The default static user cannot carry out this operation.";
@@ -2126,11 +2104,11 @@ public class RMWebServices extends WebServices {
try {
callerUGI
.doAs(new PrivilegedExceptionAction<ReservationSubmissionResponse>() {
- @Override
- public ReservationSubmissionResponse run() throws IOException,
- YarnException {
- return rm.getClientRMService().submitReservation(reservation);
- }
+ @Override
+ public ReservationSubmissionResponse run()
+ throws IOException, YarnException {
+ return rm.getClientRMService().submitReservation(reservation);
+ }
});
} catch (UndeclaredThrowableException ue) {
if (ue.getCause() instanceof YarnException) {
@@ -2179,9 +2157,8 @@ public class RMWebServices extends WebServices {
int numContainers = resReqInfo.getNumContainers();
int minConcurrency = resReqInfo.getMinConcurrency();
long duration = resReqInfo.getDuration();
- ReservationRequest rr =
- ReservationRequest.newInstance(capability, numContainers,
- minConcurrency, duration);
+ ReservationRequest rr = ReservationRequest.newInstance(capability,
+ numContainers, minConcurrency, duration);
list.add(rr);
}
ReservationRequests reqs = ReservationRequests.newInstance(list, resInt);
@@ -2189,40 +2166,29 @@ public class RMWebServices extends WebServices {
ReservationDefinition.newInstance(resInfo.getArrival(),
resInfo.getDeadline(), reqs, resInfo.getReservationName());
- ReservationId reservationId = ReservationId.parseReservationId(resContext
- .getReservationId());
- ReservationSubmissionRequest request =
- ReservationSubmissionRequest.newInstance(rDef, resContext.getQueue(),
- reservationId);
+ ReservationId reservationId =
+ ReservationId.parseReservationId(resContext.getReservationId());
+ ReservationSubmissionRequest request = ReservationSubmissionRequest
+ .newInstance(rDef, resContext.getQueue(), reservationId);
return request;
}
- /**
- * Function to update a Reservation to the RM.
- *
- * @param resContext provides information to construct the
- * ReservationUpdateRequest
- * @param hsr the servlet request
- * @return Response containing the status code
- * @throws AuthorizationException
- * @throws IOException
- * @throws InterruptedException
- */
@POST
- @Path("/reservation/update")
+ @Path(RMWSConsts.RESERVATION_UPDATE)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Override
public Response updateReservation(ReservationUpdateRequestInfo resContext,
- @Context HttpServletRequest hsr) throws AuthorizationException,
- IOException, InterruptedException {
+ @Context HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
- throw new AuthorizationException("Unable to obtain user name, "
- + "user not authenticated");
+ throw new AuthorizationException(
+ "Unable to obtain user name, " + "user not authenticated");
}
if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
String msg = "The default static user cannot carry out this operation.";
@@ -2234,16 +2200,15 @@ public class RMWebServices extends WebServices {
ReservationUpdateResponseInfo resRespInfo;
try {
- resRespInfo =
- callerUGI.doAs(
- new PrivilegedExceptionAction<ReservationUpdateResponseInfo>() {
- @Override
- public ReservationUpdateResponseInfo run() throws IOException,
- YarnException {
- rm.getClientRMService().updateReservation(reservation);
- return new ReservationUpdateResponseInfo();
- }
- });
+ resRespInfo = callerUGI
+ .doAs(new PrivilegedExceptionAction<ReservationUpdateResponseInfo>() {
+ @Override
+ public ReservationUpdateResponseInfo run()
+ throws IOException, YarnException {
+ rm.getClientRMService().updateReservation(reservation);
+ return new ReservationUpdateResponseInfo();
+ }
+ });
} catch (UndeclaredThrowableException ue) {
if (ue.getCause() instanceof YarnException) {
throw new BadRequestException(ue.getCause().getMessage());
@@ -2293,51 +2258,35 @@ public class RMWebServices extends WebServices {
int numContainers = resReqInfo.getNumContainers();
int minConcurrency = resReqInfo.getMinConcurrency();
long duration = resReqInfo.getDuration();
- ReservationRequest rr =
- ReservationRequest.newInstance(capability, numContainers,
- minConcurrency, duration);
+ ReservationRequest rr = ReservationRequest.newInstance(capability,
+ numContainers, minConcurrency, duration);
list.add(rr);
}
ReservationRequests reqs = ReservationRequests.newInstance(list, resInt);
ReservationDefinition rDef =
ReservationDefinition.newInstance(resInfo.getArrival(),
resInfo.getDeadline(), reqs, resInfo.getReservationName());
- ReservationUpdateRequest request =
- ReservationUpdateRequest.newInstance(rDef, ReservationId
- .parseReservationId(resContext.getReservationId()));
+ ReservationUpdateRequest request = ReservationUpdateRequest.newInstance(
+ rDef, ReservationId.parseReservationId(resContext.getReservationId()));
return request;
}
- /**
- * Function to delete a Reservation to the RM.
- *
- * @param resContext provides information to construct
- * the ReservationDeleteRequest
- * @param hsr the servlet request
- * @return Response containing the status code
- * @throws AuthorizationException when the user group information cannot be
- * retrieved.
- * @throws IOException when a {@link ReservationDeleteRequest} cannot be
- * created from the {@link ReservationDeleteRequestInfo}. This
- * exception is also thrown on
- * {@code ClientRMService.deleteReservation} invokation failure.
- * @throws InterruptedException if doAs action throws an InterruptedException.
- */
@POST
- @Path("/reservation/delete")
+ @Path(RMWSConsts.RESERVATION_DELETE)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Override
public Response deleteReservation(ReservationDeleteRequestInfo resContext,
- @Context HttpServletRequest hsr) throws AuthorizationException,
- IOException, InterruptedException {
+ @Context HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
- throw new AuthorizationException("Unable to obtain user name, "
- + "user not authenticated");
+ throw new AuthorizationException(
+ "Unable to obtain user name, " + "user not authenticated");
}
if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
String msg = "The default static user cannot carry out this operation.";
@@ -2349,16 +2298,15 @@ public class RMWebServices extends WebServices {
ReservationDeleteResponseInfo resRespInfo;
try {
- resRespInfo =
- callerUGI.doAs(
- new PrivilegedExceptionAction<ReservationDeleteResponseInfo>() {
- @Override
- public ReservationDeleteResponseInfo run() throws IOException,
- YarnException {
- rm.getClientRMService().deleteReservation(reservation);
- return new ReservationDeleteResponseInfo();
- }
- });
+ resRespInfo = callerUGI
+ .doAs(new PrivilegedExceptionAction<ReservationDeleteResponseInfo>() {
+ @Override
+ public ReservationDeleteResponseInfo run()
+ throws IOException, YarnException {
+ rm.getClientRMService().deleteReservation(reservation);
+ return new ReservationDeleteResponseInfo();
+ }
+ });
} catch (UndeclaredThrowableException ue) {
if (ue.getCause() instanceof YarnException) {
throw new BadRequestException(ue.getCause().getMessage());
@@ -2373,37 +2321,33 @@ public class RMWebServices extends WebServices {
private ReservationDeleteRequest createReservationDeleteRequest(
ReservationDeleteRequestInfo resContext) throws IOException {
- ReservationDeleteRequest request =
- ReservationDeleteRequest.newInstance(ReservationId
- .parseReservationId(resContext.getReservationId()));
+ ReservationDeleteRequest request = ReservationDeleteRequest.newInstance(
+ ReservationId.parseReservationId(resContext.getReservationId()));
return request;
}
- /**
- * Function to retrieve a list of all the reservations.
- */
@GET
- @Path("/reservation/list")
+ @Path(RMWSConsts.RESERVATION_LIST)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public Response listReservation(
- @QueryParam("queue") @DefaultValue("default") String queue,
- @QueryParam("reservation-id") @DefaultValue("") String reservationId,
- @QueryParam("start-time") @DefaultValue("0") long startTime,
- @QueryParam("end-time") @DefaultValue("-1") long endTime,
- @QueryParam("include-resource-allocations") @DefaultValue("false")
- boolean includeResourceAllocations, @Context HttpServletRequest hsr)
- throws Exception {
+ @QueryParam(RMWSConsts.QUEUE) @DefaultValue(DEFAULT_QUEUE) String queue,
+ @QueryParam(RMWSConsts.RESERVATION_ID) @DefaultValue(DEFAULT_RESERVATION_ID) String reservationId,
+ @QueryParam(RMWSConsts.START_TIME) @DefaultValue(DEFAULT_START_TIME) long startTime,
+ @QueryParam(RMWSConsts.END_TIME) @DefaultValue(DEFAULT_END_TIME) long endTime,
+ @QueryParam(RMWSConsts.INCLUDE_RESOURCE) @DefaultValue(DEFAULT_INCLUDE_RESOURCE) boolean includeResourceAllocations,
+ @Context HttpServletRequest hsr) throws Exception {
init();
final ReservationListRequest request = ReservationListRequest.newInstance(
- queue, reservationId, startTime, endTime, includeResourceAllocations);
+ queue, reservationId, startTime, endTime, includeResourceAllocations);
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
- throw new AuthorizationException("Unable to obtain user name, "
- + "user not authenticated");
+ throw new AuthorizationException(
+ "Unable to obtain user name, " + "user not authenticated");
}
if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
String msg = "The default static user cannot carry out this operation.";
@@ -2412,11 +2356,11 @@ public class RMWebServices extends WebServices {
ReservationListResponse resRespInfo;
try {
- resRespInfo = callerUGI.doAs(
- new PrivilegedExceptionAction<ReservationListResponse>() {
+ resRespInfo = callerUGI
+ .doAs(new PrivilegedExceptionAction<ReservationListResponse>() {
@Override
- public ReservationListResponse run() throws IOException,
- YarnException {
+ public ReservationListResponse run()
+ throws IOException, YarnException {
return rm.getClientRMService().listReservations(request);
}
});
@@ -2428,18 +2372,19 @@ public class RMWebServices extends WebServices {
throw ue;
}
- ReservationListInfo resResponse = new ReservationListInfo(resRespInfo,
- includeResourceAllocations);
+ ReservationListInfo resResponse =
+ new ReservationListInfo(resRespInfo, includeResourceAllocations);
return Response.status(Status.OK).entity(resResponse).build();
}
@GET
- @Path("/apps/{appid}/timeouts/{type}")
+ @Path(RMWSConsts.APPS_TIMEOUTS_TYPE)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public AppTimeoutInfo getAppTimeout(@Context HttpServletRequest hsr,
- @PathParam("appid") String appId, @PathParam("type") String type)
- throws AuthorizationException {
+ @PathParam(RMWSConsts.APPID) String appId,
+ @PathParam(RMWSConsts.TYPE) String type) throws AuthorizationException {
init();
RMApp app = validateAppTimeoutRequest(hsr, appId);
@@ -2478,11 +2423,12 @@ public class RMWebServices extends WebServices {
}
@GET
- @Path("/apps/{appid}/timeouts")
+ @Path(RMWSConsts.APPS_TIMEOUTS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Override
public AppTimeoutsInfo getAppTimeouts(@Context HttpServletRequest hsr,
- @PathParam("appid") String appId) throws AuthorizationException {
+ @PathParam(RMWSConsts.APPID) String appId) throws AuthorizationException {
init();
RMApp app = validateAppTimeoutRequest(hsr, appId);
@@ -2532,14 +2478,15 @@ public class RMWebServices extends WebServices {
}
@PUT
- @Path("/apps/{appid}/timeout")
+ @Path(RMWSConsts.APPS_TIMEOUT)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Override
public Response updateApplicationTimeout(AppTimeoutInfo appTimeout,
- @Context HttpServletRequest hsr, @PathParam("appid") String appId)
- throws AuthorizationException, YarnException, InterruptedException,
- IOException {
+ @Context HttpServletRequest hsr,
+ @PathParam(RMWSConsts.APPID) String appId) throws AuthorizationException,
+ YarnException, InterruptedException, IOException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
@@ -2601,8 +2548,8 @@ public class RMWebServices extends WebServices {
+ " by remote user " + userName;
return Response.status(Status.FORBIDDEN).entity(msg).build();
} else if (ye.getCause() instanceof ParseException) {
- return Response.status(Status.BAD_REQUEST)
- .entity(ye.getMessage()).build();
+ return Response.status(Status.BAD_REQUEST).entity(ye.getMessage())
+ .build();
} else {
throw ue;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/29] hadoop git commit: YARN-6634. [API] Refactor ResourceManager
WebServices to make API explicit. (Giovanni Matteo Fumarola via curino)
Posted by jh...@apache.org.
YARN-6634. [API] Refactor ResourceManager WebServices to make API explicit. (Giovanni Matteo Fumarola via curino)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/547cb53e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/547cb53e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/547cb53e
Branch: refs/heads/YARN-5734
Commit: 547cb53e8ae2f19b55f9de7670b91b009d8f3bdd
Parents: 24181f5
Author: Carlo Curino <cu...@apache.org>
Authored: Wed Jun 7 13:41:06 2017 -0700
Committer: Carlo Curino <cu...@apache.org>
Committed: Wed Jun 7 13:41:06 2017 -0700
----------------------------------------------------------------------
.../resourcemanager/webapp/RMWSConsts.java | 202 ++++
.../webapp/RMWebServiceProtocol.java | 635 +++++++++++++
.../resourcemanager/webapp/RMWebServices.java | 937 +++++++++----------
3 files changed, 1279 insertions(+), 495 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547cb53e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
new file mode 100644
index 0000000..23d4bb1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+/**
+ * Constants for {@code RMWebServiceProtocol}.
+ */
+public final class RMWSConsts {
+
+ public static final String EMPTY = "";
+ public static final String ANY = "*";
+
+ public static final String FORWARDED_FOR = "X-Forwarded-For";
+
+ // ----------------Paths for RMWebServiceProtocol----------------
+
+ /** Path for {@code RMWebServiceProtocol}. */
+ public static final String RM_WEB_SERVICE_PATH = "/ws/v1/cluster";
+
+ /** Path for {@code RMWebServiceProtocol#getClusterInfo}. */
+ public static final String INFO = "/info";
+
+ /** Path for {@code RMWebServiceProtocol#getClusterMetricsInfo}. */
+ public static final String METRICS = "/metrics";
+
+ /** Path for {@code RMWebServiceProtocol#getSchedulerInfo}. */
+ public static final String SCHEDULER = "/scheduler";
+
+ /** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */
+ public static final String SCHEDULER_LOGS = "/scheduler/logs";
+
+ /** Path for {@code RMWebServiceProtocol#getNodes}. */
+ public static final String NODES = "/nodes";
+
+ /** Path for {@code RMWebServiceProtocol#getNode}. */
+ public static final String NODES_NODEID = "/nodes/{nodeId}";
+
+ /**
+ * Path for {@code RMWebServiceProtocol#getApps} and
+ * {@code RMWebServiceProtocol#getApp}.
+ */
+ public static final String APPS = "/apps";
+
+ /** Path for {@code RMWebServiceProtocol#getActivities}. */
+ public static final String SCHEDULER_ACTIVITIES = "/scheduler/activities";
+
+ /** Path for {@code RMWebServiceProtocol#getAppActivities}. */
+ public static final String SCHEDULER_APP_ACTIVITIES =
+ "/scheduler/app-activities";
+
+ /** Path for {@code RMWebServiceProtocol#getAppStatistics}. */
+ public static final String APP_STATISTICS = "/appstatistics";
+
+ /** Path for {@code RMWebServiceProtocol#getApp}. */
+ public static final String APPS_APPID = "/apps/{appid}";
+
+ /** Path for {@code RMWebServiceProtocol#getAppAttempts}. */
+ public static final String APPS_APPID_APPATTEMPTS =
+ "/apps/{appid}/appattempts";
+
+ /** Path for {@code WebServices#getAppAttempt}. */
+ public static final String APPS_APPID_APPATTEMPTS_APPATTEMPTID =
+ "/apps/{appid}/appattempts/{appattemptid}";
+
+ /** Path for {@code WebServices#getContainers}. */
+ public static final String APPS_APPID_APPATTEMPTS_APPATTEMPTID_CONTAINERS =
+ "/apps/{appid}/appattempts/{appattemptid}/containers";
+
+ /** Path for {@code RMWebServiceProtocol#getNodeToLabels}. */
+ public static final String GET_NODE_TO_LABELS = "/get-node-to-labels";
+
+ /** Path for {@code RMWebServiceProtocol#getLabelsToNodes}. */
+ public static final String LABEL_MAPPINGS = "/label-mappings";
+
+ /** Path for {@code RMWebServiceProtocol#replaceLabelsOnNodes}. */
+ public static final String REPLACE_NODE_TO_LABELS = "/replace-node-to-labels";
+
+ /** Path for {@code RMWebServiceProtocol#replaceLabelsOnNode}. */
+ public static final String NODES_NODEID_REPLACE_LABELS =
+ "/nodes/{nodeId}/replace-labels";
+
+ /** Path for {@code RMWebServiceProtocol#getClusterNodeLabels}. */
+ public static final String GET_NODE_LABELS = "/get-node-labels";
+
+ /** Path for {@code RMWebServiceProtocol#addToClusterNodeLabels}. */
+ public static final String ADD_NODE_LABELS = "/add-node-labels";
+
+ /** Path for {@code RMWebServiceProtocol#removeFromCluserNodeLabels}. */
+ public static final String REMOVE_NODE_LABELS = "/remove-node-labels";
+
+ /** Path for {@code RMWebServiceProtocol#getLabelsOnNode}. */
+ public static final String NODES_NODEID_GETLABELS =
+ "/nodes/{nodeId}/get-labels";
+
+ /**
+ * Path for {@code RMWebServiceProtocol#getAppPriority} and
+ * {@code RMWebServiceProtocol#updateApplicationPriority}.
+ */
+ public static final String APPS_APPID_PRIORITY = "/apps/{appid}/priority";
+
+ /**
+ * Path for {@code RMWebServiceProtocol#getAppQueue} and
+ * {@code RMWebServiceProtocol#updateAppQueue}.
+ */
+ public static final String APPS_APPID_QUEUE = "/apps/{appid}/queue";
+
+ /** Path for {@code RMWebServiceProtocol#createNewApplication}. */
+ public static final String APPS_NEW_APPLICATION = "/apps/new-application";
+
+ /**
+ * Path for {@code RMWebServiceProtocol#getAppState} and
+ * {@code RMWebServiceProtocol#updateAppState}.
+ */
+ public static final String APPS_APPID_STATE = "/apps/{appid}/state";
+
+ /**
+ * Path for {@code RMWebServiceProtocol#postDelegationToken} and
+ * {@code RMWebServiceProtocol#cancelDelegationToken}.
+ */
+ public static final String DELEGATION_TOKEN = "/delegation-token";
+
+ /** Path for {@code RMWebServiceProtocol#postDelegationTokenExpiration}. */
+ public static final String DELEGATION_TOKEN_EXPIRATION =
+ "/delegation-token/expiration";
+
+ /** Path for {@code RMWebServiceProtocol#createNewReservation}. */
+ public static final String RESERVATION_NEW = "/reservation/new-reservation";
+
+ /** Path for {@code RMWebServiceProtocol#submitReservation}. */
+ public static final String RESERVATION_SUBMIT = "/reservation/submit";
+
+ /** Path for {@code RMWebServiceProtocol#updateReservation}. */
+ public static final String RESERVATION_UPDATE = "/reservation/update";
+
+ /** Path for {@code RMWebServiceProtocol#deleteReservation}. */
+ public static final String RESERVATION_DELETE = "/reservation/delete";
+
+ /** Path for {@code RMWebServiceProtocol#listReservation}. */
+ public static final String RESERVATION_LIST = "/reservation/list";
+
+ /** Path for {@code RMWebServiceProtocol#getAppTimeout}. */
+ public static final String APPS_TIMEOUTS_TYPE =
+ "/apps/{appid}/timeouts/{type}";
+
+ /**
+ * Path for {@code RMWebServiceProtocol#getAppTimeouts}.
+ */
+ public static final String APPS_TIMEOUTS = "/apps/{appid}/timeouts";
+
+ /**
+ * Path for {@code RMWebServiceProtocol#updateApplicationTimeout}.
+ */
+ public static final String APPS_TIMEOUT = "/apps/{appid}/timeout";
+
+ // ----------------QueryParams for RMWebServiceProtocol----------------
+
+ public static final String TIME = "time";
+ public static final String STATES = "states";
+ public static final String NODEID = "nodeId";
+ public static final String STATE = "state";
+ public static final String FINAL_STATUS = "finalStatus";
+ public static final String USER = "user";
+ public static final String QUEUE = "queue";
+ public static final String LIMIT = "limit";
+ public static final String STARTED_TIME_BEGIN = "startedTimeBegin";
+ public static final String STARTED_TIME_END = "startedTimeEnd";
+ public static final String FINISHED_TIME_BEGIN = "finishedTimeBegin";
+ public static final String FINISHED_TIME_END = "finishedTimeEnd";
+ public static final String APPLICATION_TYPES = "applicationTypes";
+ public static final String APPLICATION_TAGS = "applicationTags";
+ public static final String APP_ID = "appId";
+ public static final String MAX_TIME = "maxTime";
+ public static final String APPATTEMPTID = "appattemptid";
+ public static final String APPID = "appid";
+ public static final String LABELS = "labels";
+ public static final String RESERVATION_ID = "reservation-id";
+ public static final String START_TIME = "start-time";
+ public static final String END_TIME = "end-time";
+ public static final String INCLUDE_RESOURCE = "include-resource-allocations";
+ public static final String TYPE = "type";
+
+ private RMWSConsts() {
+ // not called
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547cb53e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
new file mode 100644
index 0000000..6dd9c41
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
@@ -0,0 +1,635 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.Set;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.core.Response;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
+
+/**
+ * <p>
+ * The protocol between clients and the <code>ResourceManager</code> to
+ * submit/abort jobs and to get information on applications, cluster metrics,
+ * nodes, queues, ACLs and reservations via REST calls.
+ * </p>
+ *
+ * The WebService is reachable by using {@link RMWSConsts#RM_WEB_SERVICE_PATH}
+ */
+@Private
+@Evolving
+public interface RMWebServiceProtocol {
+
+ /**
+ * This method retrieves the cluster information, and it is reachable by using
+ * {@link RMWSConsts#INFO}.
+ *
+ * @return the cluster information
+ */
+ ClusterInfo get();
+
+ /**
+ * This method retrieves the cluster information, and it is reachable by using
+ * {@link RMWSConsts#INFO}.
+ *
+ * @return the cluster information
+ */
+ ClusterInfo getClusterInfo();
+
+ /**
+ * This method retrieves the cluster metrics information, and it is reachable
+ * by using {@link RMWSConsts#METRICS}.
+ *
+ * @see ApplicationClientProtocol#getClusterMetrics
+ * @return the cluster metrics information
+ */
+ ClusterMetricsInfo getClusterMetricsInfo();
+
+ /**
+ * This method retrieves the current scheduler status, and it is reachable by
+ * using {@link RMWSConsts#SCHEDULER}.
+ *
+ * @return the current scheduler status
+ */
+ SchedulerTypeInfo getSchedulerInfo();
+
+ /**
+ * This method dumps the scheduler logs for the time got in input, and it is
+ * reachable by using {@link RMWSConsts#SCHEDULER_LOGS}.
+ *
+ * @param time the period of time
+ * @param hsr the servlet request
+ * @return the result of the operation
+ * @throws IOException when it cannot create dump log file
+ */
+ String dumpSchedulerLogs(String time, HttpServletRequest hsr)
+ throws IOException;
+
+ /**
+ * This method retrieves all the nodes information in the cluster, and it is
+ * reachable by using {@link RMWSConsts#NODES}.
+ *
+ * @see ApplicationClientProtocol#getClusterNodes
+ * @param states the states we want to filter
+ * @return all nodes in the cluster. If the states param is given, returns all
+ * nodes that are in the comma-separated list of states
+ */
+ NodesInfo getNodes(String states);
+
+ /**
+ * This method retrieves a specific node information, and it is reachable by
+ * using {@link RMWSConsts#NODES_NODEID}.
+ *
+ * @param nodeId the node we want to retrieve the information
+ * @return the information about the node in input
+ */
+ NodeInfo getNode(String nodeId);
+
+ /**
+ * This method retrieves all the app reports in the cluster, and it is
+ * reachable by using {@link RMWSConsts#APPS}.
+ *
+ * @see ApplicationClientProtocol#getApplications
+ * @param hsr the servlet request
+ * @param stateQuery right now the stateQuery is deprecated
+ * @param statesQuery filter the result by states
+ * @param finalStatusQuery filter the result by final states
+ * @param userQuery filter the result by user
+ * @param queueQuery filter the result by queue
+ * @param count set a limit of the result
+ * @param startedBegin filter the result by started begin time
+ * @param startedEnd filter the result by started end time
+ * @param finishBegin filter the result by finish begin time
+ * @param finishEnd filter the result by finish end time
+ * @param applicationTypes filter the result by types
+ * @param applicationTags filter the result by tags
+ * @return all apps in the cluster
+ */
+ @SuppressWarnings("checkstyle:parameternumber")
+ AppsInfo getApps(HttpServletRequest hsr, String stateQuery,
+ Set<String> statesQuery, String finalStatusQuery, String userQuery,
+ String queueQuery, String count, String startedBegin, String startedEnd,
+ String finishBegin, String finishEnd, Set<String> applicationTypes,
+ Set<String> applicationTags);
+
+ /**
+ * This method retrieve all the activities in a specific node, and it is
+ * reachable by using {@link RMWSConsts#SCHEDULER_ACTIVITIES}.
+ *
+ * @param hsr the servlet request
+ * @param nodeId the node we want to retrieve the activities
+ * @return all the activities in the specific node
+ */
+ ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId);
+
+ /**
+ * This method retrieves all the activities for a specific app for a specific
+ * period of time, and it is reachable by using
+ * {@link RMWSConsts#SCHEDULER_APP_ACTIVITIES}.
+ *
+ * @param hsr the servlet request
+ * @param appId the applicationId we want to retrieve the activities
+ * @param time for how long we want to retrieve the activities
+ * @return all the activities about a specific app for a specific time
+ */
+ AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId,
+ String time);
+
+ /**
+ * This method retrieves all the statistics for a specific app, and it is
+ * reachable by using {@link RMWSConsts#APP_STATISTICS}.
+ *
+ * @param hsr the servlet request
+ * @param stateQueries filter the result by states
+ * @param typeQueries filter the result by type names
+ * @return the application's statistics for specific states and types
+ */
+ ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr,
+ Set<String> stateQueries, Set<String> typeQueries);
+
+ /**
+ * This method retrieves the report for a specific app, and it is reachable by
+ * using {@link RMWSConsts#APPS_APPID}.
+ *
+ * @see ApplicationClientProtocol#getApplicationReport
+ * @param hsr the servlet request
+ * @param appId the Id of the application we want the report
+ * @return the app report for a specific application
+ */
+ AppInfo getApp(HttpServletRequest hsr, String appId);
+
+ /**
+ * This method retrieves the state for a specific app, and it is reachable by
+ * using {@link RMWSConsts#APPS_APPID_STATE}.
+ *
+ * @param hsr the servlet request
+ * @param appId the Id of the application we want the state
+ * @return the state for a specific application
+ * @throws AuthorizationException if the user is not authorized
+ */
+ AppState getAppState(HttpServletRequest hsr, String appId)
+ throws AuthorizationException;
+
+ /**
+ * This method updates the state of the app in input, and it is reachable by
+ * using {@link RMWSConsts#APPS_APPID_STATE}.
+ *
+ * @param targetState the target state for the app
+ * @param hsr the servlet request
+ * @param appId the Id of the application we want to update the state
+ * @return Response containing the status code
+ * @throws AuthorizationException if the user is not authorized to invoke this
+ * method
+ * @throws YarnException if app does not exist
+ * @throws InterruptedException if interrupted
+ * @throws IOException if doAs action throws an IOException
+ */
+ Response updateAppState(AppState targetState, HttpServletRequest hsr,
+ String appId) throws AuthorizationException, YarnException,
+ InterruptedException, IOException;
+
+ /**
+ * This method retrieves all the node labels with the respective nodes in the
+ * cluster, and it is reachable by using
+ * {@link RMWSConsts#GET_NODE_TO_LABELS}.
+ *
+ * @see ApplicationClientProtocol#getNodeToLabels
+ * @param hsr the servlet request
+ * @return all the nodes within a node label
+ * @throws IOException if an IOException happened
+ */
+ NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr) throws IOException;
+
+ /**
+ * This method retrieves all the node within multiple node labels in the
+ * cluster, and it is reachable by using {@link RMWSConsts#LABEL_MAPPINGS}.
+ *
+ * @see ApplicationClientProtocol#getLabelsToNodes
+ * @param labels filter the result by node labels
+ * @return all the nodes within multiple node labels
+ * @throws IOException if an IOException happened
+ */
+ LabelsToNodesInfo getLabelsToNodes(Set<String> labels) throws IOException;
+
+ /**
+ * This method replaces all the node labels for specific nodes, and it is
+ * reachable by using {@link RMWSConsts#REPLACE_NODE_TO_LABELS}.
+ *
+ * @see ResourceManagerAdministrationProtocol#replaceLabelsOnNode
+ * @param newNodeToLabels the list of new labels
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws Exception if an exception happened
+ */
+ Response replaceLabelsOnNodes(NodeToLabelsEntryList newNodeToLabels,
+ HttpServletRequest hsr) throws Exception;
+
+ /**
+ * This method replaces all the node labels for specific node, and it is
+ * reachable by using {@link RMWSConsts#NODES_NODEID_REPLACE_LABELS}.
+ *
+ * @see ResourceManagerAdministrationProtocol#replaceLabelsOnNode
+ * @param newNodeLabelsName the list of new labels
+ * @param hsr the servlet request
+ * @param nodeId the node we want to replace the node labels
+ * @return Response containing the status code
+ * @throws Exception if an exception happened
+ */
+ Response replaceLabelsOnNode(Set<String> newNodeLabelsName,
+ HttpServletRequest hsr, String nodeId) throws Exception;
+
+ /**
+ * This method retrieves all the node labels in the cluster, and it is
+ * reachable by using {@link RMWSConsts#GET_NODE_LABELS}.
+ *
+ * @see ApplicationClientProtocol#getClusterNodeLabels
+ * @param hsr the servlet request
+ * @return all the node labels in the cluster
+ * @throws IOException if an IOException happened
+ */
+ NodeLabelsInfo getClusterNodeLabels(HttpServletRequest hsr)
+ throws IOException;
+
+ /**
+ * This method adds specific node labels for specific nodes, and it is
+ * reachable by using {@link RMWSConsts#ADD_NODE_LABELS}.
+ *
+ * @see ResourceManagerAdministrationProtocol#addToClusterNodeLabels
+ * @param newNodeLabels the node labels to add
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws Exception in case of bad request
+ */
+ Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels,
+ HttpServletRequest hsr) throws Exception;
+
+ /**
+ * This method removes all the node labels for specific nodes, and it is
+ * reachable by using {@link RMWSConsts#REMOVE_NODE_LABELS}.
+ *
+ * @see ResourceManagerAdministrationProtocol#removeFromClusterNodeLabels
+ * @param oldNodeLabels the node labels to remove
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws Exception in case of bad request
+ */
+ Response removeFromCluserNodeLabels(Set<String> oldNodeLabels,
+ HttpServletRequest hsr) throws Exception;
+
+ /**
+ * This method retrieves all the node labels for specific node, and it is
+ * reachable by using {@link RMWSConsts#NODES_NODEID_GETLABELS}.
+ *
+ * @param hsr the servlet request
+ * @param nodeId the node we want to get all the node labels
+ * @return all the labels for a specific node.
+ * @throws IOException if an IOException happened
+ */
+ NodeLabelsInfo getLabelsOnNode(HttpServletRequest hsr, String nodeId)
+ throws IOException;
+
+ /**
+ * This method retrieves the priority for a specific app, and it is reachable
+ * by using {@link RMWSConsts#APPS_APPID_PRIORITY}.
+ *
+ * @param hsr the servlet request
+ * @param appId the app we want to get the priority
+ * @return the priority for a specific application
+ * @throws AuthorizationException in case of the user is not authorized
+ */
+ AppPriority getAppPriority(HttpServletRequest hsr, String appId)
+ throws AuthorizationException;
+
+ /**
+ * This method updates the priority for a specific application, and it is
+ * reachable by using {@link RMWSConsts#APPS_APPID_PRIORITY}.
+ *
+ * @param targetPriority the priority we want to set for the app
+ * @param hsr the servlet request
+ * @param appId the application we want to update its priority
+ * @return Response containing the status code
+ * @throws AuthorizationException if the user is not authenticated
+ * @throws YarnException if the target is null
+ * @throws IOException if the update fails.
+ * @throws InterruptedException if interrupted.
+ */
+ Response updateApplicationPriority(AppPriority targetPriority,
+ HttpServletRequest hsr, String appId) throws AuthorizationException,
+ YarnException, InterruptedException, IOException;
+
+ /**
+ * This method retrieves the queue for a specific app, and it is reachable by
+ * using {@link RMWSConsts#APPS_APPID_QUEUE}.
+ *
+ * @param hsr the servlet request
+ * @param appId the application we want to retrieve its queue
+ * @return the Queue for a specific application.
+ * @throws AuthorizationException if the user is not authenticated
+ */
+ AppQueue getAppQueue(HttpServletRequest hsr, String appId)
+ throws AuthorizationException;
+
+ /**
+ * This method updates the queue for a specific application, and it is
+ * reachable by using {@link RMWSConsts#APPS_APPID_QUEUE}.
+ *
+ * @param targetQueue the queue we want to set
+ * @param hsr the servlet request
+ * @param appId the application we want to change its queue
+ * @return Response containing the status code
+ * @throws AuthorizationException if the user is not authenticated
+ * @throws YarnException if the app is not found
+ * @throws IOException if the update fails.
+ * @throws InterruptedException if interrupted.
+ */
+ Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr,
+ String appId) throws AuthorizationException, YarnException,
+ InterruptedException, IOException;
+
+ /**
+ * Generates a new ApplicationId which is then sent to the client. This method
+ * is reachable by using {@link RMWSConsts#APPS_NEW_APPLICATION}.
+ *
+ * @see ApplicationClientProtocol#getNewApplication
+ *
+ * @param hsr the servlet request
+ * @return Response containing the app id and the maximum resource
+ * capabilities
+ * @throws AuthorizationException if the user is not authorized to invoke this
+ * method
+ * @throws IOException if the creation fails
+ * @throws InterruptedException if interrupted
+ */
+ Response createNewApplication(HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException;
+
+ /**
+ * Function to submit an app to the RM. This method is reachable by using
+ * {@link RMWSConsts#APPS}.
+ *
+ * @see ApplicationClientProtocol#submitApplication
+ *
+ * @param newApp structure containing information to construct the
+ * ApplicationSubmissionContext
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws AuthorizationException if the user is not authorized to invoke this
+ * method
+ * @throws IOException if the submission failed
+ * @throws InterruptedException if interrupted
+ */
+ Response submitApplication(ApplicationSubmissionContextInfo newApp,
+ HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException;
+
+ /**
+ * This method posts a delegation token from the client, and it is reachable
+ * by using {@link RMWSConsts#DELEGATION_TOKEN}.
+ *
+ * @see ApplicationBaseProtocol#getDelegationToken
+ * @param tokenData the token to delegate
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws AuthorizationException if Kerberos auth failed
+ * @throws IOException if the delegation failed
+ * @throws InterruptedException if interrupted
+ * @throws Exception in case of bad request
+ */
+ Response postDelegationToken(DelegationToken tokenData,
+ HttpServletRequest hsr) throws AuthorizationException, IOException,
+ InterruptedException, Exception;
+
+ /**
+ * This method updates the expiration for a delegation token from the client,
+ * and it is reachable by using
+ * {@link RMWSConsts#DELEGATION_TOKEN_EXPIRATION}.
+ *
+ * @see ApplicationBaseProtocol#renewDelegationToken
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws AuthorizationException if Kerberos auth failed
+ * @throws IOException if the delegation failed
+ * @throws Exception in case of bad request
+ */
+ Response postDelegationTokenExpiration(HttpServletRequest hsr)
+ throws AuthorizationException, IOException, Exception;
+
+ /**
+ * This method cancel the delegation token from the client, and it is
+ * reachable by using {@link RMWSConsts#DELEGATION_TOKEN}.
+ *
+ * @see ApplicationBaseProtocol#cancelDelegationToken
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws AuthorizationException if Kerberos auth failed
+ * @throws IOException if the delegation failed
+ * @throws InterruptedException if interrupted
+ * @throws Exception in case of bad request
+ */
+ Response cancelDelegationToken(HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException,
+ Exception;
+
+ /**
+ * Generates a new ReservationId which is then sent to the client. This method
+ * is reachable by using {@link RMWSConsts#RESERVATION_NEW}.
+ *
+ * @see ApplicationClientProtocol#getNewReservation
+ *
+ * @param hsr the servlet request
+ * @return Response containing the app id and the maximum resource
+ * capabilities
+ * @throws AuthorizationException if the user is not authorized to invoke this
+ * method.
+ * @throws IOException if creation failed
+ * @throws InterruptedException if interrupted
+ */
+ Response createNewReservation(HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException;
+
+ /**
+ * Function to submit a Reservation to the RM.This method is reachable by
+ * using {@link RMWSConsts#RESERVATION_SUBMIT}.
+ *
+ * @see ApplicationClientProtocol#submitReservation
+ *
+ * @param resContext provides information to construct the
+ * ReservationSubmissionRequest
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws AuthorizationException if the user is not authorized to invoke this
+ * method
+ * @throws IOException if creation failed
+ * @throws InterruptedException if interrupted
+ */
+ Response submitReservation(ReservationSubmissionRequestInfo resContext,
+ HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException;
+
+ /**
+ * Function to update a Reservation to the RM. This method is reachable by
+ * using {@link RMWSConsts#RESERVATION_UPDATE}.
+ *
+ * @see ApplicationClientProtocol#updateReservation
+ *
+ * @param resContext provides information to construct the
+ * ReservationUpdateRequest
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws AuthorizationException if the user is not authorized to invoke this
+ * method
+ * @throws IOException if the operation failed
+ * @throws InterruptedException if interrupted
+ */
+ Response updateReservation(ReservationUpdateRequestInfo resContext,
+ HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException;
+
+ /**
+ * Function to delete a Reservation to the RM. This method is reachable by
+ * using {@link RMWSConsts#RESERVATION_DELETE}.
+ *
+ * @see ApplicationClientProtocol#deleteReservation
+ *
+ * @param resContext provides information to construct the
+ * ReservationDeleteRequest
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws AuthorizationException when the user group information cannot be
+ * retrieved.
+ * @throws IOException when a {@link ReservationDeleteRequest} cannot be
+ * created from the {@link ReservationDeleteRequestInfo}. This
+ * exception is also thrown on
+ * {@code ClientRMService.deleteReservation} invokation failure.
+ * @throws InterruptedException if doAs action throws an InterruptedException.
+ */
+ Response deleteReservation(ReservationDeleteRequestInfo resContext,
+ HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException;
+
+ /**
+ * Function to retrieve a list of all the reservations. This method is
+ * reachable by using {@link RMWSConsts#RESERVATION_LIST}.
+ *
+ * @see ApplicationClientProtocol#listReservations
+ * @param queue filter the result by queue
+ * @param reservationId filter the result by reservationId
+ * @param startTime filter the result by start time
+ * @param endTime filter the result by end time
+ * @param includeResourceAllocations true if the resource allocation should be
+ * in the result, false otherwise
+ * @param hsr the servlet request
+ * @return Response containing the status code
+ * @throws Exception in case of bad request
+ */
+ Response listReservation(String queue, String reservationId, long startTime,
+ long endTime, boolean includeResourceAllocations, HttpServletRequest hsr)
+ throws Exception;
+
+ /**
+ * This method retrieves the timeout information for a specific app with a
+ * specific type, and it is reachable by using
+ * {@link RMWSConsts#APPS_TIMEOUTS_TYPE}.
+ *
+ * @param hsr the servlet request
+ * @param appId the application we want to get the timeout
+ * @param type the type of the timeouts
+ * @return the timeout for a specific application with a specific type.
+ * @throws AuthorizationException if the user is not authorized
+ */
+ AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, String appId,
+ String type) throws AuthorizationException;
+
+ /**
+ * This method retrieves the timeout information for a specific app, and it is
+ * reachable by using {@link RMWSConsts#APPS_TIMEOUTS}.
+ *
+ * @param hsr the servlet request
+ * @param appId the application we want to get the timeouts
+ * @return the timeouts for a specific application
+ * @throws AuthorizationException if the user is not authorized
+ */
+ AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId)
+ throws AuthorizationException;
+
+ /**
+ * This method updates the timeout information for a specific app, and it is
+ * reachable by using {@link RMWSConsts#APPS_TIMEOUT}.
+ *
+ * @see ApplicationClientProtocol#updateApplicationTimeouts
+ * @param appTimeout the appTimeoutInfo
+ * @param hsr the servlet request
+ * @param appId the application we want to update
+ * @return Response containing the status code
+ * @throws AuthorizationException if the user is not authorized to invoke this
+ * method
+ * @throws YarnException in case of bad request
+ * @throws IOException if the operation failed
+ * @throws InterruptedException if interrupted
+ */
+ Response updateApplicationTimeout(AppTimeoutInfo appTimeout,
+ HttpServletRequest hsr, String appId) throws AuthorizationException,
+ YarnException, InterruptedException, IOException;
+
+ /**
+ * This method retrieves all the attempts information for a specific app, and
+ * it is reachable by using {@link RMWSConsts#APPS_APPID_APPATTEMPTS}.
+ *
+ * @see ApplicationBaseProtocol#getApplicationAttempts
+ * @param hsr the servlet request
+ * @param appId the application we want to get the attempts
+ * @return all the attempts info for a specific application
+ */
+ AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId);
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/29] hadoop git commit: HADOOP-14500. Azure:
TestFileSystemOperationExceptionHandling{,
MultiThreaded} fails. Contributed by Rajesh Balamohan
Posted by jh...@apache.org.
HADOOP-14500. Azure: TestFileSystemOperationExceptionHandling{,MultiThreaded} fails. Contributed by Rajesh Balamohan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df35e4cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df35e4cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df35e4cc
Branch: refs/heads/YARN-5734
Commit: df35e4cc3b0c67bcaa3d809c6cb5a0a2d82986af
Parents: 547cb53
Author: Mingliang Liu <li...@apache.org>
Authored: Wed Jun 7 13:54:52 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed Jun 7 13:54:52 2017 -0700
----------------------------------------------------------------------
.../hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java | 1 +
.../TestFileSystemOperationsExceptionHandlingMultiThreaded.java | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df35e4cc/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java
index f5a1754..9ac25dd 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java
@@ -94,6 +94,7 @@ public class TestFileSystemOperationExceptionHandling
AzureBlobStorageTestAccount testAccount = createTestAccount();
setupInputStreamToTest(testAccount);
inputStream.seek(5);
+ inputStream.read();
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df35e4cc/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java
index 4c56795..1cd18ee 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java
@@ -91,6 +91,7 @@ public class TestFileSystemOperationsExceptionHandlingMultiThreaded
renameThread.join();
inputStream.seek(5);
+ inputStream.read();
}
@Test(expected=FileNotFoundException.class)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/29] hadoop git commit: HDFS-11929. Document missing processor of
hdfs oiv_legacy command. Contributed by LiXin Ge.
Posted by jh...@apache.org.
HDFS-11929. Document missing processor of hdfs oiv_legacy command. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ec71639
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ec71639
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ec71639
Branch: refs/heads/YARN-5734
Commit: 5ec71639cf41858c687781445b8360d50cad5b55
Parents: 70fc674
Author: Yiqun Lin <yq...@apache.org>
Authored: Wed Jun 7 13:42:13 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Wed Jun 7 13:42:13 2017 +0800
----------------------------------------------------------------------
.../hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +-
.../hadoop-hdfs/src/site/markdown/HdfsImageViewer.md | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec71639/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
index c542d90..46a9c75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
@@ -94,7 +94,7 @@ public class OfflineImageViewer {
+ "Optional command line arguments:\n"
+ "-p,--processor <arg> Select which type of processor to apply\n"
+ " against image file."
- + " (Ls|XML|Delimited|Indented|FileDistribution).\n"
+ + " (Ls|XML|Delimited|Indented|FileDistribution|NameDistribution).\n"
+ "-h,--help Display usage information and exit\n"
+ "-printToScreen For processors that write to a file, also\n"
+ " output to screen. On large image files this\n"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec71639/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 6765a8f..ffffee9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -233,7 +233,7 @@ Usage: `hdfs oiv_legacy [OPTIONS] -i INPUT_FILE -o OUTPUT_FILE`
| COMMAND\_OPTION | Description |
|:---- |:---- |
-| `-p`\|`--processor` *processor* | Specify the image processor to apply against the image file. Valid options are Ls (default), XML, Delimited, Indented, and FileDistribution. |
+| `-p`\|`--processor` *processor* | Specify the image processor to apply against the image file. Valid options are Ls (default), XML, Delimited, Indented, FileDistribution and NameDistribution. |
| `-maxSize` *size* | Specify the range [0, maxSize] of file sizes to be analyzed in bytes (128GB by default). This option is used with FileDistribution processor. |
| `-step` *size* | Specify the granularity of the distribution in bytes (2MB by default). This option is used with FileDistribution processor. |
| `-format` | Format the output result in a human-readable fashion rather than a number of bytes. (false by default). This option is used with FileDistribution processor. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec71639/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
index 10d98b2..9baadc0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
@@ -238,7 +238,7 @@ Due to the internal layout changes introduced by the ProtocolBuffer-based fsimag
|:---- |:---- |
| `-i`\|`--inputFile` *input file* | Specify the input fsimage file to process. Required. |
| `-o`\|`--outputFile` *output file* | Specify the output filename, if the specified output processor generates one. If the specified file already exists, it is silently overwritten. Required. |
-| `-p`\|`--processor` *processor* | Specify the image processor to apply against the image file. Valid options are Ls (default), XML, Delimited, Indented, and FileDistribution. |
+| `-p`\|`--processor` *processor* | Specify the image processor to apply against the image file. Valid options are Ls (default), XML, Delimited, Indented, FileDistribution and NameDistribution. |
| `-maxSize` *size* | Specify the range [0, maxSize] of file sizes to be analyzed in bytes (128GB by default). This option is used with FileDistribution processor. |
| `-step` *size* | Specify the granularity of the distribution in bytes (2MB by default). This option is used with FileDistribution processor. |
| `-format` | Format the output result in a human-readable fashion rather than a number of bytes. (false by default). This option is used with FileDistribution processor. |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/29] hadoop git commit: HADOOP-14485. Redundant 'final' modifier
in try-with-resources statement. Contributed by wenxin he.
Posted by jh...@apache.org.
HADOOP-14485. Redundant 'final' modifier in try-with-resources statement. Contributed by wenxin he.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19ef3a81
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19ef3a81
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19ef3a81
Branch: refs/heads/YARN-5734
Commit: 19ef3a81f8b90579b4a7a95839d0c3ebdd56349c
Parents: 855e047
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Tue Jun 6 23:11:47 2017 +0800
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Tue Jun 6 23:11:47 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/example/ITUseMiniCluster.java | 8 ++++----
.../src/test/java/org/apache/hadoop/ipc/TestIPC.java | 2 +-
.../org/apache/hadoop/metrics2/lib/TestRollingAverages.java | 2 +-
.../test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java | 2 +-
.../org/apache/hadoop/hdfs/server/balancer/TestBalancer.java | 2 +-
.../hdfs/server/datanode/TestDataNodeFaultInjector.java | 2 +-
.../apache/hadoop/yarn/client/api/impl/TestYarnClient.java | 2 +-
7 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ef3a81/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java b/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java
index 3f560d0..5fcbe13 100644
--- a/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java
+++ b/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java
@@ -82,7 +82,7 @@ public class ITUseMiniCluster {
@Test
public void useHdfsFileSystem() throws IOException {
- try (final FileSystem fs = cluster.getFileSystem()) {
+ try (FileSystem fs = cluster.getFileSystem()) {
simpleReadAfterWrite(fs);
}
}
@@ -94,10 +94,10 @@ public class ITUseMiniCluster {
throw new IOException("Mkdirs failed to create " +
TEST_PATH);
}
- try (final FSDataOutputStream out = fs.create(path)) {
+ try (FSDataOutputStream out = fs.create(path)) {
out.writeUTF(TEXT);
}
- try (final FSDataInputStream in = fs.open(path)) {
+ try (FSDataInputStream in = fs.open(path)) {
final String result = in.readUTF();
Assert.assertEquals("Didn't read back text we wrote.", TEXT, result);
}
@@ -105,7 +105,7 @@ public class ITUseMiniCluster {
@Test
public void useWebHDFS() throws IOException, URISyntaxException {
- try (final FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(
+ try (FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(
cluster.getConfiguration(0), WebHdfsConstants.WEBHDFS_SCHEME)) {
simpleReadAfterWrite(fs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ef3a81/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
index 611000d..1d47473 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
@@ -1425,7 +1425,7 @@ public class TestIPC {
}
private void checkVersionMismatch() throws IOException {
- try (final ServerSocket listenSocket = new ServerSocket()) {
+ try (ServerSocket listenSocket = new ServerSocket()) {
listenSocket.bind(null);
InetSocketAddress addr =
(InetSocketAddress) listenSocket.getLocalSocketAddress();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ef3a81/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestRollingAverages.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestRollingAverages.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestRollingAverages.java
index 44202e7..6ea3d26 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestRollingAverages.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestRollingAverages.java
@@ -42,7 +42,7 @@ public class TestRollingAverages {
public void testRollingAveragesEmptyRollover() throws Exception {
final MetricsRecordBuilder rb = mockMetricsRecordBuilder();
/* 5s interval and 2 windows */
- try (final RollingAverages rollingAverages =
+ try (RollingAverages rollingAverages =
new RollingAverages(5000, 2)) {
/* Check it initially */
rollingAverages.snapshot(rb, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ef3a81/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index 3d4cc72..d631b68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -254,7 +254,7 @@ public class TestMiniDFSCluster {
@Test
public void testClusterNoStorageTypeSetForDatanodes() throws IOException {
final Configuration conf = new HdfsConfiguration();
- try (final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(3).build()) {
cluster.waitActive();
ArrayList<DataNode> dataNodes = cluster.getDataNodes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ef3a81/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index dd28914..016a233 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -1847,7 +1847,7 @@ public class TestBalancer {
for(int i = 0; i < lengths.length; i++) {
final long size = lengths[i];
final Path p = new Path("/file" + i + "_size" + size);
- try(final OutputStream out = dfs.create(p)) {
+ try(OutputStream out = dfs.create(p)) {
for(int j = 0; j < size; j++) {
out.write(j);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ef3a81/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java
index fe65429..1507844 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java
@@ -153,7 +153,7 @@ public class TestDataNodeFaultInjector {
cluster.waitActive();
final FileSystem fs = cluster.getFileSystem();
- try (final FSDataOutputStream out = fs
+ try (FSDataOutputStream out = fs
.create(new Path(baseDir, "test.data"), (short) 2)) {
out.write(0x31);
out.hflush();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ef3a81/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 9adc187..7d7272a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -303,7 +303,7 @@ public class TestYarnClient {
int pollIntervalMs = 1000;
conf.setLong(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
pollIntervalMs);
- try (final YarnClient client = new MockYarnClient()) {
+ try (YarnClient client = new MockYarnClient()) {
client.init(conf);
client.start();
// Submit the application and then interrupt it while its waiting
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/29] hadoop git commit: YARN-5946: Create YarnConfigurationStore
interface and InMemoryConfigurationStore class. Contributed by Jonathan Hung
Posted by jh...@apache.org.
YARN-5946: Create YarnConfigurationStore interface and
InMemoryConfigurationStore class. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdd41333
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdd41333
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdd41333
Branch: refs/heads/YARN-5734
Commit: fdd41333a86f9a5d7fa5b4382d11a18a2464d391
Parents: c3cedd7
Author: Xuan <xg...@apache.org>
Authored: Fri Feb 24 15:58:12 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Thu Jun 8 20:03:59 2017 -0700
----------------------------------------------------------------------
.../conf/InMemoryConfigurationStore.java | 86 +++++++++++
.../capacity/conf/YarnConfigurationStore.java | 154 +++++++++++++++++++
.../conf/TestYarnConfigurationStore.java | 70 +++++++++
3 files changed, 310 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdd41333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
new file mode 100644
index 0000000..a208fb9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A default implementation of {@link YarnConfigurationStore}. Doesn't offer
+ * persistent configuration storage, just stores the configuration in memory.
+ */
+public class InMemoryConfigurationStore implements YarnConfigurationStore {
+
+ private Configuration schedConf;
+ private LinkedList<LogMutation> pendingMutations;
+ private long pendingId;
+
+ @Override
+ public void initialize(Configuration conf, Configuration schedConf) {
+ this.schedConf = schedConf;
+ this.pendingMutations = new LinkedList<>();
+ this.pendingId = 0;
+ }
+
+ @Override
+ public synchronized long logMutation(LogMutation logMutation) {
+ logMutation.setId(++pendingId);
+ pendingMutations.add(logMutation);
+ return pendingId;
+ }
+
+ @Override
+ public synchronized boolean confirmMutation(long id, boolean isValid) {
+ LogMutation mutation = pendingMutations.poll();
+ // If confirmMutation is called out of order, discard mutations until id
+ // is reached.
+ while (mutation != null) {
+ if (mutation.getId() == id) {
+ if (isValid) {
+ Map<String, String> mutations = mutation.getUpdates();
+ for (Map.Entry<String, String> kv : mutations.entrySet()) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ }
+ return true;
+ }
+ mutation = pendingMutations.poll();
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized Configuration retrieve() {
+ return schedConf;
+ }
+
+ @Override
+ public synchronized List<LogMutation> getPendingMutations() {
+ return pendingMutations;
+ }
+
+ @Override
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ // Unimplemented.
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdd41333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
new file mode 100644
index 0000000..22c0ef8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * YarnConfigurationStore exposes the methods needed for retrieving and
+ * persisting {@link CapacityScheduler} configuration via key-value
+ * using write-ahead logging. When configuration mutation is requested, caller
+ * should first log it with {@code logMutation}, which persists this pending
+ * mutation. This mutation is merged to the persisted configuration only after
+ * {@code confirmMutation} is called.
+ *
+ * On startup/recovery, caller should call {@code retrieve} to get all
+ * confirmed mutations, then get pending mutations which were not confirmed via
+ * {@code getPendingMutations}, and replay/confirm them via
+ * {@code confirmMutation} as in the normal case.
+ */
+public interface YarnConfigurationStore {
+
+ /**
+ * LogMutation encapsulates the fields needed for configuration mutation
+ * audit logging and recovery.
+ */
+ class LogMutation {
+ private Map<String, String> updates;
+ private String user;
+ private long id;
+
+ /**
+ * Create log mutation prior to logging.
+ * @param updates key-value configuration updates
+ * @param user user who requested configuration change
+ */
+ public LogMutation(Map<String, String> updates, String user) {
+ this(updates, user, 0);
+ }
+
+ /**
+ * Create log mutation for recovery.
+ * @param updates key-value configuration updates
+ * @param user user who requested configuration change
+ * @param id transaction id of configuration change
+ */
+ LogMutation(Map<String, String> updates, String user, long id) {
+ this.updates = updates;
+ this.user = user;
+ this.id = id;
+ }
+
+ /**
+ * Get key-value configuration updates.
+ * @return map of configuration updates
+ */
+ public Map<String, String> getUpdates() {
+ return updates;
+ }
+
+ /**
+ * Get user who requested configuration change.
+ * @return user who requested configuration change
+ */
+ public String getUser() {
+ return user;
+ }
+
+ /**
+ * Get transaction id of this configuration change.
+ * @return transaction id
+ */
+ public long getId() {
+ return id;
+ }
+
+ /**
+ * Set transaction id of this configuration change.
+ * @param id transaction id
+ */
+ public void setId(long id) {
+ this.id = id;
+ }
+ }
+
+ /**
+ * Initialize the configuration store.
+ * @param conf configuration to initialize store with
+ * @param schedConf Initial key-value configuration to persist
+ */
+ void initialize(Configuration conf, Configuration schedConf);
+
+ /**
+ * Logs the configuration change to backing store. Generates an id associated
+ * with this mutation, sets it in {@code logMutation}, and returns it.
+ * @param logMutation configuration change to be persisted in write ahead log
+ * @return id which configuration store associates with this mutation
+ */
+ long logMutation(LogMutation logMutation);
+
+ /**
+ * Should be called after {@code logMutation}. Gets the pending mutation
+ * associated with {@code id} and marks the mutation as persisted (no longer
+ * pending). If isValid is true, merge the mutation with the persisted
+ * configuration.
+ *
+ * If {@code confirmMutation} is called with ids in a different order than
+ * was returned by {@code logMutation}, the result is implementation
+ * dependent.
+ * @param id id of mutation to be confirmed
+ * @param isValid if true, update persisted configuration with mutation
+ * associated with {@code id}.
+ * @return true on success
+ */
+ boolean confirmMutation(long id, boolean isValid);
+
+ /**
+ * Retrieve the persisted configuration.
+ * @return configuration as key-value
+ */
+ Configuration retrieve();
+
+ /**
+ * Get the list of pending mutations, in the order they were logged.
+ * @return list of mutations
+ */
+ List<LogMutation> getPendingMutations();
+
+ /**
+ * Get a list of confirmed configuration mutations starting from a given id.
+ * @param fromId id from which to start getting mutations, inclusive
+ * @return list of configuration mutations
+ */
+ List<LogMutation> getConfirmedConfHistory(long fromId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdd41333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
new file mode 100644
index 0000000..dff4e77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class TestYarnConfigurationStore {
+
+ private YarnConfigurationStore confStore;
+ private Configuration schedConf;
+
+ private static final String testUser = "testUser";
+
+ @Before
+ public void setUp() {
+ schedConf = new Configuration(false);
+ schedConf.set("key1", "val1");
+ }
+
+ @Test
+ public void testInMemoryConfigurationStore() {
+ confStore = new InMemoryConfigurationStore();
+ confStore.initialize(new Configuration(), schedConf);
+ assertEquals("val1", confStore.retrieve().get("key1"));
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("keyUpdate1", "valUpdate1");
+ LogMutation mutation1 = new LogMutation(update1, testUser);
+ long id = confStore.logMutation(mutation1);
+ assertEquals(1, confStore.getPendingMutations().size());
+ confStore.confirmMutation(id, true);
+ assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1"));
+ assertEquals(0, confStore.getPendingMutations().size());
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("keyUpdate2", "valUpdate2");
+ LogMutation mutation2 = new LogMutation(update2, testUser);
+ id = confStore.logMutation(mutation2);
+ assertEquals(1, confStore.getPendingMutations().size());
+ confStore.confirmMutation(id, false);
+ assertNull("Configuration should not be updated",
+ confStore.retrieve().get("keyUpdate2"));
+ assertEquals(0, confStore.getPendingMutations().size());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/29] hadoop git commit: HDFS-11840. Log HDFS Mover exception
message of exit to its own log. Contributed by LiXin Ge.
Posted by jh...@apache.org.
HDFS-11840. Log HDFS Mover exception message of exit to its own log. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c31cb879
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c31cb879
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c31cb879
Branch: refs/heads/YARN-5734
Commit: c31cb879a3595f76d01408e6d066a0bde20be043
Parents: 867903d
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Jun 6 14:57:48 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Jun 6 14:57:48 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c31cb879/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 7eac87d..1a2c889 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -180,6 +180,7 @@ public class Mover {
return ExitStatus.ILLEGAL_ARGUMENTS;
} catch (IOException e) {
System.out.println(e + ". Exiting ...");
+ LOG.error(e + ". Exiting ...");
return ExitStatus.IO_EXCEPTION;
} finally {
dispatcher.shutdownNow();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/29] hadoop git commit: HADOOP-14431. ModifyTime of FileStatus
returned by SFTPFileSystem's getFileStatus method is wrong. Contributed by
Hongyuan Li.
Posted by jh...@apache.org.
HADOOP-14431. ModifyTime of FileStatus returned by SFTPFileSystem's getFileStatus method is wrong. Contributed by Hongyuan Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66c6fd83
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66c6fd83
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66c6fd83
Branch: refs/heads/YARN-5734
Commit: 66c6fd831497944f4f49c5ce42c69a302b7d7bc0
Parents: 6a28a2b
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Tue Jun 6 12:31:40 2017 +0800
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Tue Jun 6 12:31:40 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java | 2 +-
.../java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java | 9 +++++++++
2 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66c6fd83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
index d91d391..6de69fa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
@@ -277,7 +277,7 @@ public class SFTPFileSystem extends FileSystem {
// Using default block size since there is no way in SFTP channel to know of
// block sizes on server. The assumption could be less than ideal.
long blockSize = DEFAULT_BLOCK_SIZE;
- long modTime = attr.getMTime() * 1000; // convert to milliseconds
+ long modTime = attr.getMTime() * 1000L; // convert to milliseconds
long accessTime = attr.getATime() * 1000L;
FsPermission permission = getPermissions(sftpFile);
// not be able to get the real user group name, just use the user and group
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66c6fd83/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java
index 9b514e1..3d57dab 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java
@@ -319,4 +319,13 @@ public class TestSFTPFileSystem {
assertEquals(accessTime1, accessTime2);
}
+ @Test
+ public void testGetModifyTime() throws IOException {
+ Path file = touch(localFs, name.getMethodName().toLowerCase() + "1");
+ java.io.File localFile = ((LocalFileSystem) localFs).pathToFile(file);
+ long modifyTime1 = localFile.lastModified();
+ long modifyTime2 = sftpFs.getFileStatus(file).getModificationTime();
+ assertEquals(modifyTime1, modifyTime2);
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/29] hadoop git commit: YARN-5948. Implement
MutableConfigurationManager for handling storage into configuration store
Posted by jh...@apache.org.
YARN-5948. Implement MutableConfigurationManager for handling storage into configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/556a5e58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/556a5e58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/556a5e58
Branch: refs/heads/YARN-5734
Commit: 556a5e583725702dd9f632c8cf22ef3594e1b467
Parents: fdd4133
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Wed Mar 1 16:03:01 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Thu Jun 8 20:03:59 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 6 ++
.../src/main/resources/yarn-default.xml | 12 +++
.../scheduler/MutableConfigurationProvider.java | 35 ++++++++
.../scheduler/capacity/CapacityScheduler.java | 14 ++-
.../CapacitySchedulerConfiguration.java | 3 +
.../capacity/conf/CSConfigurationProvider.java | 3 +-
.../conf/MutableCSConfigurationProvider.java | 94 ++++++++++++++++++++
.../conf/YarnConfigurationStoreFactory.java | 46 ++++++++++
.../TestMutableCSConfigurationProvider.java | 83 +++++++++++++++++
9 files changed, 291 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556a5e58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 5e4c826..fa9e79d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -607,6 +607,12 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
"org.apache.hadoop.yarn.LocalConfigurationProvider";
+ public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
+ YARN_PREFIX + "scheduler.configuration.store.class";
+ public static final String MEMORY_CONFIGURATION_STORE = "memory";
+ public static final String DEFAULT_CONFIGURATION_STORE =
+ MEMORY_CONFIGURATION_STORE;
+
public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX
+ "authorization-provider";
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556a5e58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e687eef..02284e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3108,4 +3108,16 @@
<value>false</value>
</property>
+ <property>
+ <description>
+ The type of configuration store to use for storing scheduler
+ configurations, if using a mutable configuration provider.
+ Keywords such as "memory" map to certain configuration store
+ implementations. If keyword is not found, try to load this
+ value as a class.
+ </description>
+ <name>yarn.scheduler.configuration.store.class</name>
+ <value>memory</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556a5e58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
new file mode 100644
index 0000000..da30a2b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.Map;
+
+/**
+ * Interface for allowing changing scheduler configurations.
+ */
+public interface MutableConfigurationProvider {
+
+ /**
+ * Update the scheduler configuration with the provided key value pairs.
+ * @param user User issuing the request
+ * @param confUpdate Key-value pairs for configurations to be updated.
+ */
+ void mutateConfiguration(String user, Map<String, String> confUpdate);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556a5e58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index a925904..9353df2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -105,6 +105,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.MutableCSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation;
@@ -290,10 +291,15 @@ public class CapacityScheduler extends
String confProviderStr = configuration.get(
CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
- if (confProviderStr.equals(
- CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) {
- this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext);
- } else {
+ switch (confProviderStr) {
+ case CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER:
+ this.csConfProvider =
+ new FileBasedCSConfigurationProvider(rmContext);
+ break;
+ case CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER:
+ this.csConfProvider = new MutableCSConfigurationProvider(rmContext);
+ break;
+ default:
throw new IOException("Invalid CS configuration provider: " +
confProviderStr);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556a5e58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 6dd7aef..de72669 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -300,6 +300,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String FILE_CS_CONF_PROVIDER = "file";
@Private
+ public static final String STORE_CS_CONF_PROVIDER = "store";
+
+ @Private
public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556a5e58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
index c9984ac..0d2c8bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
@@ -32,8 +32,9 @@ public interface CSConfigurationProvider {
/**
* Initialize the configuration provider with given conf.
* @param conf configuration to initialize with
+ * @throws IOException if initialization fails due to misconfiguration
*/
- void init(Configuration conf);
+ void init(Configuration conf) throws IOException;
/**
* Loads capacity scheduler configuration object.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556a5e58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
new file mode 100644
index 0000000..267ab6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * CS configuration provider which implements
+ * {@link MutableConfigurationProvider} for modifying capacity scheduler
+ * configuration.
+ */
+public class MutableCSConfigurationProvider implements CSConfigurationProvider,
+ MutableConfigurationProvider {
+
+ private Configuration schedConf;
+ private YarnConfigurationStore confStore;
+ private RMContext rmContext;
+ private Configuration conf;
+
+ public MutableCSConfigurationProvider(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void init(Configuration config) throws IOException {
+ String store = config.get(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
+ switch (store) {
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ this.confStore = new InMemoryConfigurationStore();
+ break;
+ default:
+ this.confStore = YarnConfigurationStoreFactory.getStore(config);
+ break;
+ }
+ Configuration initialSchedConf = new Configuration(false);
+ initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
+ this.schedConf = initialSchedConf;
+ confStore.initialize(config, initialSchedConf);
+ this.conf = config;
+ }
+
+ @Override
+ public CapacitySchedulerConfiguration loadConfiguration(Configuration
+ configuration) throws IOException {
+ Configuration loadedConf = new Configuration(configuration);
+ loadedConf.addResource(schedConf);
+ return new CapacitySchedulerConfiguration(loadedConf, false);
+ }
+
+ @Override
+ public void mutateConfiguration(String user,
+ Map<String, String> confUpdate) {
+ Configuration oldConf = new Configuration(schedConf);
+ LogMutation log = new LogMutation(confUpdate, user);
+ long id = confStore.logMutation(log);
+ for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ try {
+ rmContext.getScheduler().reinitialize(conf, rmContext);
+ } catch (IOException e) {
+ schedConf = oldConf;
+ confStore.confirmMutation(id, false);
+ return;
+ }
+ confStore.confirmMutation(id, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556a5e58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
new file mode 100644
index 0000000..60249c8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Factory class for creating instances of {@link YarnConfigurationStore}.
+ */
+public final class YarnConfigurationStoreFactory {
+
+ private static final Log LOG = LogFactory.getLog(
+ YarnConfigurationStoreFactory.class);
+
+ private YarnConfigurationStoreFactory() {
+ // Unused.
+ }
+
+ public static YarnConfigurationStore getStore(Configuration conf) {
+ Class<? extends YarnConfigurationStore> storeClass =
+ conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ InMemoryConfigurationStore.class, YarnConfigurationStore.class);
+ LOG.info("Using YarnConfigurationStore implementation - " + storeClass);
+ return ReflectionUtils.newInstance(storeClass, conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556a5e58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
new file mode 100644
index 0000000..3f103b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests {@link MutableCSConfigurationProvider}.
+ */
+public class TestMutableCSConfigurationProvider {
+
+ private MutableCSConfigurationProvider confProvider;
+ private RMContext rmContext;
+ private Map<String, String> goodUpdate;
+ private Map<String, String> badUpdate;
+ private CapacityScheduler cs;
+
+ private static final String TEST_USER = "testUser";
+
+ @Before
+ public void setUp() {
+ cs = mock(CapacityScheduler.class);
+ rmContext = mock(RMContext.class);
+ when(rmContext.getScheduler()).thenReturn(cs);
+ confProvider = new MutableCSConfigurationProvider(rmContext);
+ goodUpdate = new HashMap<>();
+ goodUpdate.put("goodKey", "goodVal");
+ badUpdate = new HashMap<>();
+ badUpdate.put("badKey", "badVal");
+ }
+
+ @Test
+ public void testInMemoryBackedProvider() throws IOException {
+ Configuration conf = new Configuration();
+ confProvider.init(conf);
+ assertNull(confProvider.loadConfiguration(conf)
+ .get("goodKey"));
+
+ doNothing().when(cs).reinitialize(any(Configuration.class),
+ any(RMContext.class));
+ confProvider.mutateConfiguration(TEST_USER, goodUpdate);
+ assertEquals("goodVal", confProvider.loadConfiguration(conf)
+ .get("goodKey"));
+
+ assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
+ any(RMContext.class));
+ confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/29] hadoop git commit: YARN-6575. Support global configuration
mutation in MutableConfProvider. (Jonathan Hung via Xuan Gong)
Posted by jh...@apache.org.
YARN-6575. Support global configuration mutation in MutableConfProvider. (Jonathan Hung via Xuan Gong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a48d4757
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a48d4757
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a48d4757
Branch: refs/heads/YARN-5734
Commit: a48d47572e10f330b30056a666fc04d5078d1d0b
Parents: 8bf3548
Author: Xuan <xg...@apache.org>
Authored: Mon Jun 5 16:30:38 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Thu Jun 8 20:03:59 2017 -0700
----------------------------------------------------------------------
.../ConfigurationMutationACLPolicy.java | 4 +-
.../DefaultConfigurationMutationACLPolicy.java | 4 +-
.../scheduler/MutableConfScheduler.java | 4 +-
.../scheduler/MutableConfigurationProvider.java | 4 +-
.../scheduler/capacity/CapacityScheduler.java | 4 +-
.../conf/MutableCSConfigurationProvider.java | 10 +-
...ueueAdminConfigurationMutationACLPolicy.java | 22 +++-
.../resourcemanager/webapp/RMWebServices.java | 4 +-
.../webapp/dao/QueueConfigsUpdateInfo.java | 60 -----------
.../webapp/dao/SchedConfUpdateInfo.java | 69 +++++++++++++
.../TestConfigurationMutationACLPolicies.java | 28 ++++-
.../TestMutableCSConfigurationProvider.java | 10 +-
.../TestRMWebServicesConfigurationMutation.java | 101 +++++++++++++------
13 files changed, 205 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
index 724487b..3a388fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
/**
* Interface for determining whether configuration mutations are allowed.
@@ -41,7 +41,7 @@ public interface ConfigurationMutationACLPolicy {
* @param confUpdate configurations to be updated
* @return whether provided mutation is allowed or not
*/
- boolean isMutationAllowed(UserGroupInformation user, QueueConfigsUpdateInfo
+ boolean isMutationAllowed(UserGroupInformation user, SchedConfUpdateInfo
confUpdate);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
index 680c3b8..6648668 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
/**
* Default configuration mutation ACL policy. Checks if user is YARN admin.
@@ -39,7 +39,7 @@ public class DefaultConfigurationMutationACLPolicy implements
@Override
public boolean isMutationAllowed(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) {
+ SchedConfUpdateInfo confUpdate) {
return authorizer.isAdmin(user);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 93a935e..027d944 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
@@ -36,7 +36,7 @@ public interface MutableConfScheduler extends ResourceScheduler {
* @throws IOException if update is invalid
*/
void updateConfiguration(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) throws IOException;
+ SchedConfUpdateInfo confUpdate) throws IOException;
/**
* Get the scheduler configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index f04c128..6b8306c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
@@ -34,7 +34,7 @@ public interface MutableConfigurationProvider {
* @param confUpdate Key-value pairs for configurations to be updated.
* @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(UserGroupInformation user, QueueConfigsUpdateInfo
+ void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo
confUpdate) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index a0e7391..9299c4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -136,7 +136,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -2493,7 +2493,7 @@ public class CapacityScheduler extends
@Override
public void updateConfiguration(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) throws IOException {
+ SchedConfUpdateInfo confUpdate) throws IOException {
if (csConfProvider instanceof MutableConfigurationProvider) {
((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
user, confUpdate);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 8b879b0..eb97260 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.ArrayList;
@@ -98,7 +98,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
@Override
public void mutateConfiguration(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) throws IOException {
+ SchedConfUpdateInfo confUpdate) throws IOException {
if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
throw new AccessControlException("User is not admin of all modified" +
" queues.");
@@ -126,7 +126,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
private Map<String, String> constructKeyValueConfUpdate(
- QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ SchedConfUpdateInfo mutationInfo) throws IOException {
CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
CapacitySchedulerConfiguration proposedConf =
new CapacitySchedulerConfiguration(cs.getConfiguration(), false);
@@ -140,6 +140,10 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
updateQueue(updateQueueInfo, proposedConf, confUpdate);
}
+ for (Map.Entry<String, String> global : mutationInfo.getGlobalParams()
+ .entrySet()) {
+ confUpdate.put(global.getKey(), global.getValue());
+ }
return confUpdate;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
index 1f94c1c..0a82d50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
@@ -22,15 +22,17 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
/**
@@ -40,16 +42,29 @@ import java.util.Set;
public class QueueAdminConfigurationMutationACLPolicy implements
ConfigurationMutationACLPolicy {
+ private Configuration conf;
private RMContext rmContext;
+ private YarnAuthorizationProvider authorizer;
@Override
- public void init(Configuration conf, RMContext context) {
+ public void init(Configuration config, RMContext context) {
+ this.conf = config;
this.rmContext = context;
+ this.authorizer = YarnAuthorizationProvider.getInstance(conf);
}
@Override
public boolean isMutationAllowed(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) {
+ SchedConfUpdateInfo confUpdate) {
+ // If there are global config changes, check if user is admin.
+ Map<String, String> globalParams = confUpdate.getGlobalParams();
+ if (globalParams != null && globalParams.size() != 0) {
+ if (!authorizer.isAdmin(user)) {
+ return false;
+ }
+ }
+
+ // Check if user is admin of all modified queues.
Set<String> queues = new HashSet<>();
for (QueueConfigInfo addQueueInfo : confUpdate.getAddQueueInfo()) {
queues.add(addQueueInfo.getQueue());
@@ -71,7 +86,6 @@ public class QueueAdminConfigurationMutationACLPolicy implements
// Queue is not found, do nothing.
}
String parentPath = queuePath;
- // TODO: handle global config change.
while (queueInfo == null) {
// We are adding a queue (whose parent we are possibly also adding).
// Check ACL of lowest parent queue which already exists.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 6d2df76..0b90bdc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -2564,11 +2564,11 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
}
@PUT
- @Path("/queues")
+ @Path("/sched-conf")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public Response updateSchedulerConfiguration(QueueConfigsUpdateInfo
+ public Response updateSchedulerConfiguration(SchedConfUpdateInfo
mutationInfo, @Context HttpServletRequest hsr)
throws AuthorizationException, InterruptedException {
init();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
deleted file mode 100644
index 644ec90..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
-
-import java.util.ArrayList;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * Information for making scheduler configuration changes (supports adding,
- * removing, or updating a queue).
- */
-@XmlRootElement(name = "schedConf")
-@XmlAccessorType(XmlAccessType.FIELD)
-public class QueueConfigsUpdateInfo {
-
- @XmlElement(name = "add")
- private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
-
- @XmlElement(name = "remove")
- private ArrayList<String> removeQueueInfo = new ArrayList<>();
-
- @XmlElement(name = "update")
- private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
-
- public QueueConfigsUpdateInfo() {
- // JAXB needs this
- }
-
- public ArrayList<QueueConfigInfo> getAddQueueInfo() {
- return addQueueInfo;
- }
-
- public ArrayList<String> getRemoveQueueInfo() {
- return removeQueueInfo;
- }
-
- public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
- return updateQueueInfo;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
new file mode 100644
index 0000000..b7c585e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for making scheduler configuration changes (supports adding,
+ * removing, or updating a queue, as well as global scheduler conf changes).
+ */
+@XmlRootElement(name = "schedConf")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SchedConfUpdateInfo {
+
+ @XmlElement(name = "add-queue")
+ private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "remove-queue")
+ private ArrayList<String> removeQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "update-queue")
+ private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
+
+ private HashMap<String, String> global = new HashMap<>();
+
+ public SchedConfUpdateInfo() {
+ // JAXB needs this
+ }
+
+ public ArrayList<QueueConfigInfo> getAddQueueInfo() {
+ return addQueueInfo;
+ }
+
+ public ArrayList<String> getRemoveQueueInfo() {
+ return removeQueueInfo;
+ }
+
+ public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
+ return updateQueueInfo;
+ }
+
+ @XmlElementWrapper(name = "global-updates")
+ public HashMap<String, String> getGlobalParams() {
+ return global;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
index 4016dcf..0f5a3d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
@@ -77,6 +77,7 @@ public class TestConfigurationMutationACLPolicies {
.thenReturn(false);
when(scheduler.getQueue(eq(queueName))).thenReturn(queue);
}
+
@Test
public void testDefaultPolicy() {
Configuration conf = new Configuration();
@@ -98,7 +99,7 @@ public class TestConfigurationMutationACLPolicies {
ConfigurationMutationACLPolicy.class);
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
QueueConfigInfo configInfo = new QueueConfigInfo("root.a", EMPTY_MAP);
updateInfo.getUpdateQueueInfo().add(configInfo);
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
@@ -114,7 +115,7 @@ public class TestConfigurationMutationACLPolicies {
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
// Add root.b.b1. Should check ACL of root.b queue.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2", EMPTY_MAP);
updateInfo.getAddQueueInfo().add(configInfo);
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
@@ -130,7 +131,7 @@ public class TestConfigurationMutationACLPolicies {
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
// Add root.b.b1.b11. Should check ACL of root.b queue.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2.b21", EMPTY_MAP);
updateInfo.getAddQueueInfo().add(configInfo);
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
@@ -146,9 +147,26 @@ public class TestConfigurationMutationACLPolicies {
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
// Remove root.b.b1.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.b.b1");
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
}
+
+ @Test
+ public void testQueueAdminPolicyGlobal() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, GOOD_USER.getShortUserName());
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertTrue(policy.isMutationAllowed(BAD_USER, updateInfo));
+ updateInfo.getGlobalParams().put("globalKey", "globalValue");
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 13229b1..3216781 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
@@ -47,8 +47,8 @@ public class TestMutableCSConfigurationProvider {
private MutableCSConfigurationProvider confProvider;
private RMContext rmContext;
- private QueueConfigsUpdateInfo goodUpdate;
- private QueueConfigsUpdateInfo badUpdate;
+ private SchedConfUpdateInfo goodUpdate;
+ private SchedConfUpdateInfo badUpdate;
private CapacityScheduler cs;
private static final UserGroupInformation TEST_USER = UserGroupInformation
@@ -62,14 +62,14 @@ public class TestMutableCSConfigurationProvider {
when(cs.getConfiguration()).thenReturn(
new CapacitySchedulerConfiguration());
confProvider = new MutableCSConfigurationProvider(rmContext);
- goodUpdate = new QueueConfigsUpdateInfo();
+ goodUpdate = new SchedConfUpdateInfo();
Map<String, String> goodUpdateMap = new HashMap<>();
goodUpdateMap.put("goodKey", "goodVal");
QueueConfigInfo goodUpdateInfo = new
QueueConfigInfo("root.a", goodUpdateMap);
goodUpdate.getUpdateQueueInfo().add(goodUpdateInfo);
- badUpdate = new QueueConfigsUpdateInfo();
+ badUpdate = new SchedConfUpdateInfo();
Map<String, String> badUpdateMap = new HashMap<>();
badUpdateMap.put("badKey", "badVal");
QueueConfigInfo badUpdateInfo = new
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48d4757/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index d149055..5fbe36f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -162,7 +162,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Add parent queue root.d with two children d1 and d2.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> d1Capacity = new HashMap<>();
d1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
d1Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "25");
@@ -181,9 +181,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getAddQueueInfo().add(d);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -205,7 +205,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Add root.d with capacity 25, reducing root.b capacity from 75 to 50.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> dCapacity = new HashMap<>();
dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
Map<String, String> bCapacity = new HashMap<>();
@@ -216,9 +216,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(b);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -238,13 +238,13 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.a.a2");
// Remove root.a.a2
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.a.a2");
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -263,13 +263,13 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.c", "root.c.c1");
// Remove root.c (parent queue)
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.c");
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -288,7 +288,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.a", "root.a.a1", "root.a.a2");
// Remove root.a (parent queue) with capacity 25
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.a");
// Set root.b capacity to 100
@@ -298,9 +298,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(b);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -320,7 +320,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.b", "root.c", "root.c.c1");
// Remove root.b and root.c
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.b");
updateInfo.getRemoveQueueInfo().add("root.c");
Map<String, String> aCapacity = new HashMap<>();
@@ -330,9 +330,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(configInfo);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -348,7 +348,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Set state of queues to STOPPED.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> stoppedParam = new HashMap<>();
stoppedParam.put(CapacitySchedulerConfiguration.STATE,
QueueState.STOPPED.toString());
@@ -358,9 +358,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
}
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -378,7 +378,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Update config value.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> updateParam = new HashMap<>();
updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
"0.2");
@@ -393,9 +393,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
0.001f);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -411,9 +411,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -431,7 +431,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Update root.a and root.b capacity to 50.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> updateParam = new HashMap<>();
updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50");
QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
@@ -441,9 +441,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -453,6 +453,47 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
}
+ @Test
+ public void testGlobalConfChange() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Set maximum-applications to 30000.
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX +
+ "maximum-applications", "30000");
+
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("sched-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(30000, newCSConf.getMaximumSystemApplications());
+
+ updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX +
+ "maximum-applications", null);
+ // Unset maximum-applications. Should be set to default.
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("sched-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS,
+ newCSConf.getMaximumSystemApplications());
+ }
+
@Override
@After
public void tearDown() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/29] hadoop git commit: HADOOP-14491. Azure has messed doc
structure. Contributed by Mingliang Liu
Posted by jh...@apache.org.
HADOOP-14491. Azure has messed doc structure. Contributed by Mingliang Liu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/536f0571
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/536f0571
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/536f0571
Branch: refs/heads/YARN-5734
Commit: 536f057158c445a57049f6c392869ae2f0be4f24
Parents: 6b5285b
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Jun 5 18:24:40 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Jun 6 11:09:28 2017 -0700
----------------------------------------------------------------------
.../hadoop-azure/src/site/markdown/index.md | 233 ++++++++++---------
1 file changed, 123 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/536f0571/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index 1dca3b9..9c57e60 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -14,20 +14,9 @@
# Hadoop Azure Support: Azure Blob Storage
-* [Introduction](#Introduction)
-* [Features](#Features)
-* [Limitations](#Limitations)
-* [Usage](#Usage)
- * [Concepts](#Concepts)
- * [Configuring Credentials](#Configuring_Credentials)
- * [Page Blob Support and Configuration](#Page_Blob_Support_and_Configuration)
- * [Atomic Folder Rename](#Atomic_Folder_Rename)
- * [Accessing wasb URLs](#Accessing_wasb_URLs)
- * [Append API Support and Configuration](#Append_API_Support_and_Configuration)
- * [Multithread Support](#Multithread_Support)
-* [Testing the hadoop-azure Module](#Testing_the_hadoop-azure_Module)
-
-## <a name="Introduction" />Introduction
+<!-- MACRO{toc|fromDepth=1|toDepth=3} -->
+
+## Introduction
The hadoop-azure module provides support for integration with
[Azure Blob Storage](http://azure.microsoft.com/en-us/documentation/services/storage/).
@@ -38,7 +27,7 @@ on the additional artifacts it requires, notably the
To make it part of Apache Hadoop's default classpath, simply make sure that
HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-azure' in the list.
-## <a name="Features" />Features
+## Features
* Read and write data stored in an Azure Blob Storage account.
* Present a hierarchical file system view by implementing the standard Hadoop
@@ -54,15 +43,15 @@ HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-azure' in the list.
* Tested on both Linux and Windows.
* Tested at scale.
-## <a name="Limitations" />Limitations
+## Limitations
* File owner and group are persisted, but the permissions model is not enforced.
Authorization occurs at the level of the entire Azure Blob Storage account.
* File last access time is not tracked.
-## <a name="Usage" />Usage
+## Usage
-### <a name="Concepts" />Concepts
+### Concepts
The Azure Blob Storage data model presents 3 core concepts:
@@ -76,7 +65,7 @@ The Azure Blob Storage data model presents 3 core concepts:
The internal implementation also uses blobs to persist the file system
hierarchy and other metadata.
-### <a name="Configuring_Credentials" />Configuring Credentials
+### Configuring Credentials
Usage of Azure Blob Storage requires configuration of credentials. Typically
this is set in core-site.xml. The configuration property name is of the form
@@ -87,11 +76,12 @@ untrusted party.**
For example:
- <property>
- <name>fs.azure.account.key.youraccount.blob.core.windows.net</name>
- <value>YOUR ACCESS KEY</value>
- </property>
-
+```xml
+<property>
+ <name>fs.azure.account.key.youraccount.blob.core.windows.net</name>
+ <value>YOUR ACCESS KEY</value>
+</property>
+```
In many Hadoop clusters, the core-site.xml file is world-readable. It is possible to
protect the access key within a credential provider as well. This provides an encrypted
file format along with protection with file permissions.
@@ -110,14 +100,14 @@ For additional reading on the credential provider API see:
###### provision
-```
+```bash
% hadoop credential create fs.azure.account.key.youraccount.blob.core.windows.net -value 123
-provider localjceks://file/home/lmccay/wasb.jceks
```
###### configure core-site.xml or command line system property
-```
+```xml
<property>
<name>hadoop.security.credential.provider.path</name>
<value>localjceks://file/home/lmccay/wasb.jceks</value>
@@ -127,7 +117,7 @@ For additional reading on the credential provider API see:
###### distcp
-```
+```bash
% hadoop distcp
[-D hadoop.security.credential.provider.path=localjceks://file/home/lmccay/wasb.jceks]
hdfs://hostname:9001/user/lmccay/007020615 wasb://yourcontainer@youraccount.blob.core.windows.net/testDir/
@@ -145,22 +135,25 @@ specifies an external program to be invoked by Hadoop processes to decrypt the
key. The encrypted key value is passed to this external program as a command
line argument:
- <property>
- <name>fs.azure.account.keyprovider.youraccount</name>
- <value>org.apache.hadoop.fs.azure.ShellDecryptionKeyProvider</value>
- </property>
+```xml
+<property>
+ <name>fs.azure.account.keyprovider.youraccount</name>
+ <value>org.apache.hadoop.fs.azure.ShellDecryptionKeyProvider</value>
+</property>
- <property>
- <name>fs.azure.account.key.youraccount.blob.core.windows.net</name>
- <value>YOUR ENCRYPTED ACCESS KEY</value>
- </property>
+<property>
+ <name>fs.azure.account.key.youraccount.blob.core.windows.net</name>
+ <value>YOUR ENCRYPTED ACCESS KEY</value>
+</property>
- <property>
- <name>fs.azure.shellkeyprovider.script</name>
- <value>PATH TO DECRYPTION PROGRAM</value>
- </property>
+<property>
+ <name>fs.azure.shellkeyprovider.script</name>
+ <value>PATH TO DECRYPTION PROGRAM</value>
+</property>
+
+```
-### <a name="Page_Blob_Support_and_Configuration" />Page Blob Support and Configuration
+### Page Blob Support and Configuration
The Azure Blob Storage interface for Hadoop supports two kinds of blobs,
[block blobs and page blobs](http://msdn.microsoft.com/en-us/library/azure/ee691964.aspx).
@@ -182,10 +175,12 @@ folder names.
For example:
- <property>
- <name>fs.azure.page.blob.dir</name>
- <value>/hbase/WALs,/hbase/oldWALs,/data/mypageblobfiles</value>
- </property>
+```xml
+<property>
+ <name>fs.azure.page.blob.dir</name>
+ <value>/hbase/WALs,/hbase/oldWALs,/data/mypageblobfiles</value>
+</property>
+```
You can set this to simply / to make all files page blobs.
@@ -197,7 +192,7 @@ The configuration option `fs.azure.page.blob.extension.size` is the page blob
extension size. This defines the amount to extend a page blob if it starts to
get full. It must be 128MB or greater, specified as an integer number of bytes.
-### <a name="Atomic_Folder_Rename" />Atomic Folder Rename
+### Atomic Folder Rename
Azure storage stores files as a flat key/value store without formal support
for folders. The hadoop-azure file system layer simulates folders on top
@@ -216,12 +211,14 @@ the intention of the rename operation, to allow redo in event of a failure.
For example:
- <property>
- <name>fs.azure.atomic.rename.dir</name>
- <value>/hbase,/data</value>
- </property>
+```xml
+<property>
+ <name>fs.azure.atomic.rename.dir</name>
+ <value>/hbase,/data</value>
+</property>
+```
-### <a name="Accessing_wasb_URLs" />Accessing wasb URLs
+### Accessing wasb URLs
After credentials are configured in core-site.xml, any Hadoop component may
reference files in that Azure Blob Storage account by using URLs of the following
@@ -238,28 +235,32 @@ For example, the following
commands demonstrate access to a storage account named `youraccount` and a
container named `yourcontainer`.
- > hadoop fs -mkdir wasb://yourcontainer@youraccount.blob.core.windows.net/testDir
+```bash
+% hadoop fs -mkdir wasb://yourcontainer@youraccount.blob.core.windows.net/testDir
- > hadoop fs -put testFile wasb://yourcontainer@youraccount.blob.core.windows.net/testDir/testFile
+% hadoop fs -put testFile wasb://yourcontainer@youraccount.blob.core.windows.net/testDir/testFile
- > hadoop fs -cat wasbs://yourcontainer@youraccount.blob.core.windows.net/testDir/testFile
- test file content
+% hadoop fs -cat wasbs://yourcontainer@youraccount.blob.core.windows.net/testDir/testFile
+test file content
+```
It's also possible to configure `fs.defaultFS` to use a `wasb` or `wasbs` URL.
This causes all bare paths, such as `/testDir/testFile` to resolve automatically
to that file system.
-### <a name="Append_API_Support_and_Configuration" />Append API Support and Configuration
+### Append API Support and Configuration
The Azure Blob Storage interface for Hadoop has optional support for Append API for
single writer by setting the configuration `fs.azure.enable.append.support` to true.
For Example:
- <property>
- <name>fs.azure.enable.append.support</name>
- <value>true</value>
- </property>
+```xml
+<property>
+ <name>fs.azure.enable.append.support</name>
+ <value>true</value>
+</property>
+```
It must be noted Append support in Azure Blob Storage interface DIFFERS FROM HDFS SEMANTICS. Append
support does not enforce single writer internally but requires applications to guarantee this semantic.
@@ -267,25 +268,29 @@ It becomes a responsibility of the application either to ensure single-threaded
file path, or rely on some external locking mechanism of its own. Failure to do so will result in
unexpected behavior.
-### <a name="Multithread_Support" />Multithread Support
+### Multithread Support
Rename and Delete blob operations on directories with large number of files and sub directories currently is very slow as these operations are done one blob at a time serially. These files and sub folders can be deleted or renamed parallel. Following configurations can be used to enable threads to do parallel processing
To enable 10 threads for Delete operation. Set configuration value to 0 or 1 to disable threads. The default behavior is threads disabled.
- <property>
- <name>fs.azure.delete.threads</name>
- <value>10</value>
- </property>
+```xml
+<property>
+ <name>fs.azure.delete.threads</name>
+ <value>10</value>
+</property>
+```
To enable 20 threads for Rename operation. Set configuration value to 0 or 1 to disable threads. The default behavior is threads disabled.
- <property>
- <name>fs.azure.rename.threads</name>
- <value>20</value>
- </property>
+```xml
+<property>
+ <name>fs.azure.rename.threads</name>
+ <value>20</value>
+</property>
+```
-### <a name="WASB_SECURE_MODE" />WASB Secure mode and configuration
+### WASB Secure mode and configuration
WASB can operate in secure mode where the Storage access keys required to communicate with Azure storage does not have to
be in the same address space as the process using WASB. In this mode all interactions with Azure storage is performed using
@@ -295,30 +300,32 @@ Romote mode, however for testing purposes the local mode can be enabled to gener
To enable Secure mode following property needs to be set to true.
-```
- <property>
- <name>fs.azure.secure.mode</name>
- <value>true</value>
- </property>
+```xml
+<property>
+ <name>fs.azure.secure.mode</name>
+ <value>true</value>
+</property>
```
To enable SAS key generation locally following property needs to be set to true.
+```xml
+<property>
+ <name>fs.azure.local.sas.key.mode</name>
+ <value>true</value>
+</property>
```
- <property>
- <name>fs.azure.local.sas.key.mode</name>
- <value>true</value>
- </property>
-```
+
To use the remote SAS key generation mode, an external REST service is expected to provided required SAS keys.
Following property can used to provide the end point to use for remote SAS Key generation:
+```xml
+<property>
+ <name>fs.azure.cred.service.url</name>
+ <value>{URL}</value>
+</property>
```
- <property>
- <name>fs.azure.cred.service.url</name>
- <value>{URL}</value>
- </property>
-```
+
The remote service is expected to provide support for two REST calls ```{URL}/GET_CONTAINER_SAS``` and ```{URL}/GET_RELATIVE_BLOB_SAS```, for generating
container and relative blob sas keys. An example requests
@@ -326,7 +333,8 @@ container and relative blob sas keys. An example requests
```{URL}/GET_CONTAINER_SAS?storage_account=<account_name>&container=<container>&relative_path=<relative path>&sas_expiry=<expiry period>&delegation_token=<delegation token>```
The service is expected to return a response in JSON format:
-```
+
+```json
{
"responseCode" : 0 or non-zero <int>,
"responseMessage" : relavant message on failure <String>,
@@ -334,40 +342,42 @@ The service is expected to return a response in JSON format:
}
```
-## <a name="WASB Authorization" />Authorization Support in WASB.
+### Authorization Support in WASB
Authorization support can be enabled in WASB using the following configuration:
+```xml
+<property>
+ <name>fs.azure.authorization</name>
+ <value>true</value>
+</property>
```
- <property>
- <name>fs.azure.authorization</name>
- <value>true</value>
- </property>
-```
- The current implementation of authorization relies on the presence of an external service that can enforce
- the authorization. The service is expected to be running on a URL provided by the following config.
-```
- <property>
- <name>fs.azure.authorization.remote.service.url</name>
- <value>{URL}</value>
- </property>
+The current implementation of authorization relies on the presence of an external service that can enforce
+the authorization. The service is expected to be running on a URL provided by the following config.
+
+```xml
+<property>
+ <name>fs.azure.authorization.remote.service.url</name>
+ <value>{URL}</value>
+</property>
```
- The remote service is expected to provide support for the following REST call: ```{URL}/CHECK_AUTHORIZATION```
- An example request:
+The remote service is expected to provide support for the following REST call: ```{URL}/CHECK_AUTHORIZATION```
+An example request:
```{URL}/CHECK_AUTHORIZATION?wasb_absolute_path=<absolute_path>&operation_type=<operation type>&delegation_token=<delegation token>```
- The service is expected to return a response in JSON format:
- ```
- {
+The service is expected to return a response in JSON format:
+
+```json
+{
"responseCode" : 0 or non-zero <int>,
"responseMessage" : relevant message on failure <String>,
"authorizationResult" : true/false <boolean>
- }
- ```
+}
+```
-## <a name="Testing_the_hadoop-azure_Module" />Testing the hadoop-azure Module
+## Testing the hadoop-azure Module
The hadoop-azure module includes a full suite of unit tests. Most of the tests
will run without additional configuration by running `mvn test`. This includes
@@ -382,10 +392,12 @@ that runs on a local machine.
To use the emulator, install Azure SDK 2.3 and start the storage emulator. Then,
edit `src/test/resources/azure-test.xml` and add the following property:
- <property>
- <name>fs.azure.test.emulator</name>
- <value>true</value>
- </property>
+```xml
+<property>
+ <name>fs.azure.test.emulator</name>
+ <value>true</value>
+</property>
+```
There is a known issue when running tests with the emulator. You may see the
following failure message:
@@ -399,6 +411,7 @@ file to `src/test/resources/azure-auth-keys.xml` and setting
the name of the storage account and its access key.
For example:
+
```xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/29] hadoop git commit: HADOOP-14472. Azure:
TestReadAndSeekPageBlobAfterWrite fails intermittently. Contributed by
Mingliang Liu
Posted by jh...@apache.org.
HADOOP-14472. Azure: TestReadAndSeekPageBlobAfterWrite fails intermittently. Contributed by Mingliang Liu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b5285bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b5285bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b5285bb
Branch: refs/heads/YARN-5734
Commit: 6b5285bbcb439944ba6c4701571ffbb00258d5a1
Parents: 19ef3a8
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Jun 6 11:06:49 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Jun 6 11:06:49 2017 -0700
----------------------------------------------------------------------
.../fs/azure/TestReadAndSeekPageBlobAfterWrite.java | 10 ----------
1 file changed, 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b5285bb/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestReadAndSeekPageBlobAfterWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestReadAndSeekPageBlobAfterWrite.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestReadAndSeekPageBlobAfterWrite.java
index e6219df..41b8386 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestReadAndSeekPageBlobAfterWrite.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestReadAndSeekPageBlobAfterWrite.java
@@ -275,18 +275,8 @@ public class TestReadAndSeekPageBlobAfterWrite {
writesSinceHFlush++;
output.flush();
if ((i % SYNC_INTERVAL) == 0) {
- long start = Time.monotonicNow();
output.hflush();
writesSinceHFlush = 0;
- long end = Time.monotonicNow();
-
- // A true, round-trip synchronous flush to Azure must take
- // a significant amount of time or we are not syncing to storage correctly.
- LOG.debug("hflush duration = " + (end - start) + " msec.");
- assertTrue(String.format(
- "hflush duration of %d, less than minimum expected of %d",
- end - start, MINIMUM_EXPECTED_TIME),
- end - start >= MINIMUM_EXPECTED_TIME);
}
}
} finally {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/29] hadoop git commit: HDFS-11851. getGlobalJNIEnv() may deadlock
if exception is thrown. Contributed by Sailesh Mukil.
Posted by jh...@apache.org.
HDFS-11851. getGlobalJNIEnv() may deadlock if exception is thrown. Contributed by Sailesh Mukil.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/164c222c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/164c222c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/164c222c
Branch: refs/heads/YARN-5734
Commit: 164c222c4aa697c3acbaa6f34c59a23177d7f3cd
Parents: a062374
Author: John Zhuge <jz...@apache.org>
Authored: Thu Jun 8 14:15:09 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Thu Jun 8 14:15:09 2017 -0700
----------------------------------------------------------------------
.../libhdfs-tests/test_libhdfs_threaded.c | 26 ++++++++++++++++++++
.../src/main/native/libhdfs/os/posix/mutexes.c | 9 ++++++-
2 files changed, 34 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c222c/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
index f80200d..8d4b743 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
@@ -17,8 +17,11 @@
*/
#include "expect.h"
+#include "exception.h"
#include "hdfs/hdfs.h"
+#include "jni_helper.h"
#include "native_mini_dfs.h"
+#include "os/mutexes.h"
#include "os/thread.h"
#include <errno.h>
@@ -329,6 +332,23 @@ static int checkFailures(struct tlhThreadInfo *ti, int tlhNumThreads)
return EXIT_FAILURE;
}
+int testRecursiveJvmMutex() {
+ jthrowable jthr;
+ JNIEnv *env = getJNIEnv();
+ if (!env) {
+ fprintf(stderr, "testRecursiveJvmMutex: getJNIEnv failed\n");
+ return -EIO;
+ }
+ jthr = newRuntimeError(env, "Dummy error to print for testing");
+
+ /* printExceptionAndFree() takes the jvmMutex within */
+ mutexLock(&jvmMutex);
+ printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "testRecursiveJvmMutex");
+ mutexUnlock(&jvmMutex);
+
+ return 0;
+}
+
/**
* Test that we can write a file with libhdfs and then read it back
*/
@@ -341,6 +361,12 @@ int main(void)
1, /* doFormat */
};
+ /* Check that the recursive mutex works as expected */
+ if (testRecursiveJvmMutex() < 0) {
+ fprintf(stderr, "testRecursiveJvmMutex failed\n");
+ return EXIT_FAILURE;
+ }
+
tlhNumThreadsStr = getenv("TLH_NUM_THREADS");
if (!tlhNumThreadsStr) {
tlhNumThreadsStr = "3";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c222c/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/posix/mutexes.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/posix/mutexes.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/posix/mutexes.c
index c4c2f26..20dafaa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/posix/mutexes.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/posix/mutexes.c
@@ -22,7 +22,14 @@
#include <stdio.h>
mutex hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER;
-mutex jvmMutex = PTHREAD_MUTEX_INITIALIZER;
+mutex jvmMutex;
+pthread_mutexattr_t jvmMutexAttr;
+
+__attribute__((constructor)) static void init() {
+ pthread_mutexattr_init(&jvmMutexAttr);
+ pthread_mutexattr_settype(&jvmMutexAttr, PTHREAD_MUTEX_RECURSIVE);
+ pthread_mutex_init(&jvmMutex, &jvmMutexAttr);
+}
int mutexLock(mutex *m) {
int ret = pthread_mutex_lock(m);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/29] hadoop git commit: HADOOP-14035. Reduce fair call queue
backoff's impact on clients. Contributed by Daryn Sharp.
Posted by jh...@apache.org.
HADOOP-14035. Reduce fair call queue backoff's impact on clients. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/855e0477
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/855e0477
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/855e0477
Branch: refs/heads/YARN-5734
Commit: 855e0477b1706a2d5b0df6a2b0e461aeec8839c2
Parents: 66c6fd8
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Jun 6 08:34:33 2017 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Jun 6 08:35:12 2017 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/ipc/CallQueueManager.java | 127 ++++++++++++++-
.../org/apache/hadoop/ipc/FairCallQueue.java | 90 ++++++++---
.../main/java/org/apache/hadoop/ipc/Server.java | 22 ++-
.../apache/hadoop/ipc/TestCallQueueManager.java | 78 +++++++++-
.../apache/hadoop/ipc/TestFairCallQueue.java | 156 +++++++++++++++++++
.../java/org/apache/hadoop/ipc/TestRPC.java | 4 +-
6 files changed, 441 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/855e0477/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
index 50ed353..2764788 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
@@ -18,8 +18,12 @@
package org.apache.hadoop.ipc;
+import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import java.util.AbstractQueue;
+import java.util.Collection;
+import java.util.Iterator;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@@ -28,11 +32,15 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto;
+
+import com.google.common.annotations.VisibleForTesting;
/**
* Abstracts queue operations for different blocking queues.
*/
-public class CallQueueManager<E> {
+public class CallQueueManager<E extends Schedulable>
+ extends AbstractQueue<E> implements BlockingQueue<E> {
public static final Log LOG = LogFactory.getLog(CallQueueManager.class);
// Number of checkpoints for empty queue.
private static final int CHECKPOINT_NUM = 20;
@@ -76,6 +84,15 @@ public class CallQueueManager<E> {
maxQueueSize + " scheduler: " + schedulerClass);
}
+ @VisibleForTesting // only!
+ CallQueueManager(BlockingQueue<E> queue, RpcScheduler scheduler,
+ boolean clientBackOffEnabled) {
+ this.putRef = new AtomicReference<BlockingQueue<E>>(queue);
+ this.takeRef = new AtomicReference<BlockingQueue<E>>(queue);
+ this.scheduler = scheduler;
+ this.clientBackOffEnabled = clientBackOffEnabled;
+ }
+
private static <T extends RpcScheduler> T createScheduler(
Class<T> theClass, int priorityLevels, String ns, Configuration conf) {
// Used for custom, configurable scheduler
@@ -190,12 +207,40 @@ public class CallQueueManager<E> {
}
/**
- * Insert e into the backing queue or block until we can.
+ * Insert e into the backing queue or block until we can. If client
+ * backoff is enabled this method behaves like add which throws if
+ * the queue overflows.
* If we block and the queue changes on us, we will insert while the
* queue is drained.
*/
+ @Override
public void put(E e) throws InterruptedException {
- putRef.get().put(e);
+ if (!isClientBackoffEnabled()) {
+ putRef.get().put(e);
+ } else if (shouldBackOff(e)) {
+ throwBackoff();
+ } else {
+ add(e);
+ }
+ }
+
+ @Override
+ public boolean add(E e) {
+ try {
+ return putRef.get().add(e);
+ } catch (CallQueueOverflowException ex) {
+ // queue provided a custom exception that may control if the client
+ // should be disconnected.
+ throw ex;
+ } catch (IllegalStateException ise) {
+ throwBackoff();
+ }
+ return true;
+ }
+
+ // ideally this behavior should be controllable too.
+ private void throwBackoff() throws IllegalStateException {
+ throw CallQueueOverflowException.DISCONNECT;
}
/**
@@ -203,14 +248,37 @@ public class CallQueueManager<E> {
* Return true if e is queued.
* Return false if the queue is full.
*/
- public boolean offer(E e) throws InterruptedException {
+ @Override
+ public boolean offer(E e) {
return putRef.get().offer(e);
}
+ @Override
+ public boolean offer(E e, long timeout, TimeUnit unit)
+ throws InterruptedException {
+ return putRef.get().offer(e, timeout, unit);
+ }
+
+ @Override
+ public E peek() {
+ return takeRef.get().peek();
+ }
+
+ @Override
+ public E poll() {
+ return takeRef.get().poll();
+ }
+
+ @Override
+ public E poll(long timeout, TimeUnit unit) throws InterruptedException {
+ return takeRef.get().poll(timeout, unit);
+ }
+
/**
* Retrieve an E from the backing queue or block until we can.
* Guaranteed to return an element from the current queue.
*/
+ @Override
public E take() throws InterruptedException {
E e = null;
@@ -221,10 +289,16 @@ public class CallQueueManager<E> {
return e;
}
+ @Override
public int size() {
return takeRef.get().size();
}
+ @Override
+ public int remainingCapacity() {
+ return takeRef.get().remainingCapacity();
+ }
+
/**
* Read the number of levels from the configuration.
* This will affect the FairCallQueue's overall capacity.
@@ -308,4 +382,49 @@ public class CallQueueManager<E> {
private String stringRepr(Object o) {
return o.getClass().getName() + '@' + Integer.toHexString(o.hashCode());
}
+
+ @Override
+ public int drainTo(Collection<? super E> c) {
+ return takeRef.get().drainTo(c);
+ }
+
+ @Override
+ public int drainTo(Collection<? super E> c, int maxElements) {
+ return takeRef.get().drainTo(c, maxElements);
+ }
+
+ @Override
+ public Iterator<E> iterator() {
+ return takeRef.get().iterator();
+ }
+
+ // exception that mimics the standard ISE thrown by blocking queues but
+ // embeds a rpc server exception for the client to retry and indicate
+ // if the client should be disconnected.
+ @SuppressWarnings("serial")
+ static class CallQueueOverflowException extends IllegalStateException {
+ private static String TOO_BUSY = "Server too busy";
+ static final CallQueueOverflowException KEEPALIVE =
+ new CallQueueOverflowException(
+ new RetriableException(TOO_BUSY),
+ RpcStatusProto.ERROR);
+ static final CallQueueOverflowException DISCONNECT =
+ new CallQueueOverflowException(
+ new RetriableException(TOO_BUSY + " - disconnecting"),
+ RpcStatusProto.FATAL);
+
+ CallQueueOverflowException(final IOException ioe,
+ final RpcStatusProto status) {
+ super("Queue full", new RpcServerException(ioe.getMessage(), ioe){
+ @Override
+ public RpcStatusProto getRpcStatusProto() {
+ return status;
+ }
+ });
+ }
+ @Override
+ public IOException getCause() {
+ return (IOException)super.getCause();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/855e0477/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index 820f24c..8bcaf05 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -36,6 +36,7 @@ import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.CallQueueManager.CallQueueOverflowException;
import org.apache.hadoop.metrics2.util.MBeans;
/**
@@ -134,45 +135,84 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
/* AbstractQueue and BlockingQueue methods */
/**
- * Put and offer follow the same pattern:
+ * Add, put, and offer follow the same pattern:
* 1. Get the assigned priorityLevel from the call by scheduler
* 2. Get the nth sub-queue matching this priorityLevel
* 3. delegate the call to this sub-queue.
*
* But differ in how they handle overflow:
- * - Put will move on to the next queue until it lands on the last queue
+ * - Add will move on to the next queue, throw on last queue overflow
+ * - Put will move on to the next queue, block on last queue overflow
* - Offer does not attempt other queues on overflow
*/
- @Override
- public void put(E e) throws InterruptedException {
- int priorityLevel = e.getPriorityLevel();
- final int numLevels = this.queues.size();
- while (true) {
- BlockingQueue<E> q = this.queues.get(priorityLevel);
- boolean res = q.offer(e);
- if (!res) {
- // Update stats
- this.overflowedCalls.get(priorityLevel).getAndIncrement();
-
- // If we failed to insert, try again on the next level
- priorityLevel++;
-
- if (priorityLevel == numLevels) {
- // That was the last one, we will block on put in the last queue
- // Delete this line to drop the call
- this.queues.get(priorityLevel-1).put(e);
- break;
- }
- } else {
- break;
- }
+ @Override
+ public boolean add(E e) {
+ final int priorityLevel = e.getPriorityLevel();
+ // try offering to all queues.
+ if (!offerQueues(priorityLevel, e, true)) {
+ // only disconnect the lowest priority users that overflow the queue.
+ throw (priorityLevel == queues.size() - 1)
+ ? CallQueueOverflowException.DISCONNECT
+ : CallQueueOverflowException.KEEPALIVE;
}
+ return true;
+ }
+ @Override
+ public void put(E e) throws InterruptedException {
+ final int priorityLevel = e.getPriorityLevel();
+ // try offering to all but last queue, put on last.
+ if (!offerQueues(priorityLevel, e, false)) {
+ putQueue(queues.size() - 1, e);
+ }
+ }
+ /**
+ * Put the element in a queue of a specific priority.
+ * @param priority - queue priority
+ * @param e - element to add
+ */
+ @VisibleForTesting
+ void putQueue(int priority, E e) throws InterruptedException {
+ queues.get(priority).put(e);
signalNotEmpty();
}
+ /**
+ * Offer the element to queue of a specific priority.
+ * @param priority - queue priority
+ * @param e - element to add
+ * @return boolean if added to the given queue
+ */
+ @VisibleForTesting
+ boolean offerQueue(int priority, E e) {
+ boolean ret = queues.get(priority).offer(e);
+ if (ret) {
+ signalNotEmpty();
+ }
+ return ret;
+ }
+
+ /**
+ * Offer the element to queue of the given or lower priority.
+ * @param priority - starting queue priority
+ * @param e - element to add
+ * @param includeLast - whether to attempt last queue
+ * @return boolean if added to a queue
+ */
+ private boolean offerQueues(int priority, E e, boolean includeLast) {
+ int lastPriority = queues.size() - (includeLast ? 1 : 2);
+ for (int i=priority; i <= lastPriority; i++) {
+ if (offerQueue(i, e)) {
+ return true;
+ }
+ // Update stats
+ overflowedCalls.get(i).getAndIncrement();
+ }
+ return false;
+ }
+
@Override
public boolean offer(E e, long timeout, TimeUnit unit)
throws InterruptedException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/855e0477/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index f3b9a82..df108b8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -83,6 +83,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.ipc.CallQueueManager.CallQueueOverflowException;
import org.apache.hadoop.ipc.RPC.RpcInvoker;
import org.apache.hadoop.ipc.RPC.VersionMismatch;
import org.apache.hadoop.ipc.metrics.RpcDetailedMetrics;
@@ -2479,7 +2480,9 @@ public abstract class Server {
call.setPriorityLevel(callQueue.getPriorityLevel(call));
try {
- queueCall(call);
+ internalQueueCall(call);
+ } catch (RpcServerException rse) {
+ throw rse;
} catch (IOException ioe) {
throw new FatalRpcServerException(
RpcErrorCodeProto.ERROR_RPC_SERVER, ioe);
@@ -2616,9 +2619,19 @@ public abstract class Server {
}
public void queueCall(Call call) throws IOException, InterruptedException {
- if (!callQueue.isClientBackoffEnabled()) {
+ // external non-rpc calls don't need server exception wrapper.
+ try {
+ internalQueueCall(call);
+ } catch (RpcServerException rse) {
+ throw (IOException)rse.getCause();
+ }
+ }
+
+ private void internalQueueCall(Call call)
+ throws IOException, InterruptedException {
+ try {
callQueue.put(call); // queue the call; maybe blocked here
- } else if (callQueue.shouldBackOff(call) || !callQueue.offer(call)) {
+ } catch (CallQueueOverflowException cqe) {
// If rpc scheduler indicates back off based on performance degradation
// such as response time or rpc queue is full, we will ask the client
// to back off by throwing RetriableException. Whether the client will
@@ -2626,7 +2639,8 @@ public abstract class Server {
// For example, IPC clients using FailoverOnNetworkExceptionRetry handle
// RetriableException.
rpcMetrics.incrClientBackoff();
- throw new RetriableException("Server is too busy.");
+ // unwrap retriable exception.
+ throw cqe.getCause();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/855e0477/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
index 1211657..a5a0b00 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
@@ -19,8 +19,14 @@
package org.apache.hadoop.ipc;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import java.util.ArrayList;
import java.util.HashMap;
@@ -29,8 +35,10 @@ import java.util.concurrent.LinkedBlockingQueue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ipc.CallQueueManager.CallQueueOverflowException;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestCallQueueManager {
private CallQueueManager<FakeCall> manager;
@@ -311,11 +319,21 @@ public class TestCallQueueManager {
assertEquals(totalCallsConsumed, totalCallsCreated);
}
- public static class ExceptionFakeCall {
+ public static class ExceptionFakeCall implements Schedulable {
public ExceptionFakeCall() {
throw new IllegalArgumentException("Exception caused by call queue " +
"constructor.!!");
}
+
+ @Override
+ public UserGroupInformation getUserGroupInformation() {
+ return null;
+ }
+
+ @Override
+ public int getPriorityLevel() {
+ return 0;
+ }
}
public static class ExceptionFakeScheduler {
@@ -359,4 +377,62 @@ public class TestCallQueueManager {
.getMessage());
}
}
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testCallQueueOverflowExceptions() throws Exception {
+ RpcScheduler scheduler = Mockito.mock(RpcScheduler.class);
+ BlockingQueue<Schedulable> queue = Mockito.mock(BlockingQueue.class);
+ CallQueueManager<Schedulable> cqm =
+ Mockito.spy(new CallQueueManager<>(queue, scheduler, false));
+ Schedulable call = new FakeCall(0);
+
+ // call queue exceptions passed threw as-is
+ doThrow(CallQueueOverflowException.KEEPALIVE).when(queue).add(call);
+ try {
+ cqm.add(call);
+ fail("didn't throw");
+ } catch (CallQueueOverflowException cqe) {
+ assertSame(CallQueueOverflowException.KEEPALIVE, cqe);
+ }
+
+ // standard exception for blocking queue full converted to overflow
+ // exception.
+ doThrow(new IllegalStateException()).when(queue).add(call);
+ try {
+ cqm.add(call);
+ fail("didn't throw");
+ } catch (Exception ex) {
+ assertTrue(ex.toString(), ex instanceof CallQueueOverflowException);
+ }
+
+ // backoff disabled, put is put to queue.
+ reset(queue);
+ cqm.setClientBackoffEnabled(false);
+ cqm.put(call);
+ verify(queue, times(1)).put(call);
+ verify(queue, times(0)).add(call);
+
+ // backoff enabled, put is add to queue.
+ reset(queue);
+ cqm.setClientBackoffEnabled(true);
+ doReturn(Boolean.FALSE).when(cqm).shouldBackOff(call);
+ cqm.put(call);
+ verify(queue, times(0)).put(call);
+ verify(queue, times(1)).add(call);
+ reset(queue);
+
+ // backoff is enabled, put + scheduler backoff = overflow exception.
+ reset(queue);
+ cqm.setClientBackoffEnabled(true);
+ doReturn(Boolean.TRUE).when(cqm).shouldBackOff(call);
+ try {
+ cqm.put(call);
+ fail("didn't fail");
+ } catch (Exception ex) {
+ assertTrue(ex.toString(), ex instanceof CallQueueOverflowException);
+ }
+ verify(queue, times(0)).put(call);
+ verify(queue, times(0)).add(call);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/855e0477/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
index 901a771..6b1cd29 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
@@ -18,13 +18,19 @@
package org.apache.hadoop.ipc;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.anyObject;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.times;
import junit.framework.TestCase;
import javax.management.MBeanServer;
import javax.management.ObjectName;
+
+import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -34,7 +40,10 @@ import java.util.List;
import java.util.concurrent.BlockingQueue;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Test;
+import org.mockito.Mockito;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.CallQueueManager.CallQueueOverflowException;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto;
public class TestFairCallQueue extends TestCase {
private FairCallQueue<Schedulable> fcq;
@@ -133,6 +142,153 @@ public class TestFairCallQueue extends TestCase {
assertNull(fcq.poll());
}
+ @SuppressWarnings("unchecked") // for mock reset.
+ @Test
+ public void testInsertion() throws Exception {
+ Configuration conf = new Configuration();
+ // 3 queues, 2 slots each.
+ fcq = Mockito.spy(new FairCallQueue<Schedulable>(3, 6, "ns", conf));
+
+ Schedulable p0 = mockCall("a", 0);
+ Schedulable p1 = mockCall("b", 1);
+ Schedulable p2 = mockCall("c", 2);
+
+ // add to first queue.
+ Mockito.reset(fcq);
+ fcq.add(p0);
+ Mockito.verify(fcq, times(1)).offerQueue(0, p0);
+ Mockito.verify(fcq, times(0)).offerQueue(1, p0);
+ Mockito.verify(fcq, times(0)).offerQueue(2, p0);
+ Mockito.reset(fcq);
+ // 0:x- 1:-- 2:--
+
+ // add to second queue.
+ Mockito.reset(fcq);
+ fcq.add(p1);
+ Mockito.verify(fcq, times(0)).offerQueue(0, p1);
+ Mockito.verify(fcq, times(1)).offerQueue(1, p1);
+ Mockito.verify(fcq, times(0)).offerQueue(2, p1);
+ // 0:x- 1:x- 2:--
+
+ // add to first queue.
+ Mockito.reset(fcq);
+ fcq.add(p0);
+ Mockito.verify(fcq, times(1)).offerQueue(0, p0);
+ Mockito.verify(fcq, times(0)).offerQueue(1, p0);
+ Mockito.verify(fcq, times(0)).offerQueue(2, p0);
+ // 0:xx 1:x- 2:--
+
+ // add to first full queue spills over to second.
+ Mockito.reset(fcq);
+ fcq.add(p0);
+ Mockito.verify(fcq, times(1)).offerQueue(0, p0);
+ Mockito.verify(fcq, times(1)).offerQueue(1, p0);
+ Mockito.verify(fcq, times(0)).offerQueue(2, p0);
+ // 0:xx 1:xx 2:--
+
+ // add to second full queue spills over to third.
+ Mockito.reset(fcq);
+ fcq.add(p1);
+ Mockito.verify(fcq, times(0)).offerQueue(0, p1);
+ Mockito.verify(fcq, times(1)).offerQueue(1, p1);
+ Mockito.verify(fcq, times(1)).offerQueue(2, p1);
+ // 0:xx 1:xx 2:x-
+
+ // add to first and second full queue spills over to third.
+ Mockito.reset(fcq);
+ fcq.add(p0);
+ Mockito.verify(fcq, times(1)).offerQueue(0, p0);
+ Mockito.verify(fcq, times(1)).offerQueue(1, p0);
+ Mockito.verify(fcq, times(1)).offerQueue(2, p0);
+ // 0:xx 1:xx 2:xx
+
+ // adding non-lowest priority with all queues full throws a
+ // non-disconnecting rpc server exception.
+ Mockito.reset(fcq);
+ try {
+ fcq.add(p0);
+ fail("didn't fail");
+ } catch (IllegalStateException ise) {
+ checkOverflowException(ise, RpcStatusProto.ERROR);
+ }
+ Mockito.verify(fcq, times(1)).offerQueue(0, p0);
+ Mockito.verify(fcq, times(1)).offerQueue(1, p0);
+ Mockito.verify(fcq, times(1)).offerQueue(2, p0);
+
+ // adding non-lowest priority with all queues full throws a
+ // non-disconnecting rpc server exception.
+ Mockito.reset(fcq);
+ try {
+ fcq.add(p1);
+ fail("didn't fail");
+ } catch (IllegalStateException ise) {
+ checkOverflowException(ise, RpcStatusProto.ERROR);
+ }
+ Mockito.verify(fcq, times(0)).offerQueue(0, p1);
+ Mockito.verify(fcq, times(1)).offerQueue(1, p1);
+ Mockito.verify(fcq, times(1)).offerQueue(2, p1);
+
+ // adding lowest priority with all queues full throws a
+ // fatal disconnecting rpc server exception.
+ Mockito.reset(fcq);
+ try {
+ fcq.add(p2);
+ fail("didn't fail");
+ } catch (IllegalStateException ise) {
+ checkOverflowException(ise, RpcStatusProto.FATAL);
+ }
+ Mockito.verify(fcq, times(0)).offerQueue(0, p2);
+ Mockito.verify(fcq, times(0)).offerQueue(1, p2);
+ Mockito.verify(fcq, times(1)).offerQueue(2, p2);
+ Mockito.reset(fcq);
+
+ // used to abort what would be a blocking operation.
+ Exception stopPuts = new RuntimeException();
+
+ // put should offer to all but last subqueue, only put to last subqueue.
+ Mockito.reset(fcq);
+ try {
+ doThrow(stopPuts).when(fcq).putQueue(anyInt(), anyObject());
+ fcq.put(p0);
+ fail("didn't fail");
+ } catch (Exception e) {
+ assertSame(stopPuts, e);
+ }
+ Mockito.verify(fcq, times(1)).offerQueue(0, p0);
+ Mockito.verify(fcq, times(1)).offerQueue(1, p0);
+ Mockito.verify(fcq, times(0)).offerQueue(2, p0); // expect put, not offer.
+ Mockito.verify(fcq, times(1)).putQueue(2, p0);
+
+ // put with lowest priority should not offer, just put.
+ Mockito.reset(fcq);
+ try {
+ doThrow(stopPuts).when(fcq).putQueue(anyInt(), anyObject());
+ fcq.put(p2);
+ fail("didn't fail");
+ } catch (Exception e) {
+ assertSame(stopPuts, e);
+ }
+ Mockito.verify(fcq, times(0)).offerQueue(0, p2);
+ Mockito.verify(fcq, times(0)).offerQueue(1, p2);
+ Mockito.verify(fcq, times(0)).offerQueue(2, p2);
+ Mockito.verify(fcq, times(1)).putQueue(2, p2);
+ }
+
+ private void checkOverflowException(Exception ex, RpcStatusProto status) {
+ // should be an overflow exception
+ assertTrue(ex.getClass().getName() + " != CallQueueOverflowException",
+ ex instanceof CallQueueOverflowException);
+ IOException ioe = ((CallQueueOverflowException)ex).getCause();
+ assertNotNull(ioe);
+ assertTrue(ioe.getClass().getName() + " != RpcServerException",
+ ioe instanceof RpcServerException);
+ RpcServerException rse = (RpcServerException)ioe;
+ // check error/fatal status and if it embeds a retriable ex.
+ assertEquals(status, rse.getRpcStatusProto());
+ assertTrue(rse.getClass().getName() + " != RetriableException",
+ rse.getCause() instanceof RetriableException);
+ }
+
//
// Ensure that FairCallQueue properly implements BlockingQueue
//
http://git-wip-us.apache.org/repos/asf/hadoop/blob/855e0477/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
index 3cc9916..166b205 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
@@ -1123,7 +1123,7 @@ public class TestRPC extends TestRpcBase {
return null;
}
}));
- verify(spy, timeout(500).times(i + 1)).offer(Mockito.<Call>anyObject());
+ verify(spy, timeout(500).times(i + 1)).add(Mockito.<Call>anyObject());
}
try {
proxy.sleep(null, newSleepRequest(100));
@@ -1194,7 +1194,7 @@ public class TestRPC extends TestRpcBase {
return null;
}
}));
- verify(spy, timeout(500).times(i + 1)).offer(Mockito.<Call>anyObject());
+ verify(spy, timeout(500).times(i + 1)).add(Mockito.<Call>anyObject());
}
// Start another sleep RPC call and verify the call is backed off due to
// avg response time(3s) exceeds threshold (2s).
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org