You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2016/11/30 05:00:51 UTC
[01/48] hadoop git commit: YARN-5911. DrainDispatcher does not drain
all events on stop even if setDrainEventsOnStop is true. Contributed by Varun
Saxena.
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 86a67ffac -> 5e93093e6
YARN-5911. DrainDispatcher does not drain all events on stop even if setDrainEventsOnStop is true. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46675641
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46675641
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46675641
Branch: refs/heads/HADOOP-13345
Commit: 466756416214a4bbc77af8a29da1a33e01106864
Parents: 2bf9a15
Author: Naganarasimha <na...@apache.org>
Authored: Wed Nov 23 08:44:58 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Wed Nov 23 08:49:48 2016 +0530
----------------------------------------------------------------------
.../hadoop/yarn/event/AsyncDispatcher.java | 6 ++-
.../hadoop/yarn/event/DrainDispatcher.java | 9 +----
.../hadoop/yarn/event/TestAsyncDispatcher.java | 42 ++++++++++++++++++++
3 files changed, 48 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46675641/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 42a6819..94bfab6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -151,7 +151,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
while (!isDrained() && eventHandlingThread != null
&& eventHandlingThread.isAlive()
&& System.currentTimeMillis() < endTime) {
- waitForDrained.wait(1000);
+ waitForDrained.wait(100);
LOG.info("Waiting for AsyncDispatcher to drain. Thread state is :" +
eventHandlingThread.getState());
}
@@ -308,4 +308,8 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
protected boolean isDrained() {
return drained;
}
+
+ protected boolean isStopped() {
+ return stopped;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46675641/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
index 1369465..c5ba072 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
@@ -25,7 +25,6 @@ import java.util.concurrent.LinkedBlockingQueue;
@SuppressWarnings("rawtypes")
public class DrainDispatcher extends AsyncDispatcher {
private volatile boolean drained = false;
- private volatile boolean stopped = false;
private final BlockingQueue<Event> queue;
private final Object mutex;
@@ -69,7 +68,7 @@ public class DrainDispatcher extends AsyncDispatcher {
return new Runnable() {
@Override
public void run() {
- while (!stopped && !Thread.currentThread().isInterrupted()) {
+ while (!isStopped() && !Thread.currentThread().isInterrupted()) {
synchronized (mutex) {
// !drained if dispatch queued new events on this dispatcher
drained = queue.isEmpty();
@@ -109,10 +108,4 @@ public class DrainDispatcher extends AsyncDispatcher {
return drained;
}
}
-
- @Override
- protected void serviceStop() throws Exception {
- stopped = true;
- super.serviceStop();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46675641/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index 018096b..2b9d745 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.junit.Assert;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.*;
public class TestAsyncDispatcher {
@@ -77,5 +78,46 @@ public class TestAsyncDispatcher {
disp.waitForEventThreadToWait();
disp.close();
}
+
+ @SuppressWarnings("rawtypes")
+ private static class DummyHandler implements EventHandler<Event> {
+ @Override
+ public void handle(Event event) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {}
+ }
+ }
+
+ private enum DummyType {
+ DUMMY
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private void dispatchDummyEvents(Dispatcher disp, int count) {
+ for (int i = 0; i < count; i++) {
+ Event event = mock(Event.class);
+ when(event.getType()).thenReturn(DummyType.DUMMY);
+ disp.getEventHandler().handle(event);
+ }
+ }
+
+ // Test if drain dispatcher drains events on stop.
+ @SuppressWarnings({ "rawtypes" })
+ @Test(timeout=10000)
+ public void testDrainDispatcherDrainEventsOnStop() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 2000);
+ BlockingQueue<Event> queue = new LinkedBlockingQueue<Event>();
+ DrainDispatcher disp = new DrainDispatcher(queue);
+ disp.init(conf);
+ disp.register(DummyType.class, new DummyHandler());
+ disp.setDrainEventsOnStop();
+ disp.start();
+ disp.waitForEventThreadToWait();
+ dispatchDummyEvents(disp, 2);
+ disp.close();
+ assertEquals(0, queue.size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/48] hadoop git commit: HADOOP-13816. Ambiguous plugin version
warning from maven build. Contributed by Kai Sasaki.
Posted by li...@apache.org.
HADOOP-13816. Ambiguous plugin version warning from maven build. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e173d2b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e173d2b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e173d2b
Branch: refs/heads/HADOOP-13345
Commit: 9e173d2bac567d3bdec2c6c4e15a72bca9c4f6d3
Parents: c06114d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Nov 28 13:10:31 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Nov 28 13:10:31 2016 +0900
----------------------------------------------------------------------
hadoop-build-tools/pom.xml | 2 ++
hadoop-tools/hadoop-rumen/pom.xml | 5 -----
pom.xml | 2 ++
3 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e173d2b/hadoop-build-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-build-tools/pom.xml b/hadoop-build-tools/pom.xml
index 26c8aa6..a713f36 100644
--- a/hadoop-build-tools/pom.xml
+++ b/hadoop-build-tools/pom.xml
@@ -54,6 +54,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
+ <version>${maven-resources-plugin.version}</version>
<executions>
<execution>
<id>copy-resources</id>
@@ -80,6 +81,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-remote-resources-plugin</artifactId>
+ <version>${maven-remote-resources-plugin.version}</version>
<executions>
<execution>
<phase>process-resources</phase>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e173d2b/hadoop-tools/hadoop-rumen/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml
index 9e1b1f3..bc873f7 100644
--- a/hadoop-tools/hadoop-rumen/pom.xml
+++ b/hadoop-tools/hadoop-rumen/pom.xml
@@ -97,11 +97,6 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e173d2b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 101d79f..8776534 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,6 +100,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<maven-enforcer-plugin.version>1.4.1</maven-enforcer-plugin.version>
<maven-javadoc-plugin.version>2.9.1</maven-javadoc-plugin.version>
<maven-gpg-plugin.version>1.5</maven-gpg-plugin.version>
+ <maven-remote-resources-plugin.version>1.5</maven-remote-resources-plugin.version>
+ <maven-resources-plugin.version>3.0.1</maven-resources-plugin.version>
<apache-rat-plugin.version>0.10</apache-rat-plugin.version>
<wagon-ssh.version>1.0</wagon-ssh.version>
<maven-clover2-plugin.version>3.3.0</maven-clover2-plugin.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/48] hadoop git commit: HDFS-11368. Erasure Coding: Deprecate
replication-related config keys. Contributed by Rakesh R.
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
index 4fc5029..dc7f47a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
@@ -142,8 +142,9 @@ public class TestPipelinesFailover {
MethodToTestIdempotence methodToTest) throws Exception {
Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
- // Don't check replication periodically.
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000);
+ // Don't check low redundancy periodically.
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 1000);
FSDataOutputStream stm = null;
MiniDFSCluster cluster = newMiniCluster(conf, 3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNNMetricFilesInGetListingOps.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNNMetricFilesInGetListingOps.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNNMetricFilesInGetListingOps.java
index 110615d..c11f880 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNNMetricFilesInGetListingOps.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNNMetricFilesInGetListingOps.java
@@ -45,7 +45,7 @@ public class TestNNMetricFilesInGetListingOps {
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 100);
CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
- CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
}
private MiniDFSCluster cluster;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index 90d61ee..0ad6130 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -78,7 +78,7 @@ import org.junit.Test;
*/
public class TestNameNodeMetrics {
private static final Configuration CONF = new HdfsConfiguration();
- private static final int DFS_REPLICATION_INTERVAL = 1;
+ private static final int DFS_REDUNDANCY_INTERVAL = 1;
private static final Path TEST_ROOT_DIR_PATH =
new Path("/testNameNodeMetrics");
private static final String NN_METRICS = "NameNodeActivity";
@@ -96,9 +96,9 @@ public class TestNameNodeMetrics {
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 100);
CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFS_REPLICATION_INTERVAL);
- CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFS_REPLICATION_INTERVAL);
+ DFS_REDUNDANCY_INTERVAL);
+ CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ DFS_REDUNDANCY_INTERVAL);
CONF.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY,
"" + PERCENTILES_INTERVAL);
// Enable stale DataNodes checking
@@ -333,7 +333,7 @@ public class TestNameNodeMetrics {
private void waitForDeletion() throws InterruptedException {
// Wait for more than DATANODE_COUNT replication intervals to ensure all
// the blocks pending deletion are sent for deletion to the datanodes.
- Thread.sleep(DFS_REPLICATION_INTERVAL * (DATANODE_COUNT + 1) * 1000);
+ Thread.sleep(DFS_REDUNDANCY_INTERVAL * (DATANODE_COUNT + 1) * 1000);
}
/**
@@ -364,7 +364,7 @@ public class TestNameNodeMetrics {
rb = getMetrics(source);
gauge = MetricsAsserts.getLongGauge(name, rb);
while (gauge != expected && (--retries > 0)) {
- Thread.sleep(DFS_REPLICATION_INTERVAL * 500);
+ Thread.sleep(DFS_REDUNDANCY_INTERVAL * 500);
rb = getMetrics(source);
gauge = MetricsAsserts.getLongGauge(name, rb);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/48] hadoop git commit: MAPREDUCE-6565. Configuration to use host
name in delegation token service is not read from job.xml during MapReduce
job execution. Contributed by Li Lu.
Posted by li...@apache.org.
MAPREDUCE-6565. Configuration to use host name in delegation token service is not read from job.xml during MapReduce job execution. Contributed by Li Lu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f6e1439
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f6e1439
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f6e1439
Branch: refs/heads/HADOOP-13345
Commit: 8f6e14399a3e77e1bdcc5034f7601e9f62163dea
Parents: 6d8b4f6
Author: Junping Du <ju...@apache.org>
Authored: Tue Nov 29 15:51:27 2016 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Tue Nov 29 15:51:27 2016 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/mapred/YarnChild.java | 2 ++
.../main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java | 3 +++
2 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f6e1439/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
index 164f19d..97642a5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
@@ -78,6 +78,8 @@ class YarnChild {
// Initing with our JobConf allows us to avoid loading confs twice
Limits.init(job);
UserGroupInformation.setConfiguration(job);
+ // MAPREDUCE-6565: need to set configuration for SecurityUtil.
+ SecurityUtil.setConfiguration(job);
String host = args[0];
int port = Integer.parseInt(args[1]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f6e1439/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 4a8a90e..b383a02 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -123,6 +123,7 @@ import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
@@ -1690,6 +1691,8 @@ public class MRAppMaster extends CompositeService {
final JobConf conf, String jobUserName) throws IOException,
InterruptedException {
UserGroupInformation.setConfiguration(conf);
+ // MAPREDUCE-6565: need to set configuration for SecurityUtil.
+ SecurityUtil.setConfiguration(conf);
// Security framework already loaded the tokens into current UGI, just use
// them
Credentials credentials =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/48] hadoop git commit: YARN-5725. Test uncaught exception in
TestContainersMonitorResourceChange.testContainersResourceChange when setting
IP and host (Contributed by Miklos Szegedi via Daniel Templeton)
Posted by li...@apache.org.
YARN-5725. Test uncaught exception in TestContainersMonitorResourceChange.testContainersResourceChange
when setting IP and host (Contributed by Miklos Szegedi via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62b42ef5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62b42ef5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62b42ef5
Branch: refs/heads/HADOOP-13345
Commit: 62b42ef5dd04d516d33bf0890ac5cd49f8184a73
Parents: a2b1ff0
Author: Daniel Templeton <te...@apache.org>
Authored: Mon Nov 28 15:16:43 2016 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Mon Nov 28 15:19:52 2016 -0800
----------------------------------------------------------------------
.../monitor/ContainersMonitorImpl.java | 36 +++++++++++++-------
.../TestContainersMonitorResourceChange.java | 16 +++++++--
2 files changed, 36 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62b42ef5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index dda55d2..718604d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -536,15 +536,21 @@ public class ContainersMonitorImpl extends AbstractService implements
}
Container container = context.getContainers().get(containerId);
- String[] ipAndHost = containerExecutor.getIpAndHost(container);
- if (ipAndHost != null && ipAndHost[0] != null
- && ipAndHost[1] != null) {
- container.setIpAndHost(ipAndHost);
- LOG.info(containerId + "'s ip = " + ipAndHost[0]
- + ", and hostname = " + ipAndHost[1]);
+
+ if (container != null) {
+ String[] ipAndHost = containerExecutor.getIpAndHost(container);
+
+ if ((ipAndHost != null) && (ipAndHost[0] != null) &&
+ (ipAndHost[1] != null)) {
+ container.setIpAndHost(ipAndHost);
+ LOG.info(containerId + "'s ip = " + ipAndHost[0]
+ + ", and hostname = " + ipAndHost[1]);
+ } else {
+ LOG.info("Can not get both ip and hostname: "
+ + Arrays.toString(ipAndHost));
+ }
} else {
- LOG.info("Can not get both ip and hostname: " + Arrays
- .toString(ipAndHost));
+ LOG.info(containerId + " is missing. Not setting ip and hostname");
}
}
}
@@ -683,11 +689,15 @@ public class ContainersMonitorImpl extends AbstractService implements
long currentPmemUsage, float cpuUsagePercentPerCore) {
ContainerImpl container =
(ContainerImpl) context.getContainers().get(containerId);
- NMTimelinePublisher nmMetricsPublisher =
- container.getNMTimelinePublisher();
- if (nmMetricsPublisher != null) {
- nmMetricsPublisher.reportContainerResourceUsage(container,
- currentPmemUsage, cpuUsagePercentPerCore);
+ if (container != null) {
+ NMTimelinePublisher nmMetricsPublisher =
+ container.getNMTimelinePublisher();
+ if (nmMetricsPublisher != null) {
+ nmMetricsPublisher.reportContainerResourceUsage(container,
+ currentPmemUsage, cpuUsagePercentPerCore);
+ }
+ } else {
+ LOG.info(containerId + " does not exist to report");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62b42ef5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index d24f89d..29a60f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.hadoop.conf.Configuration;
@@ -37,6 +38,7 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl.ProcessTreeInfo;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
@@ -64,6 +66,7 @@ public class TestContainersMonitorResourceChange {
private AsyncDispatcher dispatcher;
private Context context;
private MockContainerEventHandler containerEventHandler;
+ private ConcurrentMap<ContainerId, Container> containerMap;
static final int WAIT_MS_PER_LOOP = 20; // 20 milli seconds
@@ -131,8 +134,10 @@ public class TestContainersMonitorResourceChange {
executor = new MockExecutor();
dispatcher = new AsyncDispatcher();
context = Mockito.mock(Context.class);
- Mockito.doReturn(new ConcurrentSkipListMap<ContainerId, Container>())
- .when(context).getContainers();
+ containerMap = new ConcurrentSkipListMap<>();
+ Container container = Mockito.mock(ContainerImpl.class);
+ containerMap.put(getContainerId(1), container);
+ Mockito.doReturn(containerMap).when(context).getContainers();
conf = new Configuration();
conf.set(
YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
@@ -181,7 +186,12 @@ public class TestContainersMonitorResourceChange {
getContainerId(1)).getProcessTree();
mockTree.setRssMemorySize(2500L);
// verify that this container is killed
- Thread.sleep(200);
+ for (int waitMs = 0; waitMs < 5000; waitMs += 50) {
+ if (containerEventHandler.isContainerKilled(getContainerId(1))) {
+ break;
+ }
+ Thread.sleep(50);
+ }
assertTrue(containerEventHandler
.isContainerKilled(getContainerId(1)));
// create container 2
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/48] hadoop git commit: YARN-4395. Typo in comment in
ClientServiceDelegate (Contributed by Alison Yu via Daniel Templeton)
Posted by li...@apache.org.
YARN-4395. Typo in comment in ClientServiceDelegate (Contributed by Alison Yu via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d8b4f6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d8b4f6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d8b4f6c
Branch: refs/heads/HADOOP-13345
Commit: 6d8b4f6c2791f861a55ac78c2950f783693e912a
Parents: dd9a96c
Author: Daniel Templeton <te...@apache.org>
Authored: Tue Nov 29 15:30:22 2016 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Tue Nov 29 15:30:22 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapred/ClientServiceDelegate.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d8b4f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
index eac8dbc..72339e5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
@@ -335,8 +335,8 @@ public class ClientServiceDelegate {
throw new IOException(e.getTargetException());
}
- // if it's AM shut down, do not decrement maxClientRetry as we wait for
- // AM to be restarted.
+ // if its AM shut down, do not decrement maxClientRetry while we wait
+ // for its AM to be restarted.
if (!usingAMProxy.get()) {
maxClientRetry--;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/48] hadoop git commit: Revert "HDFS-5517. Lower the default
maximum number of blocks per file. Contributed by Aaron T. Myers."
Posted by li...@apache.org.
Revert "HDFS-5517. Lower the default maximum number of blocks per file. Contributed by Aaron T. Myers."
This reverts commit 09451252fae90a3ec192b8d7f0c49508df65e8c5.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b407d531
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b407d531
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b407d531
Branch: refs/heads/HADOOP-13345
Commit: b407d5319597f01ebd51736e299239022bd08028
Parents: 25f9872
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Nov 29 10:50:10 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Nov 29 10:50:10 2016 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b407d531/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index df21857..d7d3c9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -399,7 +399,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_MIN_BLOCK_SIZE_KEY = "dfs.namenode.fs-limits.min-block-size";
public static final long DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT = 1024*1024;
public static final String DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY = "dfs.namenode.fs-limits.max-blocks-per-file";
- public static final long DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT = 10*1000;
+ public static final long DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT = 1024*1024;
public static final String DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY = "dfs.namenode.fs-limits.max-xattrs-per-inode";
public static final int DFS_NAMENODE_MAX_XATTRS_PER_INODE_DEFAULT = 32;
public static final String DFS_NAMENODE_MAX_XATTR_SIZE_KEY = "dfs.namenode.fs-limits.max-xattr-size";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b407d531/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 9fce84f..c9d74bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -372,7 +372,7 @@
<property>
<name>dfs.namenode.fs-limits.max-blocks-per-file</name>
- <value>10000</value>
+ <value>1048576</value>
<description>Maximum number of blocks per file, enforced by the Namenode on
write. This prevents the creation of extremely large files which can
degrade performance.</description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/48] hadoop git commit: MAPREDUCE-6743. nativetask unit tests need
to provide usable output; fix link errors during mvn test
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be88d574/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
index 4f3804f..c04205d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
@@ -126,8 +126,11 @@
// Authors: wan@google.com (Zhanyong Wan)
//
// Low-level types and utilities for porting Google Test to various
-// platforms. They are subject to change without notice. DO NOT USE
-// THEM IN USER CODE.
+// platforms. All macros ending with _ and symbols defined in an
+// internal namespace are subject to change without notice. Code
+// outside Google Test MUST NOT USE THEM DIRECTLY. Macros that don't
+// end with _ are part of Google Test's public API and can be used by
+// code outside Google Test.
//
// This file is fundamental to Google Test. All other Google Test source
// files are expected to #include this. Therefore, it cannot #include
@@ -136,9 +139,30 @@
#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_
#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_
-// The user can define the following macros in the build script to
-// control Google Test's behavior. If the user doesn't define a macro
-// in this list, Google Test will define it.
+// Environment-describing macros
+// -----------------------------
+//
+// Google Test can be used in many different environments. Macros in
+// this section tell Google Test what kind of environment it is being
+// used in, such that Google Test can provide environment-specific
+// features and implementations.
+//
+// Google Test tries to automatically detect the properties of its
+// environment, so users usually don't need to worry about these
+// macros. However, the automatic detection is not perfect.
+// Sometimes it's necessary for a user to define some of the following
+// macros in the build script to override Google Test's decisions.
+//
+// If the user doesn't define a macro in the list, Google Test will
+// provide a default definition. After this header is #included, all
+// macros in this list will be defined to either 1 or 0.
+//
+// Notes to maintainers:
+// - Each macro here is a user-tweakable knob; do not grow the list
+// lightly.
+// - Use #if to key off these macros. Don't use #ifdef or "#if
+// defined(...)", which will not work as these macros are ALWAYS
+// defined.
//
// GTEST_HAS_CLONE - Define it to 1/0 to indicate that clone(2)
// is/isn't available.
@@ -182,18 +206,23 @@
// - Define to 1 when compiling Google Test itself
// as a shared library.
-// This header defines the following utilities:
+// Platform-indicating macros
+// --------------------------
+//
+// Macros indicating the platform on which Google Test is being used
+// (a macro is defined to 1 if compiled on the given platform;
+// otherwise UNDEFINED -- it's never defined to 0.). Google Test
+// defines these macros automatically. Code outside Google Test MUST
+// NOT define them.
//
-// Macros indicating the current platform (defined to 1 if compiled on
-// the given platform; otherwise undefined):
// GTEST_OS_AIX - IBM AIX
// GTEST_OS_CYGWIN - Cygwin
+// GTEST_OS_FREEBSD - FreeBSD
// GTEST_OS_HPUX - HP-UX
// GTEST_OS_LINUX - Linux
// GTEST_OS_LINUX_ANDROID - Google Android
// GTEST_OS_MAC - Mac OS X
// GTEST_OS_IOS - iOS
-// GTEST_OS_IOS_SIMULATOR - iOS simulator
// GTEST_OS_NACL - Google Native Client (NaCl)
// GTEST_OS_OPENBSD - OpenBSD
// GTEST_OS_QNX - QNX
@@ -203,6 +232,8 @@
// GTEST_OS_WINDOWS_DESKTOP - Windows Desktop
// GTEST_OS_WINDOWS_MINGW - MinGW
// GTEST_OS_WINDOWS_MOBILE - Windows Mobile
+// GTEST_OS_WINDOWS_PHONE - Windows Phone
+// GTEST_OS_WINDOWS_RT - Windows Store App/WinRT
// GTEST_OS_ZOS - z/OS
//
// Among the platforms, Cygwin, Linux, Max OS X, and Windows have the
@@ -212,22 +243,50 @@
// googletestframework@googlegroups.com (patches for fixing them are
// even more welcome!).
//
-// Note that it is possible that none of the GTEST_OS_* macros are defined.
+// It is possible that none of the GTEST_OS_* macros are defined.
+
+// Feature-indicating macros
+// -------------------------
+//
+// Macros indicating which Google Test features are available (a macro
+// is defined to 1 if the corresponding feature is supported;
+// otherwise UNDEFINED -- it's never defined to 0.). Google Test
+// defines these macros automatically. Code outside Google Test MUST
+// NOT define them.
+//
+// These macros are public so that portable tests can be written.
+// Such tests typically surround code using a feature with an #if
+// which controls that code. For example:
+//
+// #if GTEST_HAS_DEATH_TEST
+// EXPECT_DEATH(DoSomethingDeadly());
+// #endif
//
-// Macros indicating available Google Test features (defined to 1 if
-// the corresponding feature is supported; otherwise undefined):
// GTEST_HAS_COMBINE - the Combine() function (for value-parameterized
// tests)
// GTEST_HAS_DEATH_TEST - death tests
// GTEST_HAS_PARAM_TEST - value-parameterized tests
// GTEST_HAS_TYPED_TEST - typed tests
// GTEST_HAS_TYPED_TEST_P - type-parameterized tests
+// GTEST_IS_THREADSAFE - Google Test is thread-safe.
// GTEST_USES_POSIX_RE - enhanced POSIX regex is used. Do not confuse with
// GTEST_HAS_POSIX_RE (see above) which users can
// define themselves.
// GTEST_USES_SIMPLE_RE - our own simple regex is used;
// the above two are mutually exclusive.
// GTEST_CAN_COMPARE_NULL - accepts untyped NULL in EXPECT_EQ().
+
+// Misc public macros
+// ------------------
+//
+// GTEST_FLAG(flag_name) - references the variable corresponding to
+// the given Google Test flag.
+
+// Internal utilities
+// ------------------
+//
+// The following macros and utilities are for Google Test's INTERNAL
+// use only. Code outside Google Test MUST NOT USE THEM DIRECTLY.
//
// Macros for basic C++ coding:
// GTEST_AMBIGUOUS_ELSE_BLOCKER_ - for disabling a gcc warning.
@@ -236,13 +295,18 @@
// GTEST_DISALLOW_ASSIGN_ - disables operator=.
// GTEST_DISALLOW_COPY_AND_ASSIGN_ - disables copy ctor and operator=.
// GTEST_MUST_USE_RESULT_ - declares that a function's result must be used.
+// GTEST_INTENTIONAL_CONST_COND_PUSH_ - start code section where MSVC C4127 is
+// suppressed (constant conditional).
+// GTEST_INTENTIONAL_CONST_COND_POP_ - finish code section where MSVC C4127
+// is suppressed.
+//
+// C++11 feature wrappers:
+//
+// testing::internal::move - portability wrapper for std::move.
//
// Synchronization:
// Mutex, MutexLock, ThreadLocal, GetThreadCount()
-// - synchronization primitives.
-// GTEST_IS_THREADSAFE - defined to 1 to indicate that the above
-// synchronization primitives have real implementations
-// and Google Test is thread-safe; or 0 otherwise.
+// - synchronization primitives.
//
// Template meta programming:
// is_pointer - as in TR1; needed on Symbian and IBM XL C/C++ only.
@@ -278,7 +342,6 @@
// BiggestInt - the biggest signed integer type.
//
// Command-line utilities:
-// GTEST_FLAG() - references a flag.
// GTEST_DECLARE_*() - declares a flag.
// GTEST_DEFINE_*() - defines a flag.
// GetInjectableArgvs() - returns the command line as a vector of strings.
@@ -304,23 +367,49 @@
# include <TargetConditionals.h>
#endif
+#include <algorithm> // NOLINT
#include <iostream> // NOLINT
#include <sstream> // NOLINT
#include <string> // NOLINT
+#include <utility>
+#include <vector> // NOLINT
-#define GTEST_DEV_EMAIL_ "googletestframework@@googlegroups.com"
-#define GTEST_FLAG_PREFIX_ "gtest_"
-#define GTEST_FLAG_PREFIX_DASH_ "gtest-"
-#define GTEST_FLAG_PREFIX_UPPER_ "GTEST_"
-#define GTEST_NAME_ "Google Test"
-#define GTEST_PROJECT_URL_ "http://code.google.com/p/googletest/"
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// The Google C++ Testing Framework (Google Test)
+//
+// This header file defines the GTEST_OS_* macro.
+// It is separate from gtest-port.h so that custom/gtest-port.h can include it.
-// Determines the version of gcc that is used to compile this.
-#ifdef __GNUC__
-// 40302 means version 4.3.2.
-# define GTEST_GCC_VER_ \
- (__GNUC__*10000 + __GNUC_MINOR__*100 + __GNUC_PATCHLEVEL__)
-#endif // __GNUC__
+#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_
+#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_
// Determines the platform on which Google Test is compiled.
#ifdef __CYGWIN__
@@ -333,6 +422,19 @@
# define GTEST_OS_WINDOWS_MOBILE 1
# elif defined(__MINGW__) || defined(__MINGW32__)
# define GTEST_OS_WINDOWS_MINGW 1
+# elif defined(WINAPI_FAMILY)
+# include <winapifamily.h>
+# if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
+# define GTEST_OS_WINDOWS_DESKTOP 1
+# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PHONE_APP)
+# define GTEST_OS_WINDOWS_PHONE 1
+# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP)
+# define GTEST_OS_WINDOWS_RT 1
+# else
+ // WINAPI_FAMILY defined but no known partition matched.
+ // Default to desktop.
+# define GTEST_OS_WINDOWS_DESKTOP 1
+# endif
# else
# define GTEST_OS_WINDOWS_DESKTOP 1
# endif // _WIN32_WCE
@@ -340,10 +442,9 @@
# define GTEST_OS_MAC 1
# if TARGET_OS_IPHONE
# define GTEST_OS_IOS 1
-# if TARGET_IPHONE_SIMULATOR
-# define GTEST_OS_IOS_SIMULATOR 1
-# endif
# endif
+#elif defined __FreeBSD__
+# define GTEST_OS_FREEBSD 1
#elif defined __linux__
# define GTEST_OS_LINUX 1
# if defined __ANDROID__
@@ -365,6 +466,114 @@
# define GTEST_OS_QNX 1
#endif // __CYGWIN__
+#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Injection point for custom user configurations.
+// The following macros can be defined:
+//
+// Flag related macros:
+// GTEST_FLAG(flag_name)
+// GTEST_USE_OWN_FLAGFILE_FLAG_ - Define to 0 when the system provides its
+// own flagfile flag parsing.
+// GTEST_DECLARE_bool_(name)
+// GTEST_DECLARE_int32_(name)
+// GTEST_DECLARE_string_(name)
+// GTEST_DEFINE_bool_(name, default_val, doc)
+// GTEST_DEFINE_int32_(name, default_val, doc)
+// GTEST_DEFINE_string_(name, default_val, doc)
+//
+// Test filtering:
+// GTEST_TEST_FILTER_ENV_VAR_ - The name of an environment variable that
+// will be used if --GTEST_FLAG(test_filter)
+// is not provided.
+//
+// Logging:
+// GTEST_LOG_(severity)
+// GTEST_CHECK_(condition)
+// Functions LogToStderr() and FlushInfoLog() have to be provided too.
+//
+// Threading:
+// GTEST_HAS_NOTIFICATION_ - Enabled if Notification is already provided.
+// GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ - Enabled if Mutex and ThreadLocal are
+// already provided.
+// Must also provide GTEST_DECLARE_STATIC_MUTEX_(mutex) and
+// GTEST_DEFINE_STATIC_MUTEX_(mutex)
+//
+// GTEST_EXCLUSIVE_LOCK_REQUIRED_(locks)
+// GTEST_LOCK_EXCLUDED_(locks)
+//
+// ** Custom implementation starts here **
+
+#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_
+#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_
+
+#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_
+
+#if !defined(GTEST_DEV_EMAIL_)
+# define GTEST_DEV_EMAIL_ "googletestframework@@googlegroups.com"
+# define GTEST_FLAG_PREFIX_ "gtest_"
+# define GTEST_FLAG_PREFIX_DASH_ "gtest-"
+# define GTEST_FLAG_PREFIX_UPPER_ "GTEST_"
+# define GTEST_NAME_ "Google Test"
+# define GTEST_PROJECT_URL_ "https://github.com/google/googletest/"
+#endif // !defined(GTEST_DEV_EMAIL_)
+
+#if !defined(GTEST_INIT_GOOGLE_TEST_NAME_)
+# define GTEST_INIT_GOOGLE_TEST_NAME_ "testing::InitGoogleTest"
+#endif // !defined(GTEST_INIT_GOOGLE_TEST_NAME_)
+
+// Determines the version of gcc that is used to compile this.
+#ifdef __GNUC__
+// 40302 means version 4.3.2.
+# define GTEST_GCC_VER_ \
+ (__GNUC__*10000 + __GNUC_MINOR__*100 + __GNUC_PATCHLEVEL__)
+#endif // __GNUC__
+
+// Macros for disabling Microsoft Visual C++ warnings.
+//
+// GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 4385)
+// /* code that triggers warnings C4800 and C4385 */
+// GTEST_DISABLE_MSC_WARNINGS_POP_()
+#if _MSC_VER >= 1500
+# define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) \
+ __pragma(warning(push)) \
+ __pragma(warning(disable: warnings))
+# define GTEST_DISABLE_MSC_WARNINGS_POP_() \
+ __pragma(warning(pop))
+#else
+// Older versions of MSVC don't have __pragma.
+# define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings)
+# define GTEST_DISABLE_MSC_WARNINGS_POP_()
+#endif
+
#ifndef GTEST_LANG_CXX11
// gcc and clang define __GXX_EXPERIMENTAL_CXX0X__ when
// -std={c,gnu}++{0x,11} is passed. The C++11 standard specifies a
@@ -378,19 +587,82 @@
# endif
#endif
+// Distinct from C++11 language support, some environments don't provide
+// proper C++11 library support. Notably, it's possible to build in
+// C++11 mode when targeting Mac OS X 10.6, which has an old libstdc++
+// with no C++11 support.
+//
+// libstdc++ has sufficient C++11 support as of GCC 4.6.0, __GLIBCXX__
+// 20110325, but maintenance releases in the 4.4 and 4.5 series followed
+// this date, so check for those versions by their date stamps.
+// https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning
+#if GTEST_LANG_CXX11 && \
+ (!defined(__GLIBCXX__) || ( \
+ __GLIBCXX__ >= 20110325ul && /* GCC >= 4.6.0 */ \
+ /* Blacklist of patch releases of older branches: */ \
+ __GLIBCXX__ != 20110416ul && /* GCC 4.4.6 */ \
+ __GLIBCXX__ != 20120313ul && /* GCC 4.4.7 */ \
+ __GLIBCXX__ != 20110428ul && /* GCC 4.5.3 */ \
+ __GLIBCXX__ != 20120702ul)) /* GCC 4.5.4 */
+# define GTEST_STDLIB_CXX11 1
+#endif
+
+// Only use C++11 library features if the library provides them.
+#if GTEST_STDLIB_CXX11
+# define GTEST_HAS_STD_BEGIN_AND_END_ 1
+# define GTEST_HAS_STD_FORWARD_LIST_ 1
+# define GTEST_HAS_STD_FUNCTION_ 1
+# define GTEST_HAS_STD_INITIALIZER_LIST_ 1
+# define GTEST_HAS_STD_MOVE_ 1
+# define GTEST_HAS_STD_SHARED_PTR_ 1
+# define GTEST_HAS_STD_TYPE_TRAITS_ 1
+# define GTEST_HAS_STD_UNIQUE_PTR_ 1
+#endif
+
+// C++11 specifies that <tuple> provides std::tuple.
+// Some platforms still might not have it, however.
+#if GTEST_LANG_CXX11
+# define GTEST_HAS_STD_TUPLE_ 1
+# if defined(__clang__)
+// Inspired by http://clang.llvm.org/docs/LanguageExtensions.html#__has_include
+# if defined(__has_include) && !__has_include(<tuple>)
+# undef GTEST_HAS_STD_TUPLE_
+# endif
+# elif defined(_MSC_VER)
+// Inspired by boost/config/stdlib/dinkumware.hpp
+# if defined(_CPPLIB_VER) && _CPPLIB_VER < 520
+# undef GTEST_HAS_STD_TUPLE_
+# endif
+# elif defined(__GLIBCXX__)
+// Inspired by boost/config/stdlib/libstdcpp3.hpp,
+// http://gcc.gnu.org/gcc-4.2/changes.html and
+// http://gcc.gnu.org/onlinedocs/libstdc++/manual/bk01pt01ch01.html#manual.intro.status.standard.200x
+# if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 2)
+# undef GTEST_HAS_STD_TUPLE_
+# endif
+# endif
+#endif
+
// Brings in definitions for functions used in the testing::internal::posix
// namespace (read, write, close, chdir, isatty, stat). We do not currently
// use them on Windows Mobile.
-#if !GTEST_OS_WINDOWS
+#if GTEST_OS_WINDOWS
+# if !GTEST_OS_WINDOWS_MOBILE
+# include <direct.h>
+# include <io.h>
+# endif
+// In order to avoid having to include <windows.h>, use forward declaration
+// assuming CRITICAL_SECTION is a typedef of _RTL_CRITICAL_SECTION.
+// This assumption is verified by
+// WindowsTypesTest.CRITICAL_SECTIONIs_RTL_CRITICAL_SECTION.
+struct _RTL_CRITICAL_SECTION;
+#else
// This assumes that non-Windows OSes provide unistd.h. For OSes where this
// is not the case, we need to include headers that provide the functions
// mentioned above.
# include <unistd.h>
# include <strings.h>
-#elif !GTEST_OS_WINDOWS_MOBILE
-# include <direct.h>
-# include <io.h>
-#endif
+#endif // GTEST_OS_WINDOWS
#if GTEST_OS_LINUX_ANDROID
// Used to define __ANDROID_API__ matching the target NDK API level.
@@ -407,7 +679,10 @@
# endif
#endif
-#if GTEST_HAS_POSIX_RE
+#if GTEST_USES_PCRE
+// The appropriate headers have already been included.
+
+#elif GTEST_HAS_POSIX_RE
// On some platforms, <regex.h> needs someone to define size_t, and
// won't compile otherwise. We can #include it here as we already
@@ -429,7 +704,7 @@
// simple regex implementation instead.
# define GTEST_USES_SIMPLE_RE 1
-#endif // GTEST_HAS_POSIX_RE
+#endif // GTEST_USES_PCRE
#ifndef GTEST_HAS_EXCEPTIONS
// The user didn't tell us whether exceptions are enabled, so we need
@@ -442,6 +717,15 @@
# define _HAS_EXCEPTIONS 1
# endif // _HAS_EXCEPTIONS
# define GTEST_HAS_EXCEPTIONS _HAS_EXCEPTIONS
+# elif defined(__clang__)
+// clang defines __EXCEPTIONS iff exceptions are enabled before clang 220714,
+// but iff cleanups are enabled after that. In Obj-C++ files, there can be
+// cleanups for ObjC exceptions which also need cleanups, even if C++ exceptions
+// are disabled. clang has __has_feature(cxx_exceptions) which checks for C++
+// exceptions starting at clang r206352, but which checked for cleanups prior to
+// that. To reliably check for C++ exception availability with clang, check for
+// __EXCEPTIONS && __has_feature(cxx_exceptions).
+# define GTEST_HAS_EXCEPTIONS (__EXCEPTIONS && __has_feature(cxx_exceptions))
# elif defined(__GNUC__) && __EXCEPTIONS
// gcc defines __EXCEPTIONS to 1 iff exceptions are enabled.
# define GTEST_HAS_EXCEPTIONS 1
@@ -567,13 +851,13 @@
// Determines whether Google Test can use the pthreads library.
#ifndef GTEST_HAS_PTHREAD
-// The user didn't tell us explicitly, so we assume pthreads support is
-// available on Linux and Mac.
+// The user didn't tell us explicitly, so we make reasonable assumptions about
+// which platforms have pthreads support.
//
// To disable threading support in Google Test, add -DGTEST_HAS_PTHREAD=0
// to your compiler flags.
# define GTEST_HAS_PTHREAD (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_HPUX \
- || GTEST_OS_QNX)
+ || GTEST_OS_QNX || GTEST_OS_FREEBSD || GTEST_OS_NACL)
#endif // GTEST_HAS_PTHREAD
#if GTEST_HAS_PTHREAD
@@ -585,6 +869,15 @@
# include <time.h> // NOLINT
#endif
+// Determines if hash_map/hash_set are available.
+// Only used for testing against those containers.
+#if !defined(GTEST_HAS_HASH_MAP_)
+# if _MSC_VER
+# define GTEST_HAS_HASH_MAP_ 1 // Indicates that hash_map is available.
+# define GTEST_HAS_HASH_SET_ 1 // Indicates that hash_set is available.
+# endif // _MSC_VER
+#endif // !defined(GTEST_HAS_HASH_MAP_)
+
// Determines whether Google Test can use tr1/tuple. You can define
// this macro to 0 to prevent Google Test from using tuple (any
// feature depending on tuple with be disabled in this mode).
@@ -635,8 +928,18 @@
// To avoid conditional compilation everywhere, we make it
// gtest-port.h's responsibility to #include the header implementing
-// tr1/tuple.
+// tuple.
+#if GTEST_HAS_STD_TUPLE_
+# include <tuple> // IWYU pragma: export
+# define GTEST_TUPLE_NAMESPACE_ ::std
+#endif // GTEST_HAS_STD_TUPLE_
+
+// We include tr1::tuple even if std::tuple is available to define printers for
+// them.
#if GTEST_HAS_TR1_TUPLE
+# ifndef GTEST_TUPLE_NAMESPACE_
+# define GTEST_TUPLE_NAMESPACE_ ::std::tr1
+# endif // GTEST_TUPLE_NAMESPACE_
# if GTEST_USE_OWN_TR1_TUPLE
// This file was GENERATED by command:
@@ -694,6 +997,14 @@
private:
#endif
+// Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that conflict
+// with our own definitions. Therefore using our own tuple does not work on
+// those compilers.
+#if defined(_MSC_VER) && _MSC_VER >= 1600 /* 1600 is Visual Studio 2010 */
+# error "gtest's tuple doesn't compile on Visual Studio 2010 or later. \
+GTEST_USE_OWN_TR1_TUPLE must be set to 0 on those compilers."
+#endif
+
// GTEST_n_TUPLE_(T) is the type of an n-tuple.
#define GTEST_0_TUPLE_(T) tuple<>
#define GTEST_1_TUPLE_(T) tuple<T##0, void, void, void, void, void, void, \
@@ -1681,7 +1992,7 @@ using ::std::tuple_size;
// This prevents <boost/tr1/detail/config.hpp>, which defines
// BOOST_HAS_TR1_TUPLE, from being #included by Boost's <tuple>.
# define BOOST_TR1_DETAIL_CONFIG_HPP_INCLUDED
-# include <tuple>
+# include <tuple> // IWYU pragma: export // NOLINT
# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40000)
// GCC 4.0+ implements tr1/tuple in the <tr1/tuple> header. This does
@@ -1704,7 +2015,7 @@ using ::std::tuple_size;
# else
// If the compiler is not GCC 4.0+, we assume the user is using a
// spec-conforming TR1 implementation.
-# include <tuple> // NOLINT
+# include <tuple> // IWYU pragma: export // NOLINT
# endif // GTEST_USE_OWN_TR1_TUPLE
#endif // GTEST_HAS_TR1_TUPLE
@@ -1738,7 +2049,8 @@ using ::std::tuple_size;
#ifndef GTEST_HAS_STREAM_REDIRECTION
// By default, we assume that stream redirection is supported on all
// platforms except known mobile ones.
-# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN
+# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || \
+ GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT
# define GTEST_HAS_STREAM_REDIRECTION 0
# else
# define GTEST_HAS_STREAM_REDIRECTION 1
@@ -1750,12 +2062,11 @@ using ::std::tuple_size;
// abort() in a VC 7.1 application compiled as GUI in debug config
// pops up a dialog window that cannot be suppressed programmatically.
#if (GTEST_OS_LINUX || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \
- (GTEST_OS_MAC && !GTEST_OS_IOS) || GTEST_OS_IOS_SIMULATOR || \
+ (GTEST_OS_MAC && !GTEST_OS_IOS) || \
(GTEST_OS_WINDOWS_DESKTOP && _MSC_VER >= 1400) || \
GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX || \
- GTEST_OS_OPENBSD || GTEST_OS_QNX)
+ GTEST_OS_OPENBSD || GTEST_OS_QNX || GTEST_OS_FREEBSD)
# define GTEST_HAS_DEATH_TEST 1
-# include <vector> // NOLINT
#endif
// We don't support MSVC 7.1 with exceptions disabled now. Therefore
@@ -1819,7 +2130,12 @@ using ::std::tuple_size;
// compiler the variable/parameter does not have to be used.
#if defined(__GNUC__) && !defined(COMPILER_ICC)
# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused))
-#else
+#elif defined(__clang__)
+# if __has_attribute(unused)
+# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused))
+# endif
+#endif
+#ifndef GTEST_ATTRIBUTE_UNUSED_
# define GTEST_ATTRIBUTE_UNUSED_
#endif
@@ -1845,6 +2161,19 @@ using ::std::tuple_size;
# define GTEST_MUST_USE_RESULT_
#endif // __GNUC__ && (GTEST_GCC_VER_ >= 30400) && !COMPILER_ICC
+// MS C++ compiler emits warning when a conditional expression is compile time
+// constant. In some contexts this warning is false positive and needs to be
+// suppressed. Use the following two macros in such cases:
+//
+// GTEST_INTENTIONAL_CONST_COND_PUSH_()
+// while (true) {
+// GTEST_INTENTIONAL_CONST_COND_POP_()
+// }
+# define GTEST_INTENTIONAL_CONST_COND_PUSH_() \
+ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4127)
+# define GTEST_INTENTIONAL_CONST_COND_POP_() \
+ GTEST_DISABLE_MSC_WARNINGS_POP_()
+
// Determine whether the compiler supports Microsoft's Structured Exception
// Handling. This is supported by several Windows compilers but generally
// does not exist on any other system.
@@ -1859,17 +2188,22 @@ using ::std::tuple_size;
# define GTEST_HAS_SEH 0
# endif
+#define GTEST_IS_THREADSAFE \
+ (GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ \
+ || (GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT) \
+ || GTEST_HAS_PTHREAD)
+
#endif // GTEST_HAS_SEH
#ifdef _MSC_VER
-
# if GTEST_LINKED_AS_SHARED_LIBRARY
# define GTEST_API_ __declspec(dllimport)
# elif GTEST_CREATE_SHARED_LIBRARY
# define GTEST_API_ __declspec(dllexport)
# endif
-
-#endif // _MSC_VER
+#elif __GNUC__ >= 4 || defined(__clang__)
+# define GTEST_API_ __attribute__((visibility ("default")))
+#endif // _MSC_VER
#ifndef GTEST_API_
# define GTEST_API_
@@ -1889,10 +2223,58 @@ using ::std::tuple_size;
# define GTEST_HAS_CXXABI_H_ 0
#endif
+// A function level attribute to disable checking for use of uninitialized
+// memory when built with MemorySanitizer.
+#if defined(__clang__)
+# if __has_feature(memory_sanitizer)
+# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ \
+ __attribute__((no_sanitize_memory))
+# else
+# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+# endif // __has_feature(memory_sanitizer)
+#else
+# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+#endif // __clang__
+
+// A function level attribute to disable AddressSanitizer instrumentation.
+#if defined(__clang__)
+# if __has_feature(address_sanitizer)
+# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ \
+ __attribute__((no_sanitize_address))
+# else
+# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+# endif // __has_feature(address_sanitizer)
+#else
+# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+#endif // __clang__
+
+// A function level attribute to disable ThreadSanitizer instrumentation.
+#if defined(__clang__)
+# if __has_feature(thread_sanitizer)
+# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ \
+ __attribute__((no_sanitize_thread))
+# else
+# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
+# endif // __has_feature(thread_sanitizer)
+#else
+# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
+#endif // __clang__
+
namespace testing {
class Message;
+#if defined(GTEST_TUPLE_NAMESPACE_)
+// Import tuple and friends into the ::testing namespace.
+// It is part of our interface, having them in ::testing allows us to change
+// their types as needed.
+using GTEST_TUPLE_NAMESPACE_::get;
+using GTEST_TUPLE_NAMESPACE_::make_tuple;
+using GTEST_TUPLE_NAMESPACE_::tuple;
+using GTEST_TUPLE_NAMESPACE_::tuple_size;
+using GTEST_TUPLE_NAMESPACE_::tuple_element;
+#endif // defined(GTEST_TUPLE_NAMESPACE_)
+
namespace internal {
// A secret type that Google Test users don't know about. It has no
@@ -1904,8 +2286,8 @@ class Secret;
// expression is true. For example, you could use it to verify the
// size of a static array:
//
-// GTEST_COMPILE_ASSERT_(ARRAYSIZE(content_type_names) == CONTENT_NUM_TYPES,
-// content_type_names_incorrect_size);
+// GTEST_COMPILE_ASSERT_(GTEST_ARRAY_SIZE_(names) == NUM_NAMES,
+// names_incorrect_size);
//
// or to make sure a struct is smaller than a certain size:
//
@@ -1915,16 +2297,22 @@ class Secret;
// the expression is false, most compilers will issue a warning/error
// containing the name of the variable.
+#if GTEST_LANG_CXX11
+# define GTEST_COMPILE_ASSERT_(expr, msg) static_assert(expr, #msg)
+#else // !GTEST_LANG_CXX11
template <bool>
-struct CompileAssert {
+ struct CompileAssert {
};
-#define GTEST_COMPILE_ASSERT_(expr, msg) \
+# define GTEST_COMPILE_ASSERT_(expr, msg) \
typedef ::testing::internal::CompileAssert<(static_cast<bool>(expr))> \
msg[static_cast<bool>(expr) ? 1 : -1] GTEST_ATTRIBUTE_UNUSED_
+#endif // !GTEST_LANG_CXX11
// Implementation details of GTEST_COMPILE_ASSERT_:
//
+// (In C++11, we simply use static_assert instead of the following)
+//
// - GTEST_COMPILE_ASSERT_ works by defining an array type that has -1
// elements (and thus is invalid) when the expression is false.
//
@@ -1971,7 +2359,12 @@ template <typename T1, typename T2>
struct StaticAssertTypeEqHelper;
template <typename T>
-struct StaticAssertTypeEqHelper<T, T> {};
+struct StaticAssertTypeEqHelper<T, T> {
+ enum { value = true };
+};
+
+// Evaluates to the number of elements in 'array'.
+#define GTEST_ARRAY_SIZE_(array) (sizeof(array) / sizeof(array[0]))
#if GTEST_HAS_GLOBAL_STRING
typedef ::string string;
@@ -2020,6 +2413,11 @@ class scoped_ptr {
}
}
+ friend void swap(scoped_ptr& a, scoped_ptr& b) {
+ using std::swap;
+ swap(a.ptr_, b.ptr_);
+ }
+
private:
T* ptr_;
@@ -2143,13 +2541,18 @@ class GTEST_API_ GTestLog {
GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestLog);
};
-#define GTEST_LOG_(severity) \
+#if !defined(GTEST_LOG_)
+
+# define GTEST_LOG_(severity) \
::testing::internal::GTestLog(::testing::internal::GTEST_##severity, \
__FILE__, __LINE__).GetStream()
inline void LogToStderr() {}
inline void FlushInfoLog() { fflush(NULL); }
+#endif // !defined(GTEST_LOG_)
+
+#if !defined(GTEST_CHECK_)
// INTERNAL IMPLEMENTATION - DO NOT USE.
//
// GTEST_CHECK_ is an all-mode assert. It aborts the program if the condition
@@ -2164,12 +2567,13 @@ inline void FlushInfoLog() { fflush(NULL); }
// condition itself, plus additional message streamed into it, if any,
// and then it aborts the program. It aborts the program irrespective of
// whether it is built in the debug mode or not.
-#define GTEST_CHECK_(condition) \
+# define GTEST_CHECK_(condition) \
GTEST_AMBIGUOUS_ELSE_BLOCKER_ \
if (::testing::internal::IsTrue(condition)) \
; \
else \
GTEST_LOG_(FATAL) << "Condition " #condition " failed. "
+#endif // !defined(GTEST_CHECK_)
// An all-mode assert to verify that the given POSIX-style function
// call returns 0 (indicating success). Known limitation: this
@@ -2181,6 +2585,15 @@ inline void FlushInfoLog() { fflush(NULL); }
GTEST_LOG_(FATAL) << #posix_call << "failed with error " \
<< gtest_error
+#if GTEST_HAS_STD_MOVE_
+using std::move;
+#else // GTEST_HAS_STD_MOVE_
+template <typename T>
+const T& move(const T& t) {
+ return t;
+}
+#endif // GTEST_HAS_STD_MOVE_
+
// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE.
//
// Use ImplicitCast_ as a safe version of static_cast for upcasting in
@@ -2231,7 +2644,9 @@ inline To DownCast_(From* f) { // so we only accept pointers
// for compile-time type checking, and has no overhead in an
// optimized build at run-time, as it will be optimized away
// completely.
+ GTEST_INTENTIONAL_CONST_COND_PUSH_()
if (false) {
+ GTEST_INTENTIONAL_CONST_COND_POP_()
const To to = NULL;
::testing::internal::ImplicitCast_<From*>(to);
}
@@ -2252,6 +2667,11 @@ template <class Derived, class Base>
Derived* CheckedDowncastToActualType(Base* base) {
#if GTEST_HAS_RTTI
GTEST_CHECK_(typeid(*base) == typeid(Derived));
+#endif
+
+#if GTEST_HAS_DOWNCAST_
+ return ::down_cast<Derived*>(base);
+#elif GTEST_HAS_RTTI
return dynamic_cast<Derived*>(base); // NOLINT
#else
return static_cast<Derived*>(base); // Poor man's downcast.
@@ -2273,6 +2693,17 @@ GTEST_API_ std::string GetCapturedStderr();
#endif // GTEST_HAS_STREAM_REDIRECTION
+// Returns a path to temporary directory.
+GTEST_API_ std::string TempDir();
+
+// Returns the size (in bytes) of a file.
+GTEST_API_ size_t GetFileSize(FILE* file);
+
+// Reads the entire content of a file as a string.
+GTEST_API_ std::string ReadEntireFile(FILE* file);
+
+// All command line arguments.
+GTEST_API_ const ::std::vector<testing::internal::string>& GetArgvs();
#if GTEST_HAS_DEATH_TEST
@@ -2280,18 +2711,15 @@ const ::std::vector<testing::internal::string>& GetInjectableArgvs();
void SetInjectableArgvs(const ::std::vector<testing::internal::string>*
new_argvs);
-// A copy of all command line arguments. Set by InitGoogleTest().
-extern ::std::vector<testing::internal::string> g_argvs;
#endif // GTEST_HAS_DEATH_TEST
// Defines synchronization primitives.
-
-#if GTEST_HAS_PTHREAD
-
-// Sleeps for (roughly) n milli-seconds. This function is only for
-// testing Google Test's own constructs. Don't use it in user tests,
-// either directly or indirectly.
+#if GTEST_IS_THREADSAFE
+# if GTEST_HAS_PTHREAD
+// Sleeps for (roughly) n milliseconds. This function is only for testing
+// Google Test's own constructs. Don't use it in user tests, either
+// directly or indirectly.
inline void SleepMilliseconds(int n) {
const timespec time = {
0, // 0 seconds.
@@ -2299,7 +2727,13 @@ inline void SleepMilliseconds(int n) {
};
nanosleep(&time, NULL);
}
+# endif // GTEST_HAS_PTHREAD
+
+# if GTEST_HAS_NOTIFICATION_
+// Notification has already been imported into the namespace.
+// Nothing to do here.
+# elif GTEST_HAS_PTHREAD
// Allows a controller thread to pause execution of newly created
// threads until notified. Instances of this class must be created
// and destroyed in the controller thread.
@@ -2343,6 +2777,62 @@ class Notification {
GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification);
};
+# elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
+
+GTEST_API_ void SleepMilliseconds(int n);
+
+// Provides leak-safe Windows kernel handle ownership.
+// Used in death tests and in threading support.
+class GTEST_API_ AutoHandle {
+ public:
+ // Assume that Win32 HANDLE type is equivalent to void*. Doing so allows us to
+ // avoid including <windows.h> in this header file. Including <windows.h> is
+ // undesirable because it defines a lot of symbols and macros that tend to
+ // conflict with client code. This assumption is verified by
+ // WindowsTypesTest.HANDLEIsVoidStar.
+ typedef void* Handle;
+ AutoHandle();
+ explicit AutoHandle(Handle handle);
+
+ ~AutoHandle();
+
+ Handle Get() const;
+ void Reset();
+ void Reset(Handle handle);
+
+ private:
+ // Returns true iff the handle is a valid handle object that can be closed.
+ bool IsCloseable() const;
+
+ Handle handle_;
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(AutoHandle);
+};
+
+// Allows a controller thread to pause execution of newly created
+// threads until notified. Instances of this class must be created
+// and destroyed in the controller thread.
+//
+// This class is only for testing Google Test's own constructs. Do not
+// use it in user tests, either directly or indirectly.
+class GTEST_API_ Notification {
+ public:
+ Notification();
+ void Notify();
+ void WaitForNotification();
+
+ private:
+ AutoHandle event_;
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification);
+};
+# endif // GTEST_HAS_NOTIFICATION_
+
+// On MinGW, we can have both GTEST_OS_WINDOWS and GTEST_HAS_PTHREAD
+// defined, but we don't want to use MinGW's pthreads implementation, which
+// has conformance problems with some versions of the POSIX standard.
+# if GTEST_HAS_PTHREAD && !GTEST_OS_WINDOWS_MINGW
+
// As a C-function, ThreadFuncWithCLinkage cannot be templated itself.
// Consequently, it cannot select a correct instantiation of ThreadWithParam
// in order to call its Run(). Introducing ThreadWithParamBase as a
@@ -2380,66 +2870,332 @@ extern "C" inline void* ThreadFuncWithCLinkage(void* thread) {
template <typename T>
class ThreadWithParam : public ThreadWithParamBase {
public:
- typedef void (*UserThreadFunc)(T);
+ typedef void UserThreadFunc(T);
+
+ ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start)
+ : func_(func),
+ param_(param),
+ thread_can_start_(thread_can_start),
+ finished_(false) {
+ ThreadWithParamBase* const base = this;
+ // The thread can be created only after all fields except thread_
+ // have been initialized.
+ GTEST_CHECK_POSIX_SUCCESS_(
+ pthread_create(&thread_, 0, &ThreadFuncWithCLinkage, base));
+ }
+ ~ThreadWithParam() { Join(); }
+
+ void Join() {
+ if (!finished_) {
+ GTEST_CHECK_POSIX_SUCCESS_(pthread_join(thread_, 0));
+ finished_ = true;
+ }
+ }
+
+ virtual void Run() {
+ if (thread_can_start_ != NULL)
+ thread_can_start_->WaitForNotification();
+ func_(param_);
+ }
+
+ private:
+ UserThreadFunc* const func_; // User-supplied thread function.
+ const T param_; // User-supplied parameter to the thread function.
+ // When non-NULL, used to block execution until the controller thread
+ // notifies.
+ Notification* const thread_can_start_;
+ bool finished_; // true iff we know that the thread function has finished.
+ pthread_t thread_; // The native thread object.
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam);
+};
+# endif // !GTEST_OS_WINDOWS && GTEST_HAS_PTHREAD ||
+ // GTEST_HAS_MUTEX_AND_THREAD_LOCAL_
+
+# if GTEST_HAS_MUTEX_AND_THREAD_LOCAL_
+// Mutex and ThreadLocal have already been imported into the namespace.
+// Nothing to do here.
+
+# elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
+
+// Mutex implements mutex on Windows platforms. It is used in conjunction
+// with class MutexLock:
+//
+// Mutex mutex;
+// ...
+// MutexLock lock(&mutex); // Acquires the mutex and releases it at the
+// // end of the current scope.
+//
+// A static Mutex *must* be defined or declared using one of the following
+// macros:
+// GTEST_DEFINE_STATIC_MUTEX_(g_some_mutex);
+// GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex);
+//
+// (A non-static Mutex is defined/declared in the usual way).
+class GTEST_API_ Mutex {
+ public:
+ enum MutexType { kStatic = 0, kDynamic = 1 };
+ // We rely on kStaticMutex being 0 as it is to what the linker initializes
+ // type_ in static mutexes. critical_section_ will be initialized lazily
+ // in ThreadSafeLazyInit().
+ enum StaticConstructorSelector { kStaticMutex = 0 };
+
+ // This constructor intentionally does nothing. It relies on type_ being
+ // statically initialized to 0 (effectively setting it to kStatic) and on
+ // ThreadSafeLazyInit() to lazily initialize the rest of the members.
+ explicit Mutex(StaticConstructorSelector /*dummy*/) {}
+
+ Mutex();
+ ~Mutex();
+
+ void Lock();
+
+ void Unlock();
+
+ // Does nothing if the current thread holds the mutex. Otherwise, crashes
+ // with high probability.
+ void AssertHeld();
+
+ private:
+ // Initializes owner_thread_id_ and critical_section_ in static mutexes.
+ void ThreadSafeLazyInit();
+
+ // Per http://blogs.msdn.com/b/oldnewthing/archive/2004/02/23/78395.aspx,
+ // we assume that 0 is an invalid value for thread IDs.
+ unsigned int owner_thread_id_;
+
+ // For static mutexes, we rely on these members being initialized to zeros
+ // by the linker.
+ MutexType type_;
+ long critical_section_init_phase_; // NOLINT
+ _RTL_CRITICAL_SECTION* critical_section_;
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex);
+};
+
+# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \
+ extern ::testing::internal::Mutex mutex
+
+# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \
+ ::testing::internal::Mutex mutex(::testing::internal::Mutex::kStaticMutex)
+
+// We cannot name this class MutexLock because the ctor declaration would
+// conflict with a macro named MutexLock, which is defined on some
+// platforms. That macro is used as a defensive measure to prevent against
+// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than
+// "MutexLock l(&mu)". Hence the typedef trick below.
+class GTestMutexLock {
+ public:
+ explicit GTestMutexLock(Mutex* mutex)
+ : mutex_(mutex) { mutex_->Lock(); }
+
+ ~GTestMutexLock() { mutex_->Unlock(); }
+
+ private:
+ Mutex* const mutex_;
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock);
+};
+
+typedef GTestMutexLock MutexLock;
+
+// Base class for ValueHolder<T>. Allows a caller to hold and delete a value
+// without knowing its type.
+class ThreadLocalValueHolderBase {
+ public:
+ virtual ~ThreadLocalValueHolderBase() {}
+};
+
+// Provides a way for a thread to send notifications to a ThreadLocal
+// regardless of its parameter type.
+class ThreadLocalBase {
+ public:
+ // Creates a new ValueHolder<T> object holding a default value passed to
+ // this ThreadLocal<T>'s constructor and returns it. It is the caller's
+ // responsibility not to call this when the ThreadLocal<T> instance already
+ // has a value on the current thread.
+ virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const = 0;
+
+ protected:
+ ThreadLocalBase() {}
+ virtual ~ThreadLocalBase() {}
+
+ private:
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocalBase);
+};
+
+// Maps a thread to a set of ThreadLocals that have values instantiated on that
+// thread and notifies them when the thread exits. A ThreadLocal instance is
+// expected to persist until all threads it has values on have terminated.
+class GTEST_API_ ThreadLocalRegistry {
+ public:
+ // Registers thread_local_instance as having value on the current thread.
+ // Returns a value that can be used to identify the thread from other threads.
+ static ThreadLocalValueHolderBase* GetValueOnCurrentThread(
+ const ThreadLocalBase* thread_local_instance);
+
+ // Invoked when a ThreadLocal instance is destroyed.
+ static void OnThreadLocalDestroyed(
+ const ThreadLocalBase* thread_local_instance);
+};
+
+class GTEST_API_ ThreadWithParamBase {
+ public:
+ void Join();
+
+ protected:
+ class Runnable {
+ public:
+ virtual ~Runnable() {}
+ virtual void Run() = 0;
+ };
+
+ ThreadWithParamBase(Runnable *runnable, Notification* thread_can_start);
+ virtual ~ThreadWithParamBase();
+
+ private:
+ AutoHandle thread_;
+};
+
+// Helper class for testing Google Test's multi-threading constructs.
+template <typename T>
+class ThreadWithParam : public ThreadWithParamBase {
+ public:
+ typedef void UserThreadFunc(T);
+
+ ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start)
+ : ThreadWithParamBase(new RunnableImpl(func, param), thread_can_start) {
+ }
+ virtual ~ThreadWithParam() {}
+
+ private:
+ class RunnableImpl : public Runnable {
+ public:
+ RunnableImpl(UserThreadFunc* func, T param)
+ : func_(func),
+ param_(param) {
+ }
+ virtual ~RunnableImpl() {}
+ virtual void Run() {
+ func_(param_);
+ }
+
+ private:
+ UserThreadFunc* const func_;
+ const T param_;
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(RunnableImpl);
+ };
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam);
+};
+
+// Implements thread-local storage on Windows systems.
+//
+// // Thread 1
+// ThreadLocal<int> tl(100); // 100 is the default value for each thread.
+//
+// // Thread 2
+// tl.set(150); // Changes the value for thread 2 only.
+// EXPECT_EQ(150, tl.get());
+//
+// // Thread 1
+// EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value.
+// tl.set(200);
+// EXPECT_EQ(200, tl.get());
+//
+// The template type argument T must have a public copy constructor.
+// In addition, the default ThreadLocal constructor requires T to have
+// a public default constructor.
+//
+// The users of a TheadLocal instance have to make sure that all but one
+// threads (including the main one) using that instance have exited before
+// destroying it. Otherwise, the per-thread objects managed for them by the
+// ThreadLocal instance are not guaranteed to be destroyed on all platforms.
+//
+// Google Test only uses global ThreadLocal objects. That means they
+// will die after main() has returned. Therefore, no per-thread
+// object managed by Google Test will be leaked as long as all threads
+// using Google Test have exited when main() returns.
+template <typename T>
+class ThreadLocal : public ThreadLocalBase {
+ public:
+ ThreadLocal() : default_factory_(new DefaultValueHolderFactory()) {}
+ explicit ThreadLocal(const T& value)
+ : default_factory_(new InstanceValueHolderFactory(value)) {}
+
+ ~ThreadLocal() { ThreadLocalRegistry::OnThreadLocalDestroyed(this); }
+
+ T* pointer() { return GetOrCreateValue(); }
+ const T* pointer() const { return GetOrCreateValue(); }
+ const T& get() const { return *pointer(); }
+ void set(const T& value) { *pointer() = value; }
+
+ private:
+ // Holds a value of T. Can be deleted via its base class without the caller
+ // knowing the type of T.
+ class ValueHolder : public ThreadLocalValueHolderBase {
+ public:
+ ValueHolder() : value_() {}
+ explicit ValueHolder(const T& value) : value_(value) {}
+
+ T* pointer() { return &value_; }
+
+ private:
+ T value_;
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder);
+ };
- ThreadWithParam(
- UserThreadFunc func, T param, Notification* thread_can_start)
- : func_(func),
- param_(param),
- thread_can_start_(thread_can_start),
- finished_(false) {
- ThreadWithParamBase* const base = this;
- // The thread can be created only after all fields except thread_
- // have been initialized.
- GTEST_CHECK_POSIX_SUCCESS_(
- pthread_create(&thread_, 0, &ThreadFuncWithCLinkage, base));
- }
- ~ThreadWithParam() { Join(); }
- void Join() {
- if (!finished_) {
- GTEST_CHECK_POSIX_SUCCESS_(pthread_join(thread_, 0));
- finished_ = true;
- }
+ T* GetOrCreateValue() const {
+ return static_cast<ValueHolder*>(
+ ThreadLocalRegistry::GetValueOnCurrentThread(this))->pointer();
}
- virtual void Run() {
- if (thread_can_start_ != NULL)
- thread_can_start_->WaitForNotification();
- func_(param_);
+ virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const {
+ return default_factory_->MakeNewHolder();
}
- private:
- const UserThreadFunc func_; // User-supplied thread function.
- const T param_; // User-supplied parameter to the thread function.
- // When non-NULL, used to block execution until the controller thread
- // notifies.
- Notification* const thread_can_start_;
- bool finished_; // true iff we know that the thread function has finished.
- pthread_t thread_; // The native thread object.
+ class ValueHolderFactory {
+ public:
+ ValueHolderFactory() {}
+ virtual ~ValueHolderFactory() {}
+ virtual ValueHolder* MakeNewHolder() const = 0;
- GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam);
+ private:
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolderFactory);
+ };
+
+ class DefaultValueHolderFactory : public ValueHolderFactory {
+ public:
+ DefaultValueHolderFactory() {}
+ virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(); }
+
+ private:
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultValueHolderFactory);
+ };
+
+ class InstanceValueHolderFactory : public ValueHolderFactory {
+ public:
+ explicit InstanceValueHolderFactory(const T& value) : value_(value) {}
+ virtual ValueHolder* MakeNewHolder() const {
+ return new ValueHolder(value_);
+ }
+
+ private:
+ const T value_; // The value for each thread.
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(InstanceValueHolderFactory);
+ };
+
+ scoped_ptr<ValueHolderFactory> default_factory_;
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal);
};
-// MutexBase and Mutex implement mutex on pthreads-based platforms. They
-// are used in conjunction with class MutexLock:
-//
-// Mutex mutex;
-// ...
-// MutexLock lock(&mutex); // Acquires the mutex and releases it at the end
-// // of the current scope.
-//
-// MutexBase implements behavior for both statically and dynamically
-// allocated mutexes. Do not use MutexBase directly. Instead, write
-// the following to define a static mutex:
-//
-// GTEST_DEFINE_STATIC_MUTEX_(g_some_mutex);
-//
-// You can forward declare a static mutex like this:
-//
-// GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex);
-//
-// To create a dynamic mutex, just define an object of type Mutex.
+# elif GTEST_HAS_PTHREAD
+
+// MutexBase and Mutex implement mutex on pthreads-based platforms.
class MutexBase {
public:
// Acquires this mutex.
@@ -2484,17 +3240,12 @@ class MutexBase {
};
// Forward-declares a static mutex.
-# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \
- extern ::testing::internal::MutexBase mutex
+# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \
+ extern ::testing::internal::MutexBase mutex
// Defines and statically (i.e. at link time) initializes a static mutex.
-// The initialization list here does not explicitly initialize each field,
-// instead relying on default initialization for the unspecified fields. In
-// particular, the owner_ field (a pthread_t) is not explicitly initialized.
-// This allows initialization to work whether pthread_t is a scalar or struct.
-// The flag -Wmissing-field-initializers must not be specified for this to work.
-# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \
- ::testing::internal::MutexBase mutex = { PTHREAD_MUTEX_INITIALIZER, false }
+# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \
+ ::testing::internal::MutexBase mutex = { PTHREAD_MUTEX_INITIALIZER, false, pthread_t() }
// The Mutex class can only be used for mutexes created at runtime. It
// shares its API with MutexBase otherwise.
@@ -2512,9 +3263,11 @@ class Mutex : public MutexBase {
GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex);
};
-// We cannot name this class MutexLock as the ctor declaration would
+// We cannot name this class MutexLock because the ctor declaration would
// conflict with a macro named MutexLock, which is defined on some
-// platforms. Hence the typedef trick below.
+// platforms. That macro is used as a defensive measure to prevent against
+// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than
+// "MutexLock l(&mu)". Hence the typedef trick below.
class GTestMutexLock {
public:
explicit GTestMutexLock(MutexBase* mutex)
@@ -2548,41 +3301,14 @@ extern "C" inline void DeleteThreadLocalValue(void* value_holder) {
}
// Implements thread-local storage on pthreads-based systems.
-//
-// // Thread 1
-// ThreadLocal<int> tl(100); // 100 is the default value for each thread.
-//
-// // Thread 2
-// tl.set(150); // Changes the value for thread 2 only.
-// EXPECT_EQ(150, tl.get());
-//
-// // Thread 1
-// EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value.
-// tl.set(200);
-// EXPECT_EQ(200, tl.get());
-//
-// The template type argument T must have a public copy constructor.
-// In addition, the default ThreadLocal constructor requires T to have
-// a public default constructor.
-//
-// An object managed for a thread by a ThreadLocal instance is deleted
-// when the thread exits. Or, if the ThreadLocal instance dies in
-// that thread, when the ThreadLocal dies. It's the user's
-// responsibility to ensure that all other threads using a ThreadLocal
-// have exited when it dies, or the per-thread objects for those
-// threads will not be deleted.
-//
-// Google Test only uses global ThreadLocal objects. That means they
-// will die after main() has returned. Therefore, no per-thread
-// object managed by Google Test will be leaked as long as all threads
-// using Google Test have exited when main() returns.
template <typename T>
class ThreadLocal {
public:
- ThreadLocal() : key_(CreateKey()),
- default_() {}
- explicit ThreadLocal(const T& value) : key_(CreateKey()),
- default_(value) {}
+ ThreadLocal()
+ : key_(CreateKey()), default_factory_(new DefaultValueHolderFactory()) {}
+ explicit ThreadLocal(const T& value)
+ : key_(CreateKey()),
+ default_factory_(new InstanceValueHolderFactory(value)) {}
~ThreadLocal() {
// Destroys the managed object for the current thread, if any.
@@ -2602,6 +3328,7 @@ class ThreadLocal {
// Holds a value of type T.
class ValueHolder : public ThreadLocalValueHolderBase {
public:
+ ValueHolder() : value_() {}
explicit ValueHolder(const T& value) : value_(value) {}
T* pointer() { return &value_; }
@@ -2627,22 +3354,54 @@ class ThreadLocal {
return CheckedDowncastToActualType<ValueHolder>(holder)->pointer();
}
- ValueHolder* const new_holder = new ValueHolder(default_);
+ ValueHolder* const new_holder = default_factory_->MakeNewHolder();
ThreadLocalValueHolderBase* const holder_base = new_holder;
GTEST_CHECK_POSIX_SUCCESS_(pthread_setspecific(key_, holder_base));
return new_holder->pointer();
}
+ class ValueHolderFactory {
+ public:
+ ValueHolderFactory() {}
+ virtual ~ValueHolderFactory() {}
+ virtual ValueHolder* MakeNewHolder() const = 0;
+
+ private:
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolderFactory);
+ };
+
+ class DefaultValueHolderFactory : public ValueHolderFactory {
+ public:
+ DefaultValueHolderFactory() {}
+ virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(); }
+
+ private:
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultValueHolderFactory);
+ };
+
+ class InstanceValueHolderFactory : public ValueHolderFactory {
+ public:
+ explicit InstanceValueHolderFactory(const T& value) : value_(value) {}
+ virtual ValueHolder* MakeNewHolder() const {
+ return new ValueHolder(value_);
+ }
+
+ private:
+ const T value_; // The value for each thread.
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(InstanceValueHolderFactory);
+ };
+
// A key pthreads uses for looking up per-thread values.
const pthread_key_t key_;
- const T default_; // The default value for each thread.
+ scoped_ptr<ValueHolderFactory> default_factory_;
GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal);
};
-# define GTEST_IS_THREADSAFE 1
+# endif // GTEST_HAS_MUTEX_AND_THREAD_LOCAL_
-#else // GTEST_HAS_PTHREAD
+#else // GTEST_IS_THREADSAFE
// A dummy implementation of synchronization primitives (mutex, lock,
// and thread-local variable). Necessary for compiling Google Test where
@@ -2662,6 +3421,11 @@ class Mutex {
# define GTEST_DEFINE_STATIC_MUTEX_(mutex) ::testing::internal::Mutex mutex
+// We cannot name this class MutexLock because the ctor declaration would
+// conflict with a macro named MutexLock, which is defined on some
+// platforms. That macro is used as a defensive measure to prevent against
+// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than
+// "MutexLock l(&mu)". Hence the typedef trick below.
class GTestMutexLock {
public:
explicit GTestMutexLock(Mutex*) {} // NOLINT
@@ -2682,11 +3446,7 @@ class ThreadLocal {
T value_;
};
-// The above synchronization primitives have dummy implementations.
-// Therefore Google Test is not thread-safe.
-# define GTEST_IS_THREADSAFE 0
-
-#endif // GTEST_HAS_PTHREAD
+#endif // GTEST_IS_THREADSAFE
// Returns the number of threads running in the process, or 0 to indicate that
// we cannot detect it.
@@ -2796,6 +3556,13 @@ inline char ToUpper(char ch) {
return static_cast<char>(toupper(static_cast<unsigned char>(ch)));
}
+inline std::string StripTrailingSpaces(std::string str) {
+ std::string::iterator it = str.end();
+ while (it != str.begin() && IsSpace(*--it))
+ it = str.erase(it);
+ return str;
+}
+
// The testing::internal::posix namespace holds wrappers for common
// POSIX functions. These wrappers hide the differences between
// Windows/MSVC and POSIX systems. Since some compilers define these
@@ -2859,11 +3626,7 @@ inline bool IsDir(const StatStruct& st) { return S_ISDIR(st.st_mode); }
// Functions deprecated by MSVC 8.0.
-#ifdef _MSC_VER
-// Temporarily disable warning 4996 (deprecated function).
-# pragma warning(push)
-# pragma warning(disable:4996)
-#endif
+GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996 /* deprecated function */)
inline const char* StrNCpy(char* dest, const char* src, size_t n) {
return strncpy(dest, src, n);
@@ -2873,7 +3636,7 @@ inline const char* StrNCpy(char* dest, const char* src, size_t n) {
// StrError() aren't needed on Windows CE at this time and thus not
// defined there.
-#if !GTEST_OS_WINDOWS_MOBILE
+#if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
inline int ChDir(const char* dir) { return chdir(dir); }
#endif
inline FILE* FOpen(const char* path, const char* mode) {
@@ -2897,8 +3660,9 @@ inline int Close(int fd) { return close(fd); }
inline const char* StrError(int errnum) { return strerror(errnum); }
#endif
inline const char* GetEnv(const char* name) {
-#if GTEST_OS_WINDOWS_MOBILE
+#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE | GTEST_OS_WINDOWS_RT
// We are on Windows CE, which has no environment variables.
+ static_cast<void>(name); // To prevent 'unused argument' warning.
return NULL;
#elif defined(__BORLANDC__) || defined(__SunOS_5_8) || defined(__SunOS_5_9)
// Environment variables which we programmatically clear will be set to the
@@ -2910,9 +3674,7 @@ inline const char* GetEnv(const char* name) {
#endif
}
-#ifdef _MSC_VER
-# pragma warning(pop) // Restores the warning state.
-#endif
+GTEST_DISABLE_MSC_WARNINGS_POP_()
#if GTEST_OS_WINDOWS_MOBILE
// Windows CE has no C library. The abort() function is used in
@@ -3013,11 +3775,20 @@ typedef TypeWithSize<8>::Int TimeInMillis; // Represents time in milliseconds.
// Utilities for command line flags and environment variables.
// Macro for referencing flags.
-#define GTEST_FLAG(name) FLAGS_gtest_##name
+#if !defined(GTEST_FLAG)
+# define GTEST_FLAG(name) FLAGS_gtest_##name
+#endif // !defined(GTEST_FLAG)
+
+#if !defined(GTEST_USE_OWN_FLAGFILE_FLAG_)
+# define GTEST_USE_OWN_FLAGFILE_FLAG_ 1
+#endif // !defined(GTEST_USE_OWN_FLAGFILE_FLAG_)
+
+#if !defined(GTEST_DECLARE_bool_)
+# define GTEST_FLAG_SAVER_ ::testing::internal::GTestFlagSaver
// Macros for declaring flags.
-#define GTEST_DECLARE_bool_(name) GTEST_API_ extern bool GTEST_FLAG(name)
-#define GTEST_DECLARE_int32_(name) \
+# define GTEST_DECLARE_bool_(name) GTEST_API_ extern bool GTEST_FLAG(name)
+# define GTEST_DECLARE_int32_(name) \
GTEST_API_ extern ::testing::internal::Int32 GTEST_FLAG(name)
#define GTEST_DECLARE_string_(name) \
GTEST_API_ extern ::std::string GTEST_FLAG(name)
@@ -3030,9 +3801,13 @@ typedef TypeWithSize<8>::Int TimeInMillis; // Represents time in milliseconds.
#define GTEST_DEFINE_string_(name, default_val, doc) \
GTEST_API_ ::std::string GTEST_FLAG(name) = (default_val)
+#endif // !defined(GTEST_DECLARE_bool_)
+
// Thread annotations
-#define GTEST_EXCLUSIVE_LOCK_REQUIRED_(locks)
-#define GTEST_LOCK_EXCLUDED_(locks)
+#if !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_)
+# define GTEST_EXCLUSIVE_LOCK_REQUIRED_(locks)
+# define GTEST_LOCK_EXCLUDED_(locks)
+#endif // !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_)
// Parses 'str' for a 32-bit signed integer. If successful, writes the result
// to *value and returns true; otherwise leaves *value unchanged and returns
@@ -3046,7 +3821,7 @@ bool ParseInt32(const Message& src_text, const char* str, Int32* value);
// corresponding to the given Google Test flag.
bool BoolFromGTestEnv(const char* flag, bool default_val);
GTEST_API_ Int32 Int32FromGTestEnv(const char* flag, Int32 default_val);
-const char* StringFromGTestEnv(const char* flag, const char* default_val);
+std::string StringFromGTestEnv(const char* flag, const char* default_val);
} // namespace internal
} // namespace testing
@@ -3069,7 +3844,10 @@ const char* StringFromGTestEnv(const char* flag, const char* default_val);
#include <string.h>
#include <iomanip>
#include <limits>
+#include <map>
#include <set>
+#include <string>
+#include <vector>
// Copyright 2005, Google Inc.
// All rights reserved.
@@ -7057,9 +7835,6 @@ class ScopedTrace; // Implements scoped trace.
class TestInfoImpl; // Opaque implementation of TestInfo
class UnitTestImpl; // Opaque implementation of UnitTest
-// How many times InitGoogleTest() has been called.
-GTEST_API_ extern int g_init_gtest_count;
-
// The text used in failure messages to indicate the start of the
// stack trace.
GTEST_API_ extern const char kStackTraceMarker[];
@@ -7131,6 +7906,36 @@ class GTEST_API_ ScopedTrace {
// c'tor and d'tor. Therefore it doesn't
// need to be used otherwise.
+namespace edit_distance {
+// Returns the optimal edits to go from 'left' to 'right'.
+// All edits cost the same, with replace having lower priority than
+// add/remove.
+// Simple implementation of the Wagner\u2013Fischer algorithm.
+// See http://en.wikipedia.org/wiki/Wagner-Fischer_algorithm
+enum EditType { kMatch, kAdd, kRemove, kReplace };
+GTEST_API_ std::vector<EditType> CalculateOptimalEdits(
+ const std::vector<size_t>& left, const std::vector<size_t>& right);
+
+// Same as above, but the input is represented as strings.
+GTEST_API_ std::vector<EditType> CalculateOptimalEdits(
+ const std::vector<std::string>& left,
+ const std::vector<std::string>& right);
+
+// Create a diff of the input strings in Unified diff format.
+GTEST_API_ std::string CreateUnifiedDiff(const std::vector<std::string>& left,
+ const std::vector<std::string>& right,
+ size_t context = 2);
+
+} // namespace edit_distance
+
+// Calculate the diff between 'left' and 'right' and return it in unified diff
+// format.
+// If not null, stores in 'total_line_count' the total number of lines found
+// in left + right.
+GTEST_API_ std::string DiffStrings(const std::string& left,
+ const std::string& right,
+ size_t* total_line_count);
+
// Constructs and returns the message for an equality assertion
// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure.
//
@@ -7431,6 +8236,13 @@ GTEST_API_ AssertionResult IsHRESULTFailure(const char* expr,
typedef void (*SetUpTestCaseFunc)();
typedef void (*TearDownTestCaseFunc)();
+struct CodeLocation {
+ CodeLocation(const string& a_file, int a_line) : file(a_file), line(a_line) {}
+
+ string file;
+ int line;
+};
+
// Creates a new TestInfo object and registers it with Google Test;
// returns the created object.
//
@@ -7442,6 +8254,7 @@ typedef void (*TearDownTestCaseFunc)();
// this is not a typed or a type-parameterized test.
// value_param text representation of the test's value parameter,
// or NULL if this is not a type-parameterized test.
+// code_location: code location where the test is defined
// fixture_class_id: ID of the test fixture class
// set_up_tc: pointer to the function that sets up the test case
// tear_down_tc: pointer to the function that tears down the test case
@@ -7453,6 +8266,7 @@ GTEST_API_ TestInfo* MakeAndRegisterTestInfo(
const char* name,
const char* type_param,
const char* value_param,
+ CodeLocation code_location,
TypeId fixture_class_id,
SetUpTestCaseFunc set_up_tc,
TearDownTestCaseFunc tear_down_tc,
@@ -7482,10 +8296,21 @@ class GTEST_API_ TypedTestCasePState {
fflush(stderr);
posix::Abort();
}
- defined_test_names_.insert(test_name);
+ registered_tests_.insert(
+ ::std::make_pair(test_name, CodeLocation(file, line)));
return true;
}
+ bool TestExists(const std::string& test_name) const {
+ return registered_tests_.count(test_name) > 0;
+ }
+
+ const CodeLocation& GetCodeLocation(const std::string& test_name) const {
+ RegisteredTestsMap::const_iterator it = registered_tests_.find(test_name);
+ GTEST_CHECK_(it != registered_tests_.end());
+ return it->second;
+ }
+
// Verifies that registered_tests match the test names in
// defined_test_names_; returns registered_tests if successful, or
// aborts the program otherwise.
@@ -7493,8 +8318,10 @@ class GTEST_API_ TypedTestCasePState {
const char* file, int line, const char* registered_tests);
private:
+ typedef ::std::map<std::string, CodeLocation> RegisteredTestsMap;
+
bool registered_;
- ::std::set<const char*> defined_test_names_;
+ RegisteredTestsMap registered_tests_;
};
// Skips to the first non-space char after the first comma in 'str';
@@ -7515,6 +8342,11 @@ inline std::string GetPrefixUntilComma(const char* str) {
return comma == NULL ? str : std::string(str, comma);
}
+// Splits a given string on a given delimiter, populating a given
+// vector with the fields.
+void SplitString(const ::std::string& str, char delimiter,
+ ::std::vector< ::std::string>* dest);
+
// TypeParameterizedTest<Fixture, TestSel, Types>::Register()
// registers a list of type-parameterized tests with Google Test. The
// return value is insignificant - we just need to return something
@@ -7529,8 +8361,10 @@ class TypeParameterizedTest {
// specified in INSTANTIATE_TYPED_TEST_CASE_P(Prefix, TestCase,
// Types). Valid values for 'index' are [0, N - 1] where N is the
// length of Types.
- static bool Register(const char* prefix, const char* case_name,
- const char* test_names, int index) {
+ static bool Register(const char* prefix,
+ CodeLocation code_location,
+ const char* case_name, const char* test_names,
+ int index) {
typedef typename Types::Head Type;
typedef Fixture<Type> FixtureClass;
typedef typename GTEST_BIND_(TestSel, Type) TestClass;
@@ -7540,9 +8374,10 @@ class TypeParameterizedTest {
MakeAndRegisterTestInfo(
(std::string(prefix) + (prefix[0] == '\0' ? "" : "/") + case_name + "/"
+ StreamableToString(index)).c_str(),
- GetPrefixUntilComma(test_names).c_str(),
+ StripTrailingSpaces(GetPrefixUntilComma(test_names)).c_str(),
GetTypeName<Type>().c_str(),
NULL, // No value parameter.
+ code_location,
GetTypeId<FixtureClass>(),
TestClass::SetUpTestCase,
TestClass::TearDownTestCase,
@@ -7550,7 +8385,7 @@ class TypeParameterizedTest {
// Next, recurses (at compile time) with the tail of the type list.
return TypeParameterizedTest<Fixture, TestSel, typename Types::Tail>
- ::Register(prefix, case_name, test_names, index + 1);
+ ::Register(prefix, code_location, case_name, test_names, index + 1);
}
};
@@ -7558,8 +8393,9 @@ class TypeParameterizedTest {
template <GTEST_TEMPLATE_ Fixture, class TestSel>
class TypeParameterizedTest<Fixture, TestSel, Types0> {
public:
- static bool Register(const char* /*prefix*/, const char* /*case_name*/,
- const char* /*test_names*/, int /*index*/) {
+ static bool Register(const char* /*prefix*/, CodeLocation,
+ const char* /*case_name*/, const char* /*test_names*/,
+ int /*index*/) {
return true;
}
};
@@ -7571,17 +8407,31 @@ class TypeParameterizedTest<Fixture, TestSel, Types0> {
template <GTEST_TEMPLATE_ Fixture, typename Tests, typename Types>
class TypeParameterizedTestCase {
public:
- static bool Register(const char* prefix, const char* case_name,
- const char* test_names) {
+ static bool Register(const char* prefix, CodeLocation code_location,
+ const TypedTestCasePState* state,
+ const char* case_name, const char* test_names) {
+ std::string test_name = StripTrailingSpaces(
+ GetPrefixUntilComma(test_names));
+ if (!state->TestExists(test_name)) {
+ fprintf(stderr, "Failed to get code location for test %s.%s at %s.",
+ case_name, test_name.c_str(),
+ FormatFileLocation(code_location.file.c_str(),
+ code_location.line).c_str());
+ fflush(stderr);
+ posix::Abort();
+ }
+ const CodeLocation& test_location = state->GetCodeLocation(test_name);
+
typedef typename Tests::Head Head;
// First, register the first test in 'Test' for each type in 'Types'.
TypeParameterizedTest<Fixture, Head, Types>::Register(
- prefix, case_name, test_names, 0);
+ prefix, test_location, case_name, test_names, 0);
// Next, recurses (at compile time) with the tail of the test list.
return TypeParameterizedTestCase<Fixture, typename Tests::Tail, Types>
- ::Register(prefix, case_name, SkipComma(test_names));
+ ::Register(prefix, code_location, state,
+ case_name, SkipComma(test_names));
}
};
@@ -7589,8 +8439,9 @@ class TypeParameterizedTestCase {
template <GTEST_TEMPLATE_ Fixture, typename Types>
class TypeParameterizedTestCase<Fixture, Templates0, Types> {
public:
- static bool Register(const char* /*prefix*/, const char* /*case_name*/,
- const char* /*test_names*/) {
+ static bool Register(const char* /*prefix*/, CodeLocation,
+ const TypedTestCasePState* /*state*/,
+ const char* /*case_name*/, const char* /*test_names*/) {
return true;
}
};
@@ -7744,7 +8595,7 @@ class ImplicitlyConvertible {
// MakeFrom() is an expression whose type is From. We cannot simply
// use From(), as the type From may not have a public default
// constructor.
- static From MakeFrom();
+ static typename AddReference<From>::type MakeFrom();
// These two functions are overloaded. Given an expression
// Helper(x), the compiler will pick the first version if x can be
@@ -7762,25 +8613,20 @@ class ImplicitlyConvertible {
// We have to put the 'public' section after the 'private' section,
// or MSVC refuses to compile the code.
public:
- // MSVC warns about implicitly converting from double to int for
- // possible loss of data, so we need to temporarily disable the
- // warning.
-#ifdef _MSC_VER
-# pragma warning(push) // Saves the current warning state.
-# pragma warning(disable:4244) // Temporarily disables warning 4244.
-
- static const bool value =
- sizeof(Helper(ImplicitlyConvertible::MakeFrom())) == 1;
-# pragma warning(pop) // Restores the warning state.
-#elif defined(__BORLANDC__)
+#if defined(__BORLANDC__)
// C++Builder cannot use member overload resolution during template
// instantiation. The simplest workaround is to use its C++0x type traits
// functions (C++Builder 2009 and above only).
static const bool value = __is_convertible(From, To);
#else
+ // MSVC warns about implicitly converting from double to int for
+ // possible loss of data, so we need to temporarily disable the
+ // warning.
+ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4244)
static const bool value =
sizeof(Helper(ImplicitlyConvertible::MakeFrom())) == 1;
-#endif // _MSV_VER
+ GTEST_DISABLE_MSC_WARNINGS_POP_()
+#endif // __BORLANDC__
};
template <typename From, typename To>
const bool ImplicitlyConvertible<From, To>::value;
@@ -7906,11 +8752,10 @@ void CopyArray(const T* from, size_t size, U* to) {
// The relation between an NativeArray object (see below) and the
// native array it represents.
-enum RelationToSource {
- kReference, // The NativeArray references the native array.
- kCopy // The NativeArray makes a copy of the native array and
- // owns the copy.
-};
+// We use 2 different structs to allow non-copyable types to be used, as long
+// as RelationToSourceReference() is passed.
+struct RelationToSourceReference {};
+struct RelationToSourceCopy {};
// Adapts a native array to a read-only STL-style container. Instead
// of the complete STL container concept, this adaptor only implements
@@ -7928,22 +8773,23 @@ class NativeArray {
typedef Element* iterator;
typedef const Element* const_iterator;
- // Constructs from a native array.
- NativeArray(const Element* array, size_t count, RelationToSource relation) {
- Init(array, count, relation);
+ // Constructs from a native array. References the source.
+ NativeArray(const Element* array, size_t count, RelationToSourceReference) {
+ InitRef(array, count);
+ }
+
+ // Constructs from a native array. Copies the source.
+ NativeArray(const Element* array, size_t count, RelationToSourceCopy) {
+ InitCopy(array, count);
}
// Copy constructor.
NativeArray(const NativeArray& rhs) {
- Init(rhs.array_, rhs.size_, rhs.relation_to_source_);
+ (this->*rhs.clone_)(rhs.array_, rhs.size_);
}
~NativeArray() {
- // Ensures that the user doesn't instantiate NativeArray with a
- // const or reference type.
- static_cast<void>(StaticAssertTypeEqHelper<Element,
- GTEST_REMOVE_REFERENCE_AND_CONST_(Element)>());
- if (relation_to_source_ == kCopy)
+ if (clone_ != &NativeArray::InitRef)
delete[] array_;
}
@@ -7957,23 +8803,30 @@ class NativeArray {
}
private:
- // Initializes this object; makes a copy of the input array if
- // 'relation' is kCopy.
- void Init(const Element* array, size_t a_size, RelationToSource relation) {
- if (relation == kReference) {
- array_ = array;
- } else {
- Element* const copy = new Element[a_size];
- CopyArray(array, a_size, copy);
- array_ = copy;
- }
+ enum {
+ kCheckTypeIsNotConstOrAReference = StaticAssertTypeEqHelper<
+ Element, GTEST_REMOVE_REFERENCE_AND_CONST_(Element)>::value,
+ };
+
+ // Initializes this object with a copy of the input.
+ void InitCopy(const Element* array, size_t a_size) {
+ Element* const copy = new Element[a_size];
+ CopyArray(array, a_size, copy);
+ array_ = copy;
+ size_ = a_size;
+ clone_ = &NativeArray::InitCopy;
+ }
+
+ // Initializes this object with a reference of the input.
+ void InitRef(const Element* array, size_t a_size) {
+ array_ = array;
size_ = a_size;
- relation_to_source_ = relation;
+ clone_ = &NativeArray::InitRef;
}
const Element* array_;
size_t size_;
- RelationToSource relation_to_source_;
+ void (NativeArray::*clone_)(const Element*, size_t);
GTEST_DISALLOW_ASSIGN_(NativeArray);
};
@@ -8108,6 +8961,7 @@ class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) : public parent_class {\
::test_info_ =\
::testing::internal::MakeAndRegisterTestInfo(\
#test_case_name, #test_name, NULL, NULL, \
+ ::testing::internal::CodeLocation(__FILE__, __LINE__), \
(parent_id), \
parent_class::SetUpTestCase, \
parent_class::TearDownTestCase, \
@@ -8116,6 +8970,7 @@ class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) : public parent_class {\
void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody()
#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_
+
// Copyright 2005, Google Inc.
// All rights reserved.
//
@@ -8952,7 +9807,10 @@ TEST_P(DerivedTest, DoesBlah) {
#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_
#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_
+#include <ctype.h>
+
#include <iterator>
+#include <set>
#include <utility>
#include <vector>
@@ -9070,7 +9928,12 @@ class linked_ptr_internal {
MutexLock lock(&g_linked_ptr_mutex);
linked_ptr_internal const* p = ptr;
- while (p->next_ != ptr) p = p->next_;
+ while (p->next_ != ptr) {
+ assert(p->next_ != this &&
+ "Trying to join() a linked ring we are already in. "
+ "Is GMock thread safety enabled?");
+ p = p->next_;
+ }
p->next_ = this;
next_ = ptr;
}
@@ -9083,7 +9946,12 @@ class linked_ptr_internal {
if (next_ == this) return true;
linked_ptr_internal const* p = next_;
- while (p->next_ != this) p = p->next_;
+ while (p->next_ != this) {
+ assert(p->next_ != next_ &&
+ "Trying to depart() a linked ring we are not in. "
+ "Is GMock thread safety enabled?");
+ p = p->next_;
+ }
p->next_ = next_;
return false;
}
@@ -9294,6 +10162,10 @@ linked_ptr<T> make_linked_ptr(T* ptr) {
#include <utility>
#include <vector>
+#if GTEST_HAS_STD_TUPLE_
+# include <tuple>
+#endif
+
namespace testing {
// Definitions in the 'internal' and 'internal2' name spaces are
@@ -9436,11 +10308,108 @@ void DefaultPrintNonContainerTo(const T& value, ::std::ostream* os) {
*os << value;
}
-} // namespace testing_internal
-
-namespace testing {
-namespace internal {
-
+} // namespace testing_internal
+
+namespace testing {
+namespace internal {
+
+// FormatForComparison<ToPrint, OtherOperand>::Format(value) formats a
+// value of type ToPrint that is an operand of a comparison assertion
+// (e.g. ASSERT_EQ). OtherOperand is the type of the other operand in
+// the comparison, and is used to help determine the best way to
+// format the value. In particular, when the value is a C string
+// (char pointer) and the other operand is an STL string object, we
+// want to format the C string as a string, since we know it is
+// compared by value with the string object. If the value is a char
+// pointer but the other operand is not an STL string object, we don't
+// know whether the pointer is supposed to point to a NUL-terminated
+// string, and thus want to print it as a pointer to be safe.
+//
+// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM.
+
+// The default case.
+template <typename ToPrint, typename OtherOperand>
+class FormatForComparison {
+ public:
+ static ::std::string Format(const ToPrint& value) {
+ return ::testing::PrintToString(value);
+ }
+};
+
+// Array.
+template <typename ToPrint, size_t N, typename OtherOperand>
+class FormatForComparison<ToPrint[N], OtherOperand> {
+ public:
+ static ::std::string Format(const ToPrint* value) {
+ return FormatForComparison<const ToPrint*, OtherOperand>::Format(value);
+ }
+};
+
+// By default, print C string as pointers to be safe, as we don't know
+// whether they actually point to a NUL-terminated string.
+
+#define GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(CharType) \
+ template <typename OtherOperand> \
+ class FormatForComparison<CharType*, OtherOperand> { \
+ public: \
+ static ::std::string Format(CharType* value) { \
+ return ::testing::PrintToString(static_cast<const void*>(value)); \
+ } \
+ }
+
+GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char);
+GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char);
+GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(wchar_t);
+GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const wchar_t);
+
+#undef GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_
+
+// If a C string is compared with an STL string object, we know it's meant
+// to point to a NUL-terminated string, and thus can print it as a string.
+
+#define GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(CharType, OtherStringType) \
+ template <> \
+ class FormatForComparison<CharType*, OtherStringType> { \
+ public: \
+ static ::std::string Format(CharType* value) { \
+ return ::testing::PrintToString(value); \
+ } \
+ }
+
+GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::std::string);
+GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::std::string);
+
+#if GTEST_HAS_GLOBAL_STRING
+GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::string);
+GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::string);
+#endif
+
+#if GTEST_HAS_GLOBAL_WSTRING
+GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::wstring);
+GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::wstring);
+#endif
+
+#if GTEST_HAS_STD_WSTRING
+GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::std::wstring);
+GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::std::wstring);
+#endif
+
+#undef GTEST_IMPL_FORMAT_C_STRING_AS_STRING_
+
+// Formats a comparison assertion (e.g. ASSERT_EQ, EXPECT_LT, and etc)
+// operand to be used in a failure message. The type (but not value)
+// of the other operand may affect the format. This allows us to
+// print a char* as a raw pointer when it is compared against another
+// char* or void*, and print it as a C string when it is compared
+// against an std::string object, for example.
+//
+// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM.
+template <typename T1, typename T2>
+std::string FormatForComparisonFailureMessage(
+ const T1& value, const T2& /* other_operand */) {
+ return FormatForComparison<T1, T2>::Format(value);
+}
+
// UniversalPrinter<T>::Print(value, ostream_ptr) prints the given
// value to the given ostream. The caller must ensure that
// 'ostream_ptr' is not NULL, or the behavior is undefined.
@@ -9671,14 +10640,16 @@ inline void PrintTo(const ::std::wstring& s, ::std::ostream* os) {
}
#endif // GTEST_HAS_STD_WSTRING
-#if GTEST_HAS_TR1_TUPLE
-// Overload for ::std::tr1::tuple. Needed for printing function arguments,
-// which are packed as tuples.
-
+#if GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_
// Helper function for printing a tuple. T must be instantiated with
// a tuple type.
template <typename T>
void PrintTupleTo(const T& t, ::std::ostream* os);
+#endif // GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_
+
+#if GTEST_HAS_TR1_TUPLE
+// Overload for ::std::tr1::tuple. Needed for printing function arguments,
+// which are packed as tuples.
// Overloaded PrintTo() for tuples of various arities. We support
// tuples of up-to 10 fields. The following implementation works
@@ -9752,6 +10723,13 @@ void PrintTo(
}
#endif // GTEST_HAS_TR1_TUPLE
+#if GTEST_HAS_STD_TUPLE_
+template <typename... Types>
+void PrintTo(const ::std::tuple<Types...>& t, ::std::ostream* os) {
+ PrintTupleTo(t, os);
+}
+#endif // GTEST_HAS_STD_TUPLE_
+
// Overload for std::pair.
template <typename T1, typename T2>
void PrintTo(const ::std::pair<T1, T2>& value, ::std::ostream* os) {
@@ -9771,10 +10749,7 @@ class UniversalPrinter {
public:
// MSVC warns about adding const to a function type, so we want to
// disable the warning.
-#ifdef _MSC_VER
-# pragma warning(push) // Saves the current warning state.
-# pragma warning(disable:4180) // Temporarily disables warning 4180.
-#endif // _MSC_VER
+ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180)
// Note: we deliberately don't call this PrintTo(), as that name
// conflicts with ::testing::internal::PrintTo in the body of the
@@ -9791,9 +10766,7 @@ class UniversalPrinter {
PrintTo(value, os);
}
-#ifdef _MSC_VER
-# pragma warning(pop) // Restores the warning state.
-#endif // _MSC_VER
+ GTEST_DISABLE_MSC_WARNINGS_POP_()
};
// UniversalPrintArray(begin, len, os) prints an array of 'len'
@@ -9845,10 +10818,7 @@ class UniversalPrinter<T&> {
public:
// MSVC warns about adding const to a function type, so we want to
// disable the warning.
-#ifdef _MSC_VER
-# pragma warning(push) // Saves the current warning state.
-# pragma warning(disable:4180) // Temporarily disables warning 4180.
-#endif // _MSC_VER
+ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180)
static void Print(const T& value, ::std::ostream* os) {
// Prints the address of the value. We use reinterpret_cast here
@@ -9859,9 +10829,7 @@ class UniversalPrinter<T&> {
UniversalPrint(value, os);
}
-#ifdef _MSC_VER
-# pragma warning(pop) // Restores the warning state.
-#endif // _MSC_VER
+ GTEST_DISABLE_MSC_WARNINGS_POP_()
};
// Prints a value tersely: for a reference type, the referenced value
@@ -9947,16 +10915,65 @@ void UniversalPrint(const T& value, ::std::ostream* os) {
UniversalPrinter<T1>::Print(value, os);
}
-#if GTEST_HAS_TR1_TUPLE
typedef ::std::vector<string> Strings;
+// TuplePolicy<TupleT> must provide:
+// - tuple_size
+// size of tuple TupleT.
+// - get<size_t I>(const TupleT& t)
+// static function extracting element I of tuple TupleT.
+// - tuple_element<size_t I>::type
+// type of element I of tuple TupleT.
+template <typename TupleT>
+struct TuplePolicy;
+
+#if GTEST_HAS_TR1_TUPLE
+template <typename TupleT>
+struct TuplePolicy {
+ typedef TupleT Tuple;
+ static const size_t tuple_size = ::std::tr1::tuple_size<Tuple>::value;
+
+ template <size_t I>
+ struct tuple_element : ::std::tr1::tuple_element<I, Tuple> {};
+
+ template <size_t I>
+ static typename AddReference<
+ const typename ::std::tr1::tuple_element<I, Tuple>::type>::type get(
+ const Tuple& tuple) {
+ return ::std::tr1::get<I>(tuple);
+ }
+};
+template <typename TupleT>
+const size_t TuplePolicy<TupleT>::tuple_size;
+#endif // GTEST_HAS_TR1_TUPLE
+
+#if GTEST_HAS_STD_TUPLE_
+template <typename... Types>
+struct TuplePolicy< ::std::tuple<Types...> > {
+ typedef ::std::tuple<Types...> Tuple;
+ static const size_t tuple_size = ::std::tuple_size<Tuple>::value;
+
+ template <size_t I>
+ struct tuple_element : ::std::tuple_element<I, Tuple> {};
+
+ template <size_t I>
+ static const typename ::std::tuple_element<I, Tuple>::type& get(
+ const Tuple& tuple) {
+ return ::std::get<I>(tuple);
+ }
+};
+template <typename... Types>
+const size_t TuplePolicy< ::std::tuple<Types...> >::tuple_size;
+#endif // GTEST_HAS_STD_TUPLE_
+
+#if GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_
// This helper template allows PrintTo() for tuples and
// UniversalTersePrintTupleFieldsToStrings() to be defined by
// induction on the number of tuple fields. The idea is that
// TuplePrefixPrinter<N>::PrintPrefixTo(t, os) prints the first N
// fields in tuple t, and can be defined in terms of
// TuplePrefixPrinter<N - 1>.
-
+//
// The inductive case.
template <size_t N>
struct TuplePref
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/48] hadoop git commit: HADOOP-11603. Metric Snapshot log can be
changed #MetricsSystemImpl.java since all the services will be initialized.
Contributed By Brahma Reddy Battula.
Posted by li...@apache.org.
HADOOP-11603. Metric Snapshot log can be changed #MetricsSystemImpl.java since all the services will be initialized. Contributed By Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13501053
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13501053
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13501053
Branch: refs/heads/HADOOP-13345
Commit: 13501053dd95f41ac14091f5bbd79bcb2a3896a3
Parents: abb9fa7
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Thu Nov 24 22:23:22 2016 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 24 22:23:22 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13501053/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
index 3f8696d..24173f5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
@@ -33,7 +33,6 @@ import javax.management.ObjectName;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.annotations.VisibleForTesting;
-import java.util.Locale;
import static com.google.common.base.Preconditions.*;
import org.apache.commons.configuration2.PropertiesConfiguration;
@@ -372,7 +371,8 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
}
}
}, millis, millis);
- LOG.info("Scheduled snapshot period at "+ (period/1000) +" second(s).");
+ LOG.info("Scheduled Metric snapshot period at " + (period / 1000)
+ + " second(s).");
}
synchronized void onTimerEvent() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/48] hadoop git commit: HDFS-11368. Erasure Coding: Deprecate
replication-related config keys. Contributed by Rakesh R.
Posted by li...@apache.org.
HDFS-11368. Erasure Coding: Deprecate replication-related config keys. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de489493
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de489493
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de489493
Branch: refs/heads/HADOOP-13345
Commit: de4894936a5b581572f35fa5b8979d9f23da0891
Parents: dd98a80
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Nov 23 16:42:06 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Nov 23 16:42:06 2016 -0800
----------------------------------------------------------------------
.../src/site/markdown/DeprecatedProperties.md | 7 ++-
.../apache/hadoop/hdfs/HdfsConfiguration.java | 10 +++-
.../hdfs/client/HdfsClientConfigKeys.java | 10 ++--
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 19 +++---
.../hadoop/hdfs/server/balancer/Balancer.java | 4 +-
.../server/blockmanagement/BlockManager.java | 62 ++++++++++++--------
.../BlockPlacementPolicyDefault.java | 8 +--
.../apache/hadoop/hdfs/server/mover/Mover.java | 4 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 4 +-
.../src/main/resources/hdfs-default.xml | 8 +--
.../apache/hadoop/hdfs/AdminStatesBaseTest.java | 4 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 2 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +-
.../TestDFSStripedOutputStreamWithFailure.java | 2 +-
.../hdfs/TestDecommissionWithStriped.java | 10 ++--
.../apache/hadoop/hdfs/TestDeprecatedKeys.java | 22 ++++++-
.../hadoop/hdfs/TestEncryptedTransfer.java | 2 +-
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 2 +-
.../apache/hadoop/hdfs/TestFileChecksum.java | 2 +-
.../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +-
.../hadoop/hdfs/TestMissingBlocksAlert.java | 3 +-
.../hdfs/TestReadStripedFileWithDecoding.java | 3 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 4 +-
.../hdfs/TestReplaceDatanodeOnFailure.java | 3 +-
.../org/apache/hadoop/hdfs/TestReplication.java | 3 +-
.../hadoop/hdfs/TestWriteReadStripedFile.java | 2 +-
.../hdfs/server/balancer/TestBalancer.java | 26 ++++----
.../blockmanagement/BlockManagerTestUtil.java | 21 ++++---
.../TestBlocksWithNotEnoughRacks.java | 4 +-
.../server/blockmanagement/TestNodeCount.java | 2 +-
.../TestPendingInvalidateBlock.java | 2 +-
.../TestPendingReconstruction.java | 2 +-
...constructStripedBlocksWithRackAwareness.java | 8 +--
.../blockmanagement/TestReplicationPolicy.java | 2 +-
.../TestReplicationPolicyConsiderLoad.java | 2 +-
.../TestDataNodeErasureCodingMetrics.java | 2 +-
.../fsdataset/impl/TestLazyPersistFiles.java | 12 ++--
.../hadoop/hdfs/server/mover/TestMover.java | 9 ++-
.../hdfs/server/mover/TestStorageMover.java | 4 +-
.../server/namenode/NNThroughputBenchmark.java | 10 ++--
.../TestAddOverReplicatedStripedBlocks.java | 2 +-
.../namenode/TestDecommissioningStatus.java | 5 +-
.../server/namenode/TestFSEditLogLoader.java | 2 +-
.../hdfs/server/namenode/TestHostsFiles.java | 4 +-
.../hdfs/server/namenode/TestMetaSave.java | 5 +-
.../namenode/TestReconstructStripedBlocks.java | 9 +--
.../server/namenode/TestStripedINodeFile.java | 5 +-
.../hdfs/server/namenode/ha/TestDNFencing.java | 11 ++--
.../ha/TestDNFencingWithReplication.java | 2 +-
.../namenode/ha/TestPipelinesFailover.java | 5 +-
.../TestNNMetricFilesInGetListingOps.java | 2 +-
.../namenode/metrics/TestNameNodeMetrics.java | 12 ++--
52 files changed, 216 insertions(+), 158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
index 8de1be2..38a1954 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
@@ -43,8 +43,11 @@ The following table lists the configuration property names that are deprecated i
| dfs.permissions | dfs.permissions.enabled |
| dfs.permissions.supergroup | dfs.permissions.superusergroup |
| dfs.read.prefetch.size | dfs.client.read.prefetch.size |
-| dfs.replication.considerLoad | dfs.namenode.replication.considerLoad |
-| dfs.replication.interval | dfs.namenode.replication.interval |
+| dfs.replication.considerLoad | dfs.namenode.redundancy.considerLoad |
+| dfs.namenode.replication.considerLoad | dfs.namenode.redundancy.considerLoad |
+| dfs.namenode.replication.considerLoad.factor | dfs.namenode.redundancy.considerLoad.factor |
+| dfs.replication.interval | dfs.namenode.redundancy.interval |
+| dfs.namenode.replication.interval | dfs.namenode.redundancy.interval |
| dfs.replication.min | dfs.namenode.replication.min |
| dfs.replication.pending.timeout.sec | dfs.namenode.reconstruction.pending.timeout-sec |
| dfs.namenode.replication.pending.timeout-sec | dfs.namenode.reconstruction.pending.timeout-sec |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
index 3f6eae0..4d4ade2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
@@ -115,9 +115,15 @@ public class HdfsConfiguration extends Configuration {
new DeprecationDelta("dfs.access.time.precision",
DeprecatedKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY),
new DeprecationDelta("dfs.replication.considerLoad",
- DeprecatedKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY),
+ DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY),
+ new DeprecationDelta("dfs.namenode.replication.considerLoad",
+ DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY),
+ new DeprecationDelta("dfs.namenode.replication.considerLoad.factor",
+ DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR),
new DeprecationDelta("dfs.replication.interval",
- DeprecatedKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY),
+ DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY),
+ new DeprecationDelta("dfs.namenode.replication.interval",
+ DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY),
new DeprecationDelta("dfs.replication.min",
DeprecatedKeys.DFS_NAMENODE_REPLICATION_MIN_KEY),
new DeprecationDelta("dfs.replication.pending.timeout.sec",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 4c754d9..7ad79e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -204,10 +204,12 @@ public interface HdfsClientConfigKeys {
String DFS_METRICS_SESSION_ID_KEY = "dfs.metrics.session-id";
String DFS_NAMENODE_ACCESSTIME_PRECISION_KEY =
"dfs.namenode.accesstime.precision";
- String DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY =
- "dfs.namenode.replication.considerLoad";
- String DFS_NAMENODE_REPLICATION_INTERVAL_KEY =
- "dfs.namenode.replication.interval";
+ String DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY =
+ "dfs.namenode.redundancy.considerLoad";
+ String DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR =
+ "dfs.namenode.redundancy.considerLoad.factor";
+ String DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY =
+ "dfs.namenode.redundancy.interval.seconds";
String DFS_NAMENODE_REPLICATION_MIN_KEY = "dfs.namenode.replication.min";
String DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY =
"dfs.namenode.reconstruction.pending.timeout-sec";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b9fd939..d7d3c9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -194,16 +194,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_ACCESSTIME_PRECISION_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY;
public static final long DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT = 3600000;
- public static final String DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY =
- HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY;
- public static final boolean DFS_NAMENODE_REPLICATION_CONSIDERLOAD_DEFAULT = true;
- public static final String DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR =
- "dfs.namenode.replication.considerLoad.factor";
+ public static final String DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY =
+ HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY;
+ public static final boolean DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_DEFAULT =
+ true;
+ public static final String DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR =
+ "dfs.namenode.redundancy.considerLoad.factor";
public static final double
- DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR_DEFAULT = 2.0;
- public static final String DFS_NAMENODE_REPLICATION_INTERVAL_KEY =
- HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY;
- public static final int DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT = 3;
+ DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR_DEFAULT = 2.0;
+ public static final String DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY =
+ HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY;
+ public static final int DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT = 3;
public static final String DFS_NAMENODE_REPLICATION_MIN_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REPLICATION_MIN_KEY;
public static final int DFS_NAMENODE_REPLICATION_MIN_DEFAULT = 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 61352f3..75eecde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -673,8 +673,8 @@ public class Balancer {
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
TimeUnit.SECONDS) * 2000 +
conf.getTimeDuration(
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT,
TimeUnit.SECONDS) * 1000;
LOG.info("namenodes = " + namenodes);
LOG.info("parameters = " + p);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 2f57571..1b744e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -230,8 +230,11 @@ public class BlockManager implements BlockStatsMXBean {
return pendingReconstruction.getNumTimedOuts();
}
- /**replicationRecheckInterval is how often namenode checks for new replication work*/
- private final long replicationRecheckInterval;
+ /**
+ * redundancyRecheckInterval is how often namenode checks for new
+ * reconstruction work.
+ */
+ private final long redundancyRecheckIntervalMs;
/** How often to check and the limit for the storageinfo efficiency. */
private final long storageInfoDefragmentInterval;
@@ -244,8 +247,8 @@ public class BlockManager implements BlockStatsMXBean {
*/
final BlocksMap blocksMap;
- /** Replication thread. */
- final Daemon replicationThread = new Daemon(new ReplicationMonitor());
+ /** Redundancy thread. */
+ private final Daemon redundancyThread = new Daemon(new RedundancyMonitor());
/** StorageInfoDefragmenter thread. */
private final Daemon storageInfoDefragmenterThread =
@@ -435,10 +438,10 @@ public class BlockManager implements BlockStatsMXBean {
this.blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
- this.replicationRecheckInterval =
- conf.getTimeDuration(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
- TimeUnit.SECONDS) * 1000L;
+ this.redundancyRecheckIntervalMs = conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT,
+ TimeUnit.SECONDS) * 1000;
this.storageInfoDefragmentInterval =
conf.getLong(
@@ -493,7 +496,8 @@ public class BlockManager implements BlockStatsMXBean {
LOG.info("maxReplication = " + maxReplication);
LOG.info("minReplication = " + minReplication);
LOG.info("maxReplicationStreams = " + maxReplicationStreams);
- LOG.info("replicationRecheckInterval = " + replicationRecheckInterval);
+ LOG.info("redundancyRecheckInterval = " + redundancyRecheckIntervalMs +
+ "ms");
LOG.info("encryptDataTransfer = " + encryptDataTransfer);
LOG.info("maxNumBlocksToLog = " + maxNumBlocksToLog);
}
@@ -586,7 +590,7 @@ public class BlockManager implements BlockStatsMXBean {
return blockTokenSecretManager;
}
- /** Allow silent termination of replication monitor for testing */
+ /** Allow silent termination of redundancy monitor for testing. */
@VisibleForTesting
void enableRMTerminationForTesting() {
checkNSRunning = false;
@@ -604,8 +608,8 @@ public class BlockManager implements BlockStatsMXBean {
public void activate(Configuration conf, long blockTotal) {
pendingReconstruction.start();
datanodeManager.activate(conf);
- this.replicationThread.setName("ReplicationMonitor");
- this.replicationThread.start();
+ this.redundancyThread.setName("RedundancyMonitor");
+ this.redundancyThread.start();
storageInfoDefragmenterThread.setName("StorageInfoMonitor");
storageInfoDefragmenterThread.start();
this.blockReportThread.start();
@@ -616,10 +620,10 @@ public class BlockManager implements BlockStatsMXBean {
public void close() {
bmSafeMode.close();
try {
- replicationThread.interrupt();
+ redundancyThread.interrupt();
storageInfoDefragmenterThread.interrupt();
blockReportThread.interrupt();
- replicationThread.join(3000);
+ redundancyThread.join(3000);
storageInfoDefragmenterThread.join(3000);
blockReportThread.join(3000);
} catch (InterruptedException ie) {
@@ -880,7 +884,7 @@ public class BlockManager implements BlockStatsMXBean {
/**
* If IBR is not sent from expected locations yet, add the datanodes to
- * pendingReconstruction in order to keep ReplicationMonitor from scheduling
+ * pendingReconstruction in order to keep RedundancyMonitor from scheduling
* the block.
*/
public void addExpectedReplicasToPending(BlockInfo blk) {
@@ -1884,7 +1888,7 @@ public class BlockManager implements BlockStatsMXBean {
if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum)) {
neededReconstruction.remove(block, priority);
rw.resetTargets();
- blockLog.debug("BLOCK* Removing {} from neededReplications as" +
+ blockLog.debug("BLOCK* Removing {} from neededReconstruction as" +
" it has enough replicas", block);
return false;
}
@@ -1910,8 +1914,8 @@ public class BlockManager implements BlockStatsMXBean {
// reconstructions that fail after an appropriate amount of time.
pendingReconstruction.increment(block,
DatanodeStorageInfo.toDatanodeDescriptors(targets));
- blockLog.debug("BLOCK* block {} is moved from neededReplications to "
- + "pendingReplications", block);
+ blockLog.debug("BLOCK* block {} is moved from neededReconstruction to "
+ + "pendingReconstruction", block);
int numEffectiveReplicas = numReplicas.liveReplicas() + pendingNum;
// remove from neededReconstruction
@@ -4298,32 +4302,32 @@ public class BlockManager implements BlockStatsMXBean {
/**
* Periodically calls computeBlockRecoveryWork().
*/
- private class ReplicationMonitor implements Runnable {
+ private class RedundancyMonitor implements Runnable {
@Override
public void run() {
while (namesystem.isRunning()) {
try {
- // Process replication work only when active NN is out of safe mode.
+ // Process recovery work only when active NN is out of safe mode.
if (isPopulatingReplQueues()) {
computeDatanodeWork();
processPendingReconstructions();
rescanPostponedMisreplicatedBlocks();
}
- Thread.sleep(replicationRecheckInterval);
+ TimeUnit.MILLISECONDS.sleep(redundancyRecheckIntervalMs);
} catch (Throwable t) {
if (!namesystem.isRunning()) {
- LOG.info("Stopping ReplicationMonitor.");
+ LOG.info("Stopping RedundancyMonitor.");
if (!(t instanceof InterruptedException)) {
- LOG.info("ReplicationMonitor received an exception"
+ LOG.info("RedundancyMonitor received an exception"
+ " while shutting down.", t);
}
break;
} else if (!checkNSRunning && t instanceof InterruptedException) {
- LOG.info("Stopping ReplicationMonitor for testing.");
+ LOG.info("Stopping RedundancyMonitor for testing.");
break;
}
- LOG.error("ReplicationMonitor thread received Runtime exception. ",
+ LOG.error("RedundancyMonitor thread received Runtime exception. ",
t);
terminate(1, t);
}
@@ -4692,6 +4696,14 @@ public class BlockManager implements BlockStatsMXBean {
}
}
+ /**
+ * @return redundancy thread.
+ */
+ @VisibleForTesting
+ Daemon getRedundancyThread() {
+ return redundancyThread;
+ }
+
public BlockIdManager getBlockIdManager() {
return blockIdManager;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index cb65ba8..eb54667 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -83,11 +83,11 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
NetworkTopology clusterMap,
Host2NodesMap host2datanodeMap) {
this.considerLoad = conf.getBoolean(
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_DEFAULT);
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_DEFAULT);
this.considerLoadFactor = conf.getDouble(
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR_DEFAULT);
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR,
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR_DEFAULT);
this.stats = stats;
this.clusterMap = clusterMap;
this.host2datanodeMap = host2datanodeMap;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 4fbae76..4ab55d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -610,8 +610,8 @@ public class Mover {
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
TimeUnit.SECONDS) * 2000 +
conf.getTimeDuration(
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT,
TimeUnit.SECONDS) * 1000;
AtomicInteger retryCount = new AtomicInteger(0);
LOG.info("namenodes = " + namenodes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
index 899f0d7..b859148 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
@@ -469,11 +469,11 @@ public class BackupNode extends NameNode {
* (not run or not pass any control commands to DataNodes)
* on BackupNode:
* {@link LeaseManager.Monitor} protected by SafeMode.
- * {@link BlockManager.ReplicationMonitor} protected by SafeMode.
+ * {@link BlockManager.RedundancyMonitor} protected by SafeMode.
* {@link HeartbeatManager.Monitor} protected by SafeMode.
* {@link DecommissionManager.Monitor} need to prohibit refreshNodes().
* {@link PendingReconstructionBlocks.PendingReconstructionMonitor}
- * harmless, because ReplicationMonitor is muted.
+ * harmless, because RedundancyMonitor is muted.
*/
@Override
public void startActiveServices() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index acb24fc..c9d74bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -296,14 +296,14 @@
</property>
<property>
- <name>dfs.namenode.replication.considerLoad</name>
+ <name>dfs.namenode.redundancy.considerLoad</name>
<value>true</value>
<description>Decide if chooseTarget considers the target's load or not
</description>
</property>
<property>
- <name>dfs.namenode.replication.considerLoad.factor</name>
+ <name>dfs.namenode.redundancy.considerLoad.factor</name>
<value>2.0</value>
<description>The factor by which a node's load can exceed the average
before being rejected for writes, only if considerLoad is true.
@@ -980,10 +980,10 @@
</property>
<property>
- <name>dfs.namenode.replication.interval</name>
+ <name>dfs.namenode.redundancy.interval.seconds</name>
<value>3s</value>
<description>The periodicity in seconds with which the namenode computes
- replication work for datanodes. Support multiple time unit suffix(case insensitive),
+ low redundancy work for datanodes. Support multiple time unit suffix(case insensitive),
as described in dfs.heartbeat.interval.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
index 534c5e0..0ed01f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
@@ -90,14 +90,14 @@ public class AdminStatesBaseTest {
}
// Setup conf
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
200);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL);
conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
BLOCKREPORT_INTERVAL_MSEC);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
NAMENODE_REPLICATION_INTERVAL);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 624fc74..6d55c46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -58,7 +58,7 @@ public class TestBlockStoragePolicy {
static {
conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
POLICY_SUITE = BlockStoragePolicySuite.createDefaultSuite();
DEFAULT_STORAGE_POLICY = POLICY_SUITE.getDefaultPolicy();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index 47d5e02..b686f28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -67,7 +67,7 @@ public class TestDFSStripedOutputStream {
int numDNs = dataBlocks + parityBlocks + 2;
conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index 53059ce..cde07a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure {
private HdfsConfiguration newHdfsConfiguration() {
final HdfsConfiguration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
index 1574a03..2eab309 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
@@ -111,26 +111,26 @@ public class TestDecommissionWithStriped {
writeConfigFile(excludeFile, null);
// Setup conf
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
conf.set(DFSConfigKeys.DFS_HOSTS, hostsFile.toUri().getPath());
conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath());
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
2000);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
BLOCKREPORT_INTERVAL_MSEC);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
4);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
NAMENODE_REPLICATION_INTERVAL);
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setInt(
DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY,
cellSize - 1);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
numDNs = dataBlocks + parityBlocks + 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeprecatedKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeprecatedKeys.java
index 7152092..ad5c2a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeprecatedKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeprecatedKeys.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import org.apache.hadoop.conf.Configuration;
import org.junit.Test;
@@ -33,8 +34,23 @@ public class TestDeprecatedKeys {
String scriptFile = conf.get(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY);
assertTrue(scriptFile.equals("xyz")) ;
conf.setInt("dfs.replication.interval", 1);
- String alpha = DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY;
- int repInterval = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 3) ;
- assertTrue(repInterval == 1) ;
+ int redundancyInterval = conf
+ .getInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 3);
+ assertTrue(redundancyInterval == 1);
+ int repInterval = conf.getInt("dfs.replication.interval", 3);
+ assertTrue(repInterval == 1);
+ repInterval = conf.getInt("dfs.namenode.replication.interval", 3);
+ assertTrue(repInterval == 1);
+
+ conf.setBoolean("dfs.replication.considerLoad", false);
+ assertFalse(conf.getBoolean(
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, true));
+ assertFalse(conf.getBoolean("dfs.replication.considerLoad", true));
+
+ conf.setDouble("dfs.namenode.replication.considerLoad.factor", 5.0);
+ assertTrue(5.0 == conf.getDouble(
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR, 2.0));
+ assertTrue(5.0 == conf
+ .getDouble("dfs.namenode.replication.considerLoad.factor", 2.0));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java
index 78020b2..cc90863 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java
@@ -329,7 +329,7 @@ public class TestEncryptedTransfer {
// client only retries establishing pipeline with the 4th node.
int numDataNodes = 4;
// do not consider load factor when selecting a data node
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
setEncryptionConfigKeys();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
index ae0f0c2..40bc314 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
@@ -86,7 +86,7 @@ public class TestFileAppend4 {
// handle under-replicated blocks quickly (for replication asserts)
conf.setInt(
DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, 5);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
// handle failures in the DFSClient pipeline quickly
// (for cluster.shutdown(); fs.close() idiom)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
index 1c75f90..9d60798 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
@@ -75,7 +75,7 @@ public class TestFileChecksum {
int numDNs = dataBlocks + parityBlocks + 2;
conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index ca8f6db..246818b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -85,7 +85,7 @@ public class TestLeaseRecoveryStriped {
conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setLong(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 6000L);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
index 90d42b6..e225141 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
@@ -59,7 +59,8 @@ public class TestMissingBlocksAlert {
try {
Configuration conf = new HdfsConfiguration();
//minimize test delay
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 0);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 0);
conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
int fileLen = 10*1024;
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, fileLen/2);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
index 6014332..506a005 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
@@ -99,7 +99,8 @@ public class TestReadStripedFileWithDecoding {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY, 0);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
+ false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
fs = cluster.getFileSystem();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index a3d9ef6..4960c58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -95,8 +95,8 @@ public class TestReconstructStripedFile {
conf.setInt(
DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY,
cellSize - 1);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java
index 3dda7e8..aa5c70f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java
@@ -118,7 +118,8 @@ public class TestReplaceDatanodeOnFailure {
public void testReplaceDatanodeOnFailure() throws Exception {
final Configuration conf = new HdfsConfiguration();
// do not consider load factor when selecting a data node
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
+ false);
//always replace a datanode
ReplaceDatanodeOnFailure.write(Policy.ALWAYS, true, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
index 3e23db3..0f1bedd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
@@ -228,7 +228,8 @@ public class TestReplication {
*/
public void runReplication(boolean simulated) throws IOException {
Configuration conf = new HdfsConfiguration();
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
+ false);
if (simulated) {
SimulatedFSDataset.setFactory(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
index 8ada593..712f326 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
@@ -75,7 +75,7 @@ public class TestWriteReadStripedFile {
@Before
public void setup() throws IOException {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
fs = cluster.getFileSystem();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 730f655..1906d74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -179,7 +179,8 @@ public class TestBalancer {
conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
- conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 1L);
SimulatedFSDataset.setFactory(conf);
conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
@@ -212,10 +213,12 @@ public class TestBalancer {
void initConfWithStripe(Configuration conf) {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultBlockSize);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
+ false);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
SimulatedFSDataset.setFactory(conf);
- conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 1L);
conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
}
@@ -922,11 +925,13 @@ public class TestBalancer {
private static int runBalancer(Collection<URI> namenodes,
final BalancerParameters p,
Configuration conf) throws IOException, InterruptedException {
- final long sleeptime =
- conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000 +
- conf.getLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000;
+ final long sleeptime = conf.getLong(
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000
+ + conf.getLong(
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT)
+ * 1000;
LOG.info("namenodes = " + namenodes);
LOG.info("parameters = " + p);
LOG.info("Print stack trace", new Throwable());
@@ -1603,7 +1608,7 @@ public class TestBalancer {
Configuration conf = new HdfsConfiguration();
conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1);
conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
- conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
@@ -1670,7 +1675,8 @@ public class TestBalancer {
int blockSize = 5 * 1024 * 1024 ;
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
- conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 1L);
conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
index 5fb3b25..1c7442a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
@@ -105,25 +105,24 @@ public class BlockManagerTestUtil {
}
/**
- * @return replication monitor thread instance from block manager.
+ * @return redundancy monitor thread instance from block manager.
*/
- public static Daemon getReplicationThread(final BlockManager blockManager)
- {
- return blockManager.replicationThread;
+ public static Daemon getRedundancyThread(final BlockManager blockManager) {
+ return blockManager.getRedundancyThread();
}
-
+
/**
- * Stop the replication monitor thread
+ * Stop the redundancy monitor thread.
*/
- public static void stopReplicationThread(final BlockManager blockManager)
+ public static void stopRedundancyThread(final BlockManager blockManager)
throws IOException {
blockManager.enableRMTerminationForTesting();
- blockManager.replicationThread.interrupt();
+ blockManager.getRedundancyThread().interrupt();
try {
- blockManager.replicationThread.join();
- } catch(InterruptedException ie) {
+ blockManager.getRedundancyThread().join();
+ } catch (InterruptedException ie) {
throw new IOException(
- "Interrupted while trying to stop ReplicationMonitor");
+ "Interrupted while trying to stop RedundancyMonitor");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
index 197e3a6..0298cbc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
@@ -62,9 +62,9 @@ public class TestBlocksWithNotEnoughRacks {
// commands quickly (as replies to heartbeats)
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
- // Have the NN ReplicationMonitor compute the replication and
+ // Have the NN RedundancyMonitor compute the reconstruction and
// invalidation commands to send DNs every second.
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
// Have the NN check for pending replications every second so it
// quickly schedules additional replicas as they are identified.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
index 6058727..d915b6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
@@ -56,7 +56,7 @@ public class TestNodeCount {
60);
// reduce intervals to make test execution time shorter
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
// start a mini dfs cluster of 2 nodes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
index d856065..674f883 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
@@ -65,7 +65,7 @@ public class TestPendingInvalidateBlock {
// set the block report interval to 2s
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 2000);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
// disable the RPC timeout for debug
conf.setLong(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, 0);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java
index ba3be91..aeaaf4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java
@@ -374,7 +374,7 @@ public class TestPendingReconstruction {
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
DFS_REPLICATION_INTERVAL);
- CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
DFS_REPLICATION_INTERVAL);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(
DATANODE_COUNT).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
index 94ee623..ace8732 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
@@ -96,8 +96,8 @@ public class TestReconstructStripedBlocksWithRackAwareness {
@BeforeClass
public static void setup() throws Exception {
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 1);
}
@@ -139,7 +139,7 @@ public class TestReconstructStripedBlocksWithRackAwareness {
*
* In this test, we first need to create a scenario that a striped block has
* all the internal blocks but distributed in <6 racks. Then we check if the
- * replication monitor can correctly schedule the reconstruction work for it.
+ * redundancy monitor can correctly schedule the reconstruction work for it.
*/
@Test
public void testReconstructForNotEnoughRacks() throws Exception {
@@ -194,7 +194,7 @@ public class TestReconstructStripedBlocksWithRackAwareness {
fsn.writeUnlock();
}
- // check if replication monitor correctly schedule the replication work
+ // check if redundancy monitor correctly schedule the reconstruction work.
boolean scheduled = false;
for (int i = 0; i < 5; i++) { // retry 5 times
for (DatanodeStorageInfo storage : blockInfo.storages) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 9c62383..1af013d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -836,7 +836,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
int DFS_NAMENODE_REPLICATION_INTERVAL = 1000;
int HIGH_PRIORITY = 0;
Configuration conf = new Configuration();
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
.format(true).build();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java
index 8336fec..bcd8245 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java
@@ -53,7 +53,7 @@ public class TestReplicationPolicyConsiderLoad
@Override
DatanodeDescriptor[] getDatanodeDescriptors(Configuration conf) {
conf.setDouble(DFSConfigKeys
- .DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR, 1.2);
+ .DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR, 1.2);
final String[] racks = {
"/rack1",
"/rack1",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
index 98efc13c..1b0526b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
@@ -78,7 +78,7 @@ public class TestDataNodeErasureCodingMetrics {
conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 8c43592..d268d01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -104,8 +104,8 @@ public class TestLazyPersistFiles extends LazyPersistTestCase {
Thread.sleep(30000L);
assertThat(cluster.getNamesystem().getNumDeadDataNodes(), is(1));
- // Next, wait for the replication monitor to mark the file as corrupt
- Thread.sleep(2 * DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT * 1000);
+ // Next, wait for the redundancy monitor to mark the file as corrupt.
+ Thread.sleep(2 * DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT * 1000);
// Wait for the LazyPersistFileScrubber to run
Thread.sleep(2 * LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC * 1000);
@@ -137,8 +137,8 @@ public class TestLazyPersistFiles extends LazyPersistTestCase {
cluster.shutdownDataNodes();
Thread.sleep(30000L);
- // Next, wait for the replication monitor to mark the file as corrupt
- Thread.sleep(2 * DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT * 1000);
+ // Next, wait for the redundancy monitor to mark the file as corrupt.
+ Thread.sleep(2 * DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT * 1000);
// Wait for the LazyPersistFileScrubber to run
Thread.sleep(2 * LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC * 1000);
@@ -164,8 +164,8 @@ public class TestLazyPersistFiles extends LazyPersistTestCase {
cluster.restartNameNodes();
- // wait for the replication monitor to mark the file as corrupt
- Thread.sleep(2 * DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT * 1000);
+ // wait for the redundancy monitor to mark the file as corrupt.
+ Thread.sleep(2 * DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT * 1000);
Long corruptBlkCount = (long) Iterators.size(cluster.getNameNode()
.getNamesystem().getBlockManager().getCorruptReplicaBlockIterator());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index 4342dab..20a6959 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -105,7 +105,8 @@ public class TestMover {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
- conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 1L);
conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
}
@@ -482,8 +483,10 @@ public class TestMover {
void initConfWithStripe(Configuration conf) {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultBlockSize);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
- conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 1L);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
+ false);
}
@Test(timeout = 300000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index 1b5bd81..077997f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -93,8 +93,8 @@ public class TestStorageMover {
static {
DEFAULT_CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
DEFAULT_CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
- DEFAULT_CONF.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- 2L);
+ DEFAULT_CONF.setLong(
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 2L);
DEFAULT_CONF.setLong(DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_KEY, 2000L);
DEFAULT_POLICIES = BlockStoragePolicySuite.createDefaultSuite();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 4b9db07..a3d0be5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -1237,8 +1237,8 @@ public class NNThroughputBenchmark implements Tool {
} // end BlockReportStats
/**
- * Measures how fast replication monitor can compute data-node work.
- *
+ * Measures how fast redundancy monitor can compute data-node work.
+ *
* It runs only one thread until no more work can be scheduled.
*/
class ReplicationStats extends OperationStatsBase {
@@ -1265,7 +1265,7 @@ public class NNThroughputBenchmark implements Tool {
parseArguments(args);
// number of operations is 4 times the number of decommissioned
// blocks divided by the number of needed replications scanned
- // by the replication monitor in one iteration
+ // by the redundancy monitor in one iteration
numOpsRequired = (totalBlocks*replication*nodesToDecommission*2)
/ (numDatanodes*numDatanodes);
@@ -1314,8 +1314,8 @@ public class NNThroughputBenchmark implements Tool {
// start data-nodes; create a bunch of files; generate block reports.
blockReportObject.generateInputs(ignore);
- // stop replication monitor
- BlockManagerTestUtil.stopReplicationThread(namesystem.getBlockManager());
+ // stop redundancy monitor thread.
+ BlockManagerTestUtil.stopRedundancyThread(namesystem.getBlockManager());
// report blocks once
int nrDatanodes = blockReportObject.getNumDatanodes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
index 15a8756..c8b25564 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
@@ -74,7 +74,7 @@ public class TestAddOverReplicatedStripedBlocks {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
// disable block recovery
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
SimulatedFSDataset.setFactory(conf);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
index 7f1cc9a..8bdaa74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
@@ -78,7 +78,7 @@ public class TestDecommissioningStatus {
@Before
public void setUp() throws Exception {
conf = new HdfsConfiguration();
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
// Set up the hosts/exclude files.
@@ -89,7 +89,8 @@ public class TestDecommissioningStatus {
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
conf.setInt(
DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, 4);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 1000);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 1);
conf.setLong(DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
index f6e99ee..a57a936 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
@@ -156,7 +156,7 @@ public class TestFSEditLogLoader {
// start a cluster
Configuration conf = getConf();
// Replicate and heartbeat fast to shave a few seconds off test
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
MiniDFSCluster cluster = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java
index cb19c2c..79e7acc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java
@@ -76,9 +76,9 @@ public class TestHostsFiles {
// commands quickly (as replies to heartbeats)
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
- // Have the NN ReplicationMonitor compute the replication and
+ // Have the NN RedundancyMonitor compute the low redundant blocks and
// invalidation commands to send DNs every second.
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
// Have the NN check for pending replications every second so it
// quickly schedules additional replicas as they are identified.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
index 9816d00..0303a5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
@@ -64,8 +64,9 @@ public class TestMetaSave {
Configuration conf = new HdfsConfiguration();
// High value of replication interval
- // so that blocks remain under-replicated
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000);
+ // so that blocks remain less redundant
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 1000);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
conf.setLong(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1L);
conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, 5L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
index d43804a..cf1585c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
@@ -192,7 +192,8 @@ public class TestReconstructStripedBlocks {
public void test2RecoveryTasksForSameBlockGroup() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1000);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 1000);
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2)
.build();
@@ -255,8 +256,8 @@ public class TestReconstructStripedBlocks {
@Test
public void testCountLiveReplicas() throws Exception {
final HdfsConfiguration conf = new HdfsConfiguration();
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2)
.build();
@@ -299,7 +300,7 @@ public class TestReconstructStripedBlocks {
FSNamesystem fsn = cluster.getNamesystem();
BlockManager bm = fsn.getBlockManager();
- Thread.sleep(3000); // wait 3 running cycles of replication monitor
+ Thread.sleep(3000); // wait 3 running cycles of redundancy monitor
for (DataNode dn : cluster.getDataNodes()) {
DataNodeTestUtils.triggerHeartbeat(dn);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
index 6c4d73c..d92ec1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -307,8 +307,9 @@ public class TestStripedINodeFile {
int defaultStripedBlockSize = testECPolicy.getCellSize() * 4;
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultStripedBlockSize);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
- conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 1L);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
// start 10 datanodes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
index 15357b0..c1521bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
@@ -87,9 +87,10 @@ public class TestDNFencing {
public void setupCluster() throws Exception {
conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, SMALL_BLOCK);
- // Bump up replication interval so that we only run replication
+ // Bump up redundancy interval so that we only run low redundancy
// checks explicitly.
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 600);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+ 600);
// Increase max streams so that we re-replicate quickly.
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 1000);
// See RandomDeleterPolicy javadoc.
@@ -167,7 +168,7 @@ public class TestDNFencing {
// The blocks should no longer be postponed.
assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
- // Wait for NN2 to enact its deletions (replication monitor has to run, etc)
+ // Wait for NN2 to enact its deletions (redundancy monitor has to run, etc)
BlockManagerTestUtil.computeInvalidationWork(
nn2.getNamesystem().getBlockManager());
cluster.triggerHeartbeats();
@@ -258,7 +259,7 @@ public class TestDNFencing {
// The block should no longer be postponed.
assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
- // Wait for NN2 to enact its deletions (replication monitor has to run, etc)
+ // Wait for NN2 to enact its deletions (redundancy monitor has to run, etc)
BlockManagerTestUtil.computeInvalidationWork(
nn2.getNamesystem().getBlockManager());
@@ -358,7 +359,7 @@ public class TestDNFencing {
// The block should no longer be postponed.
assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
- // Wait for NN2 to enact its deletions (replication monitor has to run, etc)
+ // Wait for NN2 to enact its deletions (redundancy monitor has to run, etc)
BlockManagerTestUtil.computeInvalidationWork(
nn2.getNamesystem().getBlockManager());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java
index e4fe230..c91d4de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java
@@ -110,7 +110,7 @@ public class TestDNFencingWithReplication {
harness.conf.setInt(
DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1);
harness.conf.setInt(
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
final MiniDFSCluster cluster = harness.startCluster();
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/48] hadoop git commit: HDFS-11058. Implement 'hadoop fs -df'
command for ViewFileSystem. Contributed by Manoj Govindassamy.
Posted by li...@apache.org.
HDFS-11058. Implement 'hadoop fs -df' command for ViewFileSystem. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd98a800
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd98a800
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd98a800
Branch: refs/heads/HADOOP-13345
Commit: dd98a8005ad8939ffb6faba1ff0170387e91a8de
Parents: 1f12867
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Nov 23 16:40:39 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Nov 23 16:40:39 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/fs/shell/FsUsage.java | 124 ++++++++++----
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 56 ++++---
.../hadoop/fs/viewfs/ViewFileSystemUtil.java | 164 +++++++++++++++++++
.../hadoop/fs/viewfs/TestViewfsFileStatus.java | 9 +-
.../fs/viewfs/ViewFileSystemBaseTest.java | 74 ++++++++-
.../fs/viewfs/TestViewFileSystemHdfs.java | 24 +++
6 files changed, 395 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd98a800/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
index 197920f..6b1d7e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
@@ -20,19 +20,24 @@ package org.apache.hadoop.fs.shell;
import java.io.IOException;
import java.io.PrintStream;
+import java.net.URI;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
+import org.apache.hadoop.fs.viewfs.ViewFileSystemUtil;
import org.apache.hadoop.util.StringUtils;
-/** Base class for commands related to viewing filesystem usage, such as
- * du and df
+/**
+ * Base class for commands related to viewing filesystem usage,
+ * such as du and df.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -44,15 +49,27 @@ class FsUsage extends FsCommand {
factory.addClass(Dus.class, "-dus");
}
- protected boolean humanReadable = false;
- protected TableBuilder usagesTable;
-
+ private boolean humanReadable = false;
+ private TableBuilder usagesTable;
+
protected String formatSize(long size) {
return humanReadable
? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
: String.valueOf(size);
}
+ public TableBuilder getUsagesTable() {
+ return usagesTable;
+ }
+
+ public void setUsagesTable(TableBuilder usagesTable) {
+ this.usagesTable = usagesTable;
+ }
+
+ public void setHumanReadable(boolean humanReadable) {
+ this.humanReadable = humanReadable;
+ }
+
/** Show the size of a partition in the filesystem */
public static class Df extends FsUsage {
public static final String NAME = "df";
@@ -70,38 +87,74 @@ class FsUsage extends FsCommand {
throws IOException {
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h");
cf.parse(args);
- humanReadable = cf.getOpt("h");
+ setHumanReadable(cf.getOpt("h"));
if (args.isEmpty()) args.add(Path.SEPARATOR);
}
@Override
protected void processArguments(LinkedList<PathData> args)
throws IOException {
- usagesTable = new TableBuilder(
- "Filesystem", "Size", "Used", "Available", "Use%");
- usagesTable.setRightAlign(1, 2, 3, 4);
-
+ setUsagesTable(new TableBuilder(
+ "Filesystem", "Size", "Used", "Available", "Use%", "Mounted on"));
+ getUsagesTable().setRightAlign(1, 2, 3, 4);
+
super.processArguments(args);
- if (!usagesTable.isEmpty()) {
- usagesTable.printToStream(out);
+ if (!getUsagesTable().isEmpty()) {
+ getUsagesTable().printToStream(out);
}
}
- @Override
- protected void processPath(PathData item) throws IOException {
- FsStatus fsStats = item.fs.getStatus(item.path);
- long size = fsStats.getCapacity();
- long used = fsStats.getUsed();
- long free = fsStats.getRemaining();
-
- usagesTable.addRow(
- item.fs.getUri(),
+ /**
+ * Add a new row to the usages table for the given FileSystem URI.
+ *
+ * @param uri - FileSystem URI
+ * @param fsStatus - FileSystem status
+ * @param mountedOnPath - FileSystem mounted on path
+ */
+ private void addToUsagesTable(URI uri, FsStatus fsStatus,
+ String mountedOnPath) {
+ long size = fsStatus.getCapacity();
+ long used = fsStatus.getUsed();
+ long free = fsStatus.getRemaining();
+ getUsagesTable().addRow(
+ uri,
formatSize(size),
formatSize(used),
formatSize(free),
- StringUtils.formatPercent((double)used/(double)size, 0)
+ StringUtils.formatPercent((double) used / (double) size, 0),
+ mountedOnPath
);
}
+
+ @Override
+ protected void processPath(PathData item) throws IOException {
+ if (ViewFileSystemUtil.isViewFileSystem(item.fs)) {
+ ViewFileSystem viewFileSystem = (ViewFileSystem) item.fs;
+ Map<ViewFileSystem.MountPoint, FsStatus> fsStatusMap =
+ ViewFileSystemUtil.getStatus(viewFileSystem, item.path);
+
+ for (Map.Entry<ViewFileSystem.MountPoint, FsStatus> entry :
+ fsStatusMap.entrySet()) {
+ ViewFileSystem.MountPoint viewFsMountPoint = entry.getKey();
+ FsStatus fsStatus = entry.getValue();
+
+ // Add the viewfs mount point status to report
+ URI[] mountPointFileSystemURIs =
+ viewFsMountPoint.getTargetFileSystemURIs();
+ // Since LinkMerge is not supported yet, we
+ // should ideally see mountPointFileSystemURIs
+ // array with only one element.
+ addToUsagesTable(mountPointFileSystemURIs[0],
+ fsStatus, viewFsMountPoint.getMountedOnPath().toString());
+ }
+ } else {
+ // Hide the columns specific to ViewFileSystem
+ getUsagesTable().setColumnHide(5, true);
+ FsStatus fsStatus = item.fs.getStatus(item.path);
+ addToUsagesTable(item.fs.getUri(), fsStatus, "/");
+ }
+ }
+
}
/** show disk usage */
@@ -128,7 +181,7 @@ class FsUsage extends FsCommand {
protected void processOptions(LinkedList<String> args) throws IOException {
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s", "x");
cf.parse(args);
- humanReadable = cf.getOpt("h");
+ setHumanReadable(cf.getOpt("h"));
summary = cf.getOpt("s");
excludeSnapshots = cf.getOpt("x");
if (args.isEmpty()) args.add(Path.CUR_DIR);
@@ -137,10 +190,10 @@ class FsUsage extends FsCommand {
@Override
protected void processArguments(LinkedList<PathData> args)
throws IOException {
- usagesTable = new TableBuilder(3);
+ setUsagesTable(new TableBuilder(3));
super.processArguments(args);
- if (!usagesTable.isEmpty()) {
- usagesTable.printToStream(out);
+ if (!getUsagesTable().isEmpty()) {
+ getUsagesTable().printToStream(out);
}
}
@@ -163,7 +216,8 @@ class FsUsage extends FsCommand {
length -= contentSummary.getSnapshotLength();
spaceConsumed -= contentSummary.getSnapshotSpaceConsumed();
}
- usagesTable.addRow(formatSize(length), formatSize(spaceConsumed), item);
+ getUsagesTable().addRow(formatSize(length),
+ formatSize(spaceConsumed), item);
}
}
/** show disk usage summary */
@@ -191,6 +245,7 @@ class FsUsage extends FsCommand {
protected List<String[]> rows;
protected int[] widths;
protected boolean[] rightAlign;
+ private boolean[] hide;
/**
* Create a table w/o headers
@@ -200,6 +255,7 @@ class FsUsage extends FsCommand {
rows = new ArrayList<String[]>();
widths = new int[columns];
rightAlign = new boolean[columns];
+ hide = new boolean[columns];
}
/**
@@ -219,7 +275,14 @@ class FsUsage extends FsCommand {
public void setRightAlign(int ... indexes) {
for (int i : indexes) rightAlign[i] = true;
}
-
+
+ /**
+ * Hide the given column index
+ */
+ public void setColumnHide(int columnIndex, boolean hideCol) {
+ hide[columnIndex] = hideCol;
+ }
+
/**
* Add a row of objects to the table
* @param objects the values
@@ -234,7 +297,7 @@ class FsUsage extends FsCommand {
}
/**
- * Render the table to a stream
+ * Render the table to a stream.
* @param out PrintStream for output
*/
public void printToStream(PrintStream out) {
@@ -242,6 +305,9 @@ class FsUsage extends FsCommand {
StringBuilder fmt = new StringBuilder();
for (int i=0; i < widths.length; i++) {
+ if (hide[i]) {
+ continue;
+ }
if (fmt.length() != 0) fmt.append(" ");
if (rightAlign[i]) {
fmt.append("%"+widths[i]+"s");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd98a800/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 76606e9..fa9f712 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -33,7 +33,6 @@ import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -49,6 +48,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
@@ -90,34 +90,35 @@ public class ViewFileSystem extends FileSystem {
return readOnlyMountTable(operation, p.toString());
}
- static public class MountPoint {
+ /**
+ * MountPoint representation built from the configuration.
+ */
+ public static class MountPoint {
+
/**
- * The source of the mount.
+ * The mounted on path location.
*/
- private Path src;
+ private final Path mountedOnPath;
/**
- * One or more targets of the mount.
- * Multiple targets imply MergeMount.
+ * Array of target FileSystem URIs.
*/
- private URI[] targets;
+ private final URI[] targetFileSystemURIs;
- MountPoint(Path srcPath, URI[] targetURIs) {
- src = srcPath;
- targets = targetURIs;
+ MountPoint(Path srcPath, URI[] targetFs) {
+ mountedOnPath = srcPath;
+ targetFileSystemURIs = targetFs;
}
- @VisibleForTesting
- Path getSrc() {
- return src;
+ public Path getMountedOnPath() {
+ return mountedOnPath;
}
- @VisibleForTesting
- URI[] getTargets() {
- return targets;
+ public URI[] getTargetFileSystemURIs() {
+ return targetFileSystemURIs;
}
}
-
+
final long creationTime; // of the the mount table
final UserGroupInformation ugi; // the user/group of user who created mtable
URI myUri;
@@ -134,7 +135,7 @@ public class ViewFileSystem extends FileSystem {
* @param p path
* @return path-part of the Path p
*/
- private String getUriPath(final Path p) {
+ String getUriPath(final Path p) {
checkPath(p);
return makeAbsolute(p).toUri().getPath();
}
@@ -732,8 +733,8 @@ public class ViewFileSystem extends FileSystem {
MountPoint[] result = new MountPoint[mountPoints.size()];
for ( int i = 0; i < mountPoints.size(); ++i ) {
- result[i] = new MountPoint(new Path(mountPoints.get(i).src),
- mountPoints.get(i).target.targetDirLinkList);
+ result[i] = new MountPoint(new Path(mountPoints.get(i).src),
+ mountPoints.get(i).target.targetDirLinkList);
}
return result;
}
@@ -833,6 +834,21 @@ public class ViewFileSystem extends FileSystem {
return trashRoots;
}
+ @Override
+ public FsStatus getStatus() throws IOException {
+ return getStatus(null);
+ }
+
+ @Override
+ public FsStatus getStatus(Path p) throws IOException {
+ if (p == null) {
+ p = InodeTree.SlashPath;
+ }
+ InodeTree.ResolveResult<FileSystem> res = fsState.resolve(
+ getUriPath(p), true);
+ return res.targetFileSystem.getStatus(p);
+ }
+
/**
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd98a800/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java
new file mode 100644
index 0000000..c8a1d78
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.FsStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem.MountPoint;
+
+/**
+ * Utility APIs for ViewFileSystem.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class ViewFileSystemUtil {
+
+ private ViewFileSystemUtil() {
+ // Private Constructor
+ }
+
+ /**
+ * Check if the FileSystem is a ViewFileSystem.
+ *
+ * @param fileSystem
+ * @return true if the fileSystem is ViewFileSystem
+ */
+ public static boolean isViewFileSystem(final FileSystem fileSystem) {
+ return fileSystem.getScheme().equals(FsConstants.VIEWFS_SCHEME);
+ }
+
+ /**
+ * Get FsStatus for all ViewFsMountPoints matching path for the given
+ * ViewFileSystem.
+ *
+ * Say ViewFileSystem has following mount points configured
+ * (1) hdfs://NN0_host:port/sales mounted on /dept/sales
+ * (2) hdfs://NN1_host:port/marketing mounted on /dept/marketing
+ * (3) hdfs://NN2_host:port/eng_usa mounted on /dept/eng/usa
+ * (4) hdfs://NN3_host:port/eng_asia mounted on /dept/eng/asia
+ *
+ * For the above config, here is a sample list of paths and their matching
+ * mount points while getting FsStatus
+ *
+ * Path Description Matching MountPoint
+ *
+ * "/" Root ViewFileSystem lists all (1), (2), (3), (4)
+ * mount points.
+ *
+ * "/dept" Not a mount point, but a valid (1), (2), (3), (4)
+ * internal dir in the mount tree
+ * and resolved down to "/" path.
+ *
+ * "/dept/sales" Matches a mount point (1)
+ *
+ * "/dept/sales/india" Path is over a valid mount point (1)
+ * and resolved down to
+ * "/dept/sales"
+ *
+ * "/dept/eng" Not a mount point, but a valid (1), (2), (3), (4)
+ * internal dir in the mount tree
+ * and resolved down to "/" path.
+ *
+ * "/erp" Doesn't match or leads to or
+ * over any valid mount points None
+ *
+ *
+ * @param fileSystem - ViewFileSystem on which mount point exists
+ * @param path - URI for which FsStatus is requested
+ * @return Map of ViewFsMountPoint and FsStatus
+ */
+ public static Map<MountPoint, FsStatus> getStatus(
+ FileSystem fileSystem, Path path) throws IOException {
+ if (!isViewFileSystem(fileSystem)) {
+ throw new UnsupportedFileSystemException("FileSystem '"
+ + fileSystem.getUri() + "'is not a ViewFileSystem.");
+ }
+ ViewFileSystem viewFileSystem = (ViewFileSystem) fileSystem;
+ String viewFsUriPath = viewFileSystem.getUriPath(path);
+ boolean isPathOverMountPoint = false;
+ boolean isPathLeadingToMountPoint = false;
+ boolean isPathIncludesAllMountPoint = false;
+ Map<MountPoint, FsStatus> mountPointMap = new HashMap<>();
+ for (MountPoint mountPoint : viewFileSystem.getMountPoints()) {
+ String[] mountPointPathComponents = InodeTree.breakIntoPathComponents(
+ mountPoint.getMountedOnPath().toString());
+ String[] incomingPathComponents =
+ InodeTree.breakIntoPathComponents(viewFsUriPath);
+
+ int pathCompIndex;
+ for (pathCompIndex = 0; pathCompIndex < mountPointPathComponents.length &&
+ pathCompIndex < incomingPathComponents.length; pathCompIndex++) {
+ if (!mountPointPathComponents[pathCompIndex].equals(
+ incomingPathComponents[pathCompIndex])) {
+ break;
+ }
+ }
+
+ if (pathCompIndex >= mountPointPathComponents.length) {
+ // Patch matches or over a valid mount point
+ isPathOverMountPoint = true;
+ mountPointMap.clear();
+ updateMountPointFsStatus(viewFileSystem, mountPointMap, mountPoint,
+ new Path(viewFsUriPath));
+ break;
+ } else {
+ if (pathCompIndex > 1) {
+ // Path is in the mount tree
+ isPathLeadingToMountPoint = true;
+ } else if (incomingPathComponents.length <= 1) {
+ // Special case of "/" path
+ isPathIncludesAllMountPoint = true;
+ }
+ updateMountPointFsStatus(viewFileSystem, mountPointMap, mountPoint,
+ mountPoint.getMountedOnPath());
+ }
+ }
+
+ if (!isPathOverMountPoint && !isPathLeadingToMountPoint &&
+ !isPathIncludesAllMountPoint) {
+ throw new NotInMountpointException(path, "getStatus");
+ }
+ return mountPointMap;
+ }
+
+ /**
+ * Update FsStatus for the given the mount point.
+ *
+ * @param viewFileSystem
+ * @param mountPointMap
+ * @param mountPoint
+ * @param path
+ */
+ private static void updateMountPointFsStatus(
+ final ViewFileSystem viewFileSystem,
+ final Map<MountPoint, FsStatus> mountPointMap,
+ final MountPoint mountPoint, final Path path) throws IOException {
+ FsStatus fsStatus = viewFileSystem.getStatus(path);
+ mountPointMap.put(mountPoint, fsStatus);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd98a800/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
index be1caec..e5489cb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
@@ -86,19 +86,20 @@ public class TestViewfsFileStatus {
// .getFileChecksum with res.remainingPath and not with f
@Test
public void testGetFileChecksum() throws IOException {
+ final Path path = new Path("/tmp/someFile");
FileSystem mockFS = Mockito.mock(FileSystem.class);
InodeTree.ResolveResult<FileSystem> res =
new InodeTree.ResolveResult<FileSystem>(null, mockFS , null,
new Path("someFile"));
@SuppressWarnings("unchecked")
InodeTree<FileSystem> fsState = Mockito.mock(InodeTree.class);
- Mockito.when(fsState.resolve("/tmp/someFile", true)).thenReturn(res);
+ Mockito.when(fsState.resolve(path.toString(), true)).thenReturn(res);
ViewFileSystem vfs = Mockito.mock(ViewFileSystem.class);
vfs.fsState = fsState;
- Mockito.when(vfs.getFileChecksum(new Path("/tmp/someFile")))
- .thenCallRealMethod();
- vfs.getFileChecksum(new Path("/tmp/someFile"));
+ Mockito.when(vfs.getFileChecksum(path)).thenCallRealMethod();
+ Mockito.when(vfs.getUriPath(path)).thenCallRealMethod();
+ vfs.getFileChecksum(path);
Mockito.verify(mockFS).getFileChecksum(new Path("someFile"));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd98a800/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index 12c310b..407b885 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -24,8 +24,9 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.ArrayList;
+import java.util.Map;
+import java.util.Map.Entry;
-import com.google.common.base.Joiner;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStoragePolicySpi;
@@ -33,6 +34,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
@@ -153,8 +155,8 @@ abstract public class ViewFileSystemBaseTest {
ViewFileSystem viewfs = (ViewFileSystem) fsView;
MountPoint[] mountPoints = viewfs.getMountPoints();
for (MountPoint mountPoint : mountPoints) {
- LOG.info("MountPoint: " + mountPoint.getSrc() + " => "
- + Joiner.on(",").join(mountPoint.getTargets()));
+ LOG.info("MountPoint: " + mountPoint.getMountedOnPath() + " => "
+ + mountPoint.getTargetFileSystemURIs()[0]);
}
Assert.assertEquals(getExpectedMountPoints(), mountPoints.length);
}
@@ -1020,4 +1022,70 @@ abstract public class ViewFileSystemBaseTest {
// target mounted FileSystem.
Assert.assertTrue("", fsView.getTrashRoots(true).size() > 0);
}
+
+ @Test(expected = NotInMountpointException.class)
+ public void testViewFileSystemUtil() throws Exception {
+ Configuration newConf = new Configuration(conf);
+
+ FileSystem fileSystem = FileSystem.get(FsConstants.LOCAL_FS_URI,
+ newConf);
+ Assert.assertFalse("Unexpected FileSystem: " + fileSystem,
+ ViewFileSystemUtil.isViewFileSystem(fileSystem));
+
+ fileSystem = FileSystem.get(FsConstants.VIEWFS_URI,
+ newConf);
+ Assert.assertTrue("Unexpected FileSystem: " + fileSystem,
+ ViewFileSystemUtil.isViewFileSystem(fileSystem));
+
+ // Case 1: Verify FsStatus of root path returns all MountPoints status.
+ Map<MountPoint, FsStatus> mountPointFsStatusMap =
+ ViewFileSystemUtil.getStatus(fileSystem, InodeTree.SlashPath);
+ Assert.assertEquals(getExpectedMountPoints(), mountPointFsStatusMap.size());
+
+ // Case 2: Verify FsStatus of an internal dir returns all
+ // MountPoints status.
+ mountPointFsStatusMap =
+ ViewFileSystemUtil.getStatus(fileSystem, new Path("/internalDir"));
+ Assert.assertEquals(getExpectedMountPoints(), mountPointFsStatusMap.size());
+
+ // Case 3: Verify FsStatus of a matching MountPoint returns exactly
+ // the corresponding MountPoint status.
+ mountPointFsStatusMap =
+ ViewFileSystemUtil.getStatus(fileSystem, new Path("/user"));
+ Assert.assertEquals(1, mountPointFsStatusMap.size());
+ for (Entry<MountPoint, FsStatus> entry : mountPointFsStatusMap.entrySet()) {
+ Assert.assertEquals(entry.getKey().getMountedOnPath().toString(),
+ "/user");
+ }
+
+ // Case 4: Verify FsStatus of a path over a MountPoint returns the
+ // corresponding MountPoint status.
+ mountPointFsStatusMap =
+ ViewFileSystemUtil.getStatus(fileSystem, new Path("/user/cloud"));
+ Assert.assertEquals(1, mountPointFsStatusMap.size());
+ for (Entry<MountPoint, FsStatus> entry : mountPointFsStatusMap.entrySet()) {
+ Assert.assertEquals(entry.getKey().getMountedOnPath().toString(),
+ "/user");
+ }
+
+ // Case 5: Verify FsStatus of any level of an internal dir
+ // returns all MountPoints status.
+ mountPointFsStatusMap =
+ ViewFileSystemUtil.getStatus(fileSystem,
+ new Path("/internalDir/internalDir2"));
+ Assert.assertEquals(getExpectedMountPoints(), mountPointFsStatusMap.size());
+
+ // Case 6: Verify FsStatus of a MountPoint URI returns
+ // the MountPoint status.
+ mountPointFsStatusMap =
+ ViewFileSystemUtil.getStatus(fileSystem, new Path("viewfs:/user/"));
+ Assert.assertEquals(1, mountPointFsStatusMap.size());
+ for (Entry<MountPoint, FsStatus> entry : mountPointFsStatusMap.entrySet()) {
+ Assert.assertEquals(entry.getKey().getMountedOnPath().toString(),
+ "/user");
+ }
+
+ // Case 7: Verify FsStatus of a non MountPoint path throws exception
+ ViewFileSystemUtil.getStatus(fileSystem, new Path("/non-existing"));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd98a800/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
index df84b4c..0e420d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
@@ -194,4 +194,28 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
Assert.assertTrue("ViewFileSystem trash roots should include EZ zone trash",
(fsView.getTrashRoots(true).size() == 2));
}
+
+ @Test
+ public void testDf() throws Exception {
+ Configuration newConf = new Configuration(conf);
+
+ // Verify if DF on non viewfs produces output as before, that is
+ // without "Mounted On" header.
+ DFSTestUtil.FsShellRun("-df", 0, "Use%" + System.lineSeparator(), newConf);
+
+ // Setting the default Fs to viewfs
+ newConf.set("fs.default.name", "viewfs:///");
+
+ // Verify if DF on viewfs produces a new header "Mounted on"
+ DFSTestUtil.FsShellRun("-df /user", 0, "Mounted on", newConf);
+
+ DFSTestUtil.FsShellRun("-df viewfs:///user", 0, "/user", newConf);
+ DFSTestUtil.FsShellRun("-df /user3", 1, "/user3", newConf);
+ DFSTestUtil.FsShellRun("-df /user2/abc", 1, "No such file or directory",
+ newConf);
+ DFSTestUtil.FsShellRun("-df /user2/", 0, "/user2", newConf);
+ DFSTestUtil.FsShellRun("-df /internalDir", 0, "/internalDir", newConf);
+ DFSTestUtil.FsShellRun("-df /", 0, null, newConf);
+ DFSTestUtil.FsShellRun("-df", 0, null, newConf);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/48] hadoop git commit: YARN-5921. Incorrect synchronization in
RMContextImpl#setHAServiceState/getHAServiceState. Contributed by Varun
Saxena.
Posted by li...@apache.org.
YARN-5921. Incorrect synchronization in RMContextImpl#setHAServiceState/getHAServiceState. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f2960be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f2960be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f2960be
Branch: refs/heads/HADOOP-13345
Commit: 9f2960be19aa9c07c6b247bfec576c6b8e5f89bd
Parents: 07825f2
Author: Naganarasimha <na...@apache.org>
Authored: Sun Nov 27 23:31:27 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Sun Nov 27 23:31:27 2016 +0530
----------------------------------------------------------------------
.../TestTimelineReaderWebServicesHBaseStorage.java | 2 +-
.../timelineservice/storage/DataGeneratorForTest.java | 4 ++--
.../storage/TestHBaseTimelineStorageApps.java | 8 ++++----
.../storage/TestHBaseTimelineStorageEntities.java | 6 +++---
.../storage/flow/TestHBaseStorageFlowActivity.java | 6 +++---
.../storage/flow/TestHBaseStorageFlowRun.java | 14 +++++++-------
.../flow/TestHBaseStorageFlowRunCompaction.java | 2 +-
.../storage/HBaseTimelineWriterImpl.java | 5 -----
8 files changed, 21 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f2960be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index 2ed5d96..17c01b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -334,7 +334,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, flowVersion, runid, entity.getId(), te);
hbi.write(cluster, user, flow, flowVersion, runid, entity1.getId(), te1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f2960be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
index 0938e9e..5cbb781 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
@@ -182,7 +182,7 @@ final class DataGeneratorForTest {
te2.addEntity(entity2);
HBaseTimelineWriterImpl hbi = null;
try {
- hbi = new HBaseTimelineWriterImpl(util.getConfiguration());
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(util.getConfiguration());
hbi.start();
String cluster = "cluster1";
@@ -360,7 +360,7 @@ final class DataGeneratorForTest {
te.addEntity(entity2);
HBaseTimelineWriterImpl hbi = null;
try {
- hbi = new HBaseTimelineWriterImpl(util.getConfiguration());
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(util.getConfiguration());
hbi.start();
String cluster = "cluster1";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f2960be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
index e70198a..b3e5197 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
@@ -155,7 +155,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "cluster_check_null_application";
@@ -275,7 +275,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_write_app";
@@ -502,7 +502,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_events";
@@ -620,7 +620,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
// Writing application entity.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f2960be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
index 3076709..4b4c3e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
@@ -194,7 +194,7 @@ public class TestHBaseTimelineStorageEntities {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_write_entity";
@@ -391,7 +391,7 @@ public class TestHBaseTimelineStorageEntities {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_empty_eventkey";
@@ -496,7 +496,7 @@ public class TestHBaseTimelineStorageEntities {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "clus!ter_\ttest_ev ents";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f2960be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index 1906574..3ddb230 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -119,7 +119,7 @@ public class TestHBaseStorageFlowActivity {
.getEntityMinStartTime(minStartTs);
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -228,7 +228,7 @@ public class TestHBaseStorageFlowActivity {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_1111999999_1234";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -341,7 +341,7 @@ public class TestHBaseStorageFlowActivity {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_11888888888_1111";
hbi.write(cluster, user, flow, flowVersion1, runid1, appName, te);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f2960be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index 74b9e50..377611f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -184,7 +184,7 @@ public class TestHBaseStorageFlowRun {
.getEntityMinStartTime(minStartTs);
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -290,7 +290,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_11111111111111_1111";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -559,7 +559,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_11111111111111_1111";
hbi.write(cluster, user, flow, flowVersion, 1002345678919L, appName, te);
@@ -644,7 +644,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_11111111111111_1111";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -737,7 +737,7 @@ public class TestHBaseStorageFlowRun {
TimelineEntity entityApp1 = null;
TimelineEntity entityApp2 = null;
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
for (int i = start; i < count; i++) {
@@ -826,7 +826,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L,
"application_11111111111111_1111", te);
@@ -908,7 +908,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L,
"application_11111111111111_1111", te);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f2960be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
index 3094088..756d57b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
@@ -287,7 +287,7 @@ public class TestHBaseStorageFlowRunCompaction {
TimelineEntities te1 = null;
TimelineEntity entityApp1 = null;
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
// now insert count * ( 100 + 100) metrics
// each call to getEntityMetricsApp1 brings back 100 values
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f2960be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 3511a2f..b94b85f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -101,11 +101,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
super(HBaseTimelineWriterImpl.class.getName());
}
- public HBaseTimelineWriterImpl(Configuration conf) throws IOException {
- super(conf.get("yarn.application.id",
- HBaseTimelineWriterImpl.class.getName()));
- }
-
/**
* initializes the hbase connection to write to the entity table.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/48] hadoop git commit: HDFS-11175. Document uppercase key names
are not supported in TransparentEncryption.md. Contributed by Yiqun Lin.
Posted by li...@apache.org.
HDFS-11175. Document uppercase key names are not supported in TransparentEncryption.md. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/636f7547
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/636f7547
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/636f7547
Branch: refs/heads/HADOOP-13345
Commit: 636f7547b7067c315cf5448d77e49a8fe55651b3
Parents: 01665e4
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Nov 25 17:48:54 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Nov 25 17:48:54 2016 +0900
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/TransparentEncryption.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/636f7547/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
index b82b400..1a403c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
@@ -173,7 +173,7 @@ Create a new encryption zone.
| | |
|:---- |:---- |
| *path* | The path of the encryption zone to create. It must be an empty directory. A trash directory is provisioned under this path.|
-| *keyName* | Name of the key to use for the encryption zone. |
+| *keyName* | Name of the key to use for the encryption zone. Uppercase key names are unsupported. |
### <a name="listZones"></a>listZones
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/48] hadoop git commit: HDFS-5517. Lower the default maximum
number of blocks per file. Contributed by Aaron T. Myers.
Posted by li...@apache.org.
HDFS-5517. Lower the default maximum number of blocks per file. Contributed by Aaron T. Myers.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09451252
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09451252
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09451252
Branch: refs/heads/HADOOP-13345
Commit: 09451252fae90a3ec192b8d7f0c49508df65e8c5
Parents: 67d9f28
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Nov 29 00:12:35 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Nov 29 00:12:35 2016 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09451252/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index d7d3c9d..df21857 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -399,7 +399,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_MIN_BLOCK_SIZE_KEY = "dfs.namenode.fs-limits.min-block-size";
public static final long DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT = 1024*1024;
public static final String DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY = "dfs.namenode.fs-limits.max-blocks-per-file";
- public static final long DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT = 1024*1024;
+ public static final long DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT = 10*1000;
public static final String DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY = "dfs.namenode.fs-limits.max-xattrs-per-inode";
public static final int DFS_NAMENODE_MAX_XATTRS_PER_INODE_DEFAULT = 32;
public static final String DFS_NAMENODE_MAX_XATTR_SIZE_KEY = "dfs.namenode.fs-limits.max-xattr-size";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09451252/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index c9d74bb..9fce84f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -372,7 +372,7 @@
<property>
<name>dfs.namenode.fs-limits.max-blocks-per-file</name>
- <value>1048576</value>
+ <value>10000</value>
<description>Maximum number of blocks per file, enforced by the Namenode on
write. This prevents the creation of extremely large files which can
degrade performance.</description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/48] hadoop git commit: HDFS-11174. Wrong HttpFS test command in
doc. Contributed by John Zhuge.
Posted by li...@apache.org.
HDFS-11174. Wrong HttpFS test command in doc. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb1b5e08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb1b5e08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb1b5e08
Branch: refs/heads/HADOOP-13345
Commit: bb1b5e083b6f32646412b292d061ce5721f5dbe1
Parents: 636f754
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Nov 25 21:28:29 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Nov 25 21:28:29 2016 +0900
----------------------------------------------------------------------
.../hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb1b5e08/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
index 6a03a45..a4f02e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
@@ -64,12 +64,8 @@ NOTE: Invoking the script without any parameters list all possible parameters (s
Test HttpFS is working
----------------------
- ~ $ curl -i "http://<HTTPFSHOSTNAME>:14000?user.name=babu&op=homedir"
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
-
- {"homeDir":"http:\/\/<HTTPFS_HOST>:14000\/user\/babu"}
+ $ curl -sS 'http://<HTTPFSHOSTNAME>:14000/webhdfs/v1?op=gethomedirectory&user.name=hdfs'
+ {"Path":"\/user\/hdfs"}
Embedded Tomcat Configuration
-----------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/48] hadoop git commit: HADOOP-13605. Clean up FileSystem
javadocs, logging;
improve diagnostics on FS load. Contributed by Steve Loughran
Posted by li...@apache.org.
HADOOP-13605. Clean up FileSystem javadocs, logging; improve diagnostics on FS load. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/860d49aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/860d49aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/860d49aa
Branch: refs/heads/HADOOP-13345
Commit: 860d49aa6ae7790d970d7f2322fed890b0e5cda2
Parents: de48949
Author: Mingliang Liu <li...@apache.org>
Authored: Wed Nov 23 16:32:42 2016 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed Nov 23 16:42:27 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 1449 +++++++++++-------
.../src/site/markdown/filesystem/filesystem.md | 2 +-
.../org/apache/hadoop/fs/TestDefaultUri.java | 40 +-
.../apache/hadoop/fs/TestFileSystemCaching.java | 8 +-
4 files changed, 920 insertions(+), 579 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/48] hadoop git commit: YARN-4330. MiniYARNCluster is showing
multiple Failed to instantiate default resource calculator warning messages.
Contributed by Varun Saxena
Posted by li...@apache.org.
YARN-4330. MiniYARNCluster is showing multiple Failed to instantiate default resource calculator warning messages. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3541ed80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3541ed80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3541ed80
Branch: refs/heads/HADOOP-13345
Commit: 3541ed80685f25486f33ef0f553854ccbdeb51d4
Parents: 4667564
Author: Naganarasimha <na...@apache.org>
Authored: Wed Nov 23 14:12:23 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Wed Nov 23 14:12:23 2016 +0530
----------------------------------------------------------------------
.../yarn/util/ResourceCalculatorPlugin.java | 3 +
.../src/main/resources/yarn-default.xml | 6 +-
.../nodemanager/NodeResourceMonitorImpl.java | 5 +
.../monitor/ContainersMonitorImpl.java | 3 +-
.../util/NodeManagerHardwareUtils.java | 160 +++++++++++--------
.../hadoop/yarn/server/MiniYARNCluster.java | 7 +-
6 files changed, 110 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3541ed80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
index e7e4c8a..fd63d98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
@@ -190,6 +190,9 @@ public class ResourceCalculatorPlugin extends Configured {
}
try {
return new ResourceCalculatorPlugin();
+ } catch (UnsupportedOperationException ue) {
+ LOG.warn("Failed to instantiate default resource calculator. "
+ + ue.getMessage());
} catch (Throwable t) {
LOG.warn(t + ": Failed to instantiate default resource calculator.", t);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3541ed80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index c436289..47d12d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1344,7 +1344,8 @@
</property>
<property>
- <description>How often to monitor the node and the containers.</description>
+ <description>How often to monitor the node and the containers.
+ If 0 or negative, monitoring is disabled.</description>
<name>yarn.nodemanager.resource-monitor.interval-ms</name>
<value>3000</value>
</property>
@@ -1362,7 +1363,8 @@
<property>
<description>How often to monitor containers. If not set, the value for
- yarn.nodemanager.resource-monitor.interval-ms will be used.</description>
+ yarn.nodemanager.resource-monitor.interval-ms will be used.
+ If 0 or negative, container monitoring is disabled.</description>
<name>yarn.nodemanager.container-monitor.interval-ms</name>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3541ed80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
index 8ee27de..e1116da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
@@ -78,6 +78,11 @@ public class NodeResourceMonitorImpl extends AbstractService implements
* @return <em>true</em> if we can monitor the node resource utilization.
*/
private boolean isEnabled() {
+ if (this.monitoringInterval <= 0) {
+ LOG.info("Node Resource monitoring interval is <=0. "
+ + this.getClass().getName() + " is disabled.");
+ return false;
+ }
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
+ this.getClass().getName() + " is disabled.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3541ed80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 1914b42..dda55d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -172,7 +172,8 @@ public class ContainersMonitorImpl extends AbstractService implements
LOG.info("Physical memory check enabled: " + pmemCheckEnabled);
LOG.info("Virtual memory check enabled: " + vmemCheckEnabled);
- containersMonitorEnabled = isContainerMonitorEnabled();
+ containersMonitorEnabled =
+ isContainerMonitorEnabled() && monitoringInterval > 0;
LOG.info("ContainersMonitor enabled: " + containersMonitorEnabled);
nodeCpuPercentageForYARN =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3541ed80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
index f3c95d3..2726a41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
@@ -37,6 +37,12 @@ public class NodeManagerHardwareUtils {
private static final Log LOG = LogFactory
.getLog(NodeManagerHardwareUtils.class);
+ private static boolean isHardwareDetectionEnabled(Configuration conf) {
+ return conf.getBoolean(
+ YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION,
+ YarnConfiguration.DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION);
+ }
+
/**
*
* Returns the number of CPUs on the node. This value depends on the
@@ -138,6 +144,15 @@ public class NodeManagerHardwareUtils {
return nodeCpuPercentage;
}
+ private static int getConfiguredVCores(Configuration conf) {
+ int cores = conf.getInt(YarnConfiguration.NM_VCORES,
+ YarnConfiguration.DEFAULT_NM_VCORES);
+ if (cores == -1) {
+ cores = YarnConfiguration.DEFAULT_NM_VCORES;
+ }
+ return cores;
+ }
+
/**
* Function to return the number of vcores on the system that can be used for
* YARN containers. If a number is specified in the configuration file, then
@@ -154,11 +169,16 @@ public class NodeManagerHardwareUtils {
*
*/
public static int getVCores(Configuration conf) {
+ if (!isHardwareDetectionEnabled(conf)) {
+ return getConfiguredVCores(conf);
+ }
// is this os for which we can determine cores?
ResourceCalculatorPlugin plugin =
ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf);
-
- return NodeManagerHardwareUtils.getVCores(plugin, conf);
+ if (plugin == null) {
+ return getConfiguredVCores(conf);
+ }
+ return getVCoresInternal(plugin, conf);
}
/**
@@ -180,43 +200,35 @@ public class NodeManagerHardwareUtils {
*/
public static int getVCores(ResourceCalculatorPlugin plugin,
Configuration conf) {
+ if (!isHardwareDetectionEnabled(conf) || plugin == null) {
+ return getConfiguredVCores(conf);
+ }
+ return getVCoresInternal(plugin, conf);
+ }
- int cores;
- boolean hardwareDetectionEnabled =
- conf.getBoolean(
- YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION,
- YarnConfiguration.DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION);
-
+ private static int getVCoresInternal(ResourceCalculatorPlugin plugin,
+ Configuration conf) {
String message;
- if (!hardwareDetectionEnabled || plugin == null) {
- cores =
- conf.getInt(YarnConfiguration.NM_VCORES,
- YarnConfiguration.DEFAULT_NM_VCORES);
- if (cores == -1) {
- cores = YarnConfiguration.DEFAULT_NM_VCORES;
- }
- } else {
- cores = conf.getInt(YarnConfiguration.NM_VCORES, -1);
- if (cores == -1) {
- float physicalCores =
- NodeManagerHardwareUtils.getContainersCPUs(plugin, conf);
- float multiplier =
- conf.getFloat(YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER,
- YarnConfiguration.DEFAULT_NM_PCORES_VCORES_MULTIPLIER);
- if (multiplier > 0) {
- float tmp = physicalCores * multiplier;
- if (tmp > 0 && tmp < 1) {
- // on a single core machine - tmp can be between 0 and 1
- cores = 1;
- } else {
- cores = (int) tmp;
- }
+ int cores = conf.getInt(YarnConfiguration.NM_VCORES, -1);
+ if (cores == -1) {
+ float physicalCores =
+ NodeManagerHardwareUtils.getContainersCPUs(plugin, conf);
+ float multiplier =
+ conf.getFloat(YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER,
+ YarnConfiguration.DEFAULT_NM_PCORES_VCORES_MULTIPLIER);
+ if (multiplier > 0) {
+ float tmp = physicalCores * multiplier;
+ if (tmp > 0 && tmp < 1) {
+ // on a single core machine - tmp can be between 0 and 1
+ cores = 1;
} else {
- message = "Illegal value for "
- + YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER
- + ". Value must be greater than 0.";
- throw new IllegalArgumentException(message);
+ cores = (int) tmp;
}
+ } else {
+ message = "Illegal value for "
+ + YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER
+ + ". Value must be greater than 0.";
+ throw new IllegalArgumentException(message);
}
}
if(cores <= 0) {
@@ -228,6 +240,15 @@ public class NodeManagerHardwareUtils {
return cores;
}
+ private static int getConfiguredMemoryMB(Configuration conf) {
+ int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB,
+ YarnConfiguration.DEFAULT_NM_PMEM_MB);
+ if (memoryMb == -1) {
+ memoryMb = YarnConfiguration.DEFAULT_NM_PMEM_MB;
+ }
+ return memoryMb;
+ }
+
/**
* Function to return how much memory we should set aside for YARN containers.
* If a number is specified in the configuration file, then that number is
@@ -244,8 +265,15 @@ public class NodeManagerHardwareUtils {
* @return the amount of memory that will be used for YARN containers in MB.
*/
public static int getContainerMemoryMB(Configuration conf) {
- return NodeManagerHardwareUtils.getContainerMemoryMB(
- ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf), conf);
+ if (!isHardwareDetectionEnabled(conf)) {
+ return getConfiguredMemoryMB(conf);
+ }
+ ResourceCalculatorPlugin plugin =
+ ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf);
+ if (plugin == null) {
+ return getConfiguredMemoryMB(conf);
+ }
+ return getContainerMemoryMBInternal(plugin, conf);
}
/**
@@ -267,41 +295,35 @@ public class NodeManagerHardwareUtils {
*/
public static int getContainerMemoryMB(ResourceCalculatorPlugin plugin,
Configuration conf) {
+ if (!isHardwareDetectionEnabled(conf) || plugin == null) {
+ return getConfiguredMemoryMB(conf);
+ }
+ return getContainerMemoryMBInternal(plugin, conf);
+ }
- int memoryMb;
- boolean hardwareDetectionEnabled = conf.getBoolean(
- YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION,
- YarnConfiguration.DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION);
-
- if (!hardwareDetectionEnabled || plugin == null) {
- memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB,
- YarnConfiguration.DEFAULT_NM_PMEM_MB);
- if (memoryMb == -1) {
- memoryMb = YarnConfiguration.DEFAULT_NM_PMEM_MB;
+ private static int getContainerMemoryMBInternal(ResourceCalculatorPlugin plugin,
+ Configuration conf) {
+ int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, -1);
+ if (memoryMb == -1) {
+ int physicalMemoryMB =
+ (int) (plugin.getPhysicalMemorySize() / (1024 * 1024));
+ int hadoopHeapSizeMB =
+ (int) (Runtime.getRuntime().maxMemory() / (1024 * 1024));
+ int containerPhysicalMemoryMB =
+ (int) (0.8f * (physicalMemoryMB - (2 * hadoopHeapSizeMB)));
+ int reservedMemoryMB =
+ conf.getInt(YarnConfiguration.NM_SYSTEM_RESERVED_PMEM_MB, -1);
+ if (reservedMemoryMB != -1) {
+ containerPhysicalMemoryMB = physicalMemoryMB - reservedMemoryMB;
}
- } else {
- memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, -1);
- if (memoryMb == -1) {
- int physicalMemoryMB =
- (int) (plugin.getPhysicalMemorySize() / (1024 * 1024));
- int hadoopHeapSizeMB =
- (int) (Runtime.getRuntime().maxMemory() / (1024 * 1024));
- int containerPhysicalMemoryMB =
- (int) (0.8f * (physicalMemoryMB - (2 * hadoopHeapSizeMB)));
- int reservedMemoryMB =
- conf.getInt(YarnConfiguration.NM_SYSTEM_RESERVED_PMEM_MB, -1);
- if (reservedMemoryMB != -1) {
- containerPhysicalMemoryMB = physicalMemoryMB - reservedMemoryMB;
- }
- if(containerPhysicalMemoryMB <= 0) {
- LOG.error("Calculated memory for YARN containers is too low."
- + " Node memory is " + physicalMemoryMB
- + " MB, system reserved memory is "
- + reservedMemoryMB + " MB.");
- }
- containerPhysicalMemoryMB = Math.max(containerPhysicalMemoryMB, 0);
- memoryMb = containerPhysicalMemoryMB;
+ if(containerPhysicalMemoryMB <= 0) {
+ LOG.error("Calculated memory for YARN containers is too low."
+ + " Node memory is " + physicalMemoryMB
+ + " MB, system reserved memory is "
+ + reservedMemoryMB + " MB.");
}
+ containerPhysicalMemoryMB = Math.max(containerPhysicalMemoryMB, 0);
+ memoryMb = containerPhysicalMemoryMB;
}
if(memoryMb <= 0) {
String message = "Illegal value for " + YarnConfiguration.NM_PMEM_MB
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3541ed80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 89ebeb1..4f484ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -552,13 +552,16 @@ public class MiniYARNCluster extends CompositeService {
.setNMWebAppHostNameAndPort(config,
MiniYARNCluster.getHostname(), 0);
+ config.setBoolean(
+ YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, false);
// Disable resource checks by default
if (!config.getBoolean(
YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
YarnConfiguration.
DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
- config.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
- config.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
+ config.setBoolean(
+ YarnConfiguration.NM_CONTAINER_MONITOR_ENABLED, false);
+ config.setLong(YarnConfiguration.NM_RESOURCE_MON_INTERVAL_MS, 0);
}
LOG.info("Starting NM: " + index);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/48] hadoop git commit: YARN-4752. Improved preemption in
FairScheduler. (kasha)
Posted by li...@apache.org.
YARN-4752. Improved preemption in FairScheduler. (kasha)
Contains:
YARN-5605. Preempt containers (all on one node) to meet the requirement of starved applications
YARN-5821. Drop left-over preemption-related code and clean up method visibilities in the Schedulable hierarchy
YARN-5783. Verify identification of starved applications.
YARN-5819. Verify fairshare and minshare preemption
YARN-5885. Cleanup YARN-4752 branch for merge
Change-Id: Iee0962377d019dd64dc69a020725d2eaf360858c
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10468529
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10468529
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10468529
Branch: refs/heads/HADOOP-13345
Commit: 10468529a9b858bd945e7ecb063c9c1438efa474
Parents: c7a5f29
Author: Daniel Templeton <te...@apache.org>
Authored: Thu Sep 22 14:08:15 2016 -0700
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Wed Nov 23 19:48:59 2016 -1000
----------------------------------------------------------------------
.../hadoop/yarn/util/resource/Resources.java | 4 +
.../scheduler/AppSchedulingInfo.java | 17 +
.../scheduler/SchedulerApplicationAttempt.java | 16 +
.../scheduler/common/fica/FiCaSchedulerApp.java | 16 +
.../scheduler/fair/FSAppAttempt.java | 242 ++-
.../scheduler/fair/FSContext.java | 54 +
.../scheduler/fair/FSLeafQueue.java | 283 ++--
.../scheduler/fair/FSParentQueue.java | 52 +-
.../scheduler/fair/FSPreemptionThread.java | 188 +++
.../resourcemanager/scheduler/fair/FSQueue.java | 30 +-
.../scheduler/fair/FSSchedulerNode.java | 36 +-
.../scheduler/fair/FSStarvedApps.java | 85 +
.../scheduler/fair/FairScheduler.java | 269 +--
.../scheduler/fair/Schedulable.java | 29 +-
.../scheduler/fair/FairSchedulerTestBase.java | 39 +-
.../fair/FairSchedulerWithMockPreemption.java | 58 +
.../scheduler/fair/FakeSchedulable.java | 5 -
.../scheduler/fair/TestFSAppStarvation.java | 256 +++
.../scheduler/fair/TestFSLeafQueue.java | 165 +-
.../fair/TestFairSchedulerPreemption.java | 1540 ++----------------
.../fair/TestQueueManagerRealScheduler.java | 128 ++
.../scheduler/fair/TestSchedulingPolicy.java | 5 -
22 files changed, 1444 insertions(+), 2073 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 760b0ea..462e02a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -143,6 +143,10 @@ public class Resources {
public static Resource none() {
return NONE;
}
+
+ public static boolean isNone(Resource other) {
+ return NONE.equals(other);
+ }
public static Resource unbounded() {
return UNBOUNDED;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 80811b1..feb20ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -624,6 +624,23 @@ public class AppSchedulingInfo {
}
/**
+ * Method to return the next resource request to be serviced.
+ *
+ * In the initial implementation, we just pick any {@link ResourceRequest}
+ * corresponding to the highest priority.
+ *
+ * @return next {@link ResourceRequest} to allocate resources for.
+ */
+ @Unstable
+ public synchronized ResourceRequest getNextResourceRequest() {
+ for (ResourceRequest rr:
+ resourceRequestMap.get(schedulerKeys.firstKey()).values()) {
+ return rr;
+ }
+ return null;
+ }
+
+ /**
* Returns if the place (node/rack today) is either blacklisted by the
* application (user) or the system
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index bc52816..b3ef471 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -1253,6 +1253,22 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
unconfirmedAllocatedVcores.addAndGet(-res.getVirtualCores());
}
+ @Override
+ public int hashCode() {
+ return getApplicationAttemptId().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (! (o instanceof SchedulerApplicationAttempt)) {
+ return false;
+ }
+
+ SchedulerApplicationAttempt other = (SchedulerApplicationAttempt) o;
+ return (this == other ||
+ this.getApplicationAttemptId().equals(other.getApplicationAttemptId()));
+ }
+
/**
* Different state for Application Master, user can see this state from web UI
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index f076e4f..498f34f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -1101,4 +1101,20 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
}
}
}
+
+ /*
+ * Overriding to appease findbugs
+ */
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ /*
+ * Overriding to appease findbugs
+ */
+ @Override
+ public boolean equals(Object o) {
+ return super.equals(o);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index df20117..39f4a3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -18,18 +18,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import java.io.Serializable;
import java.text.DecimalFormat;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
-import java.util.Comparator;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -53,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFini
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
@@ -78,10 +78,17 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
private ResourceWeights resourceWeights;
private Resource demand = Resources.createResource(0);
private FairScheduler scheduler;
+ private FSQueue fsQueue;
private Resource fairShare = Resources.createResource(0, 0);
- private Resource preemptedResources = Resources.createResource(0);
- private RMContainerComparator comparator = new RMContainerComparator();
- private final Map<RMContainer, Long> preemptionMap = new HashMap<RMContainer, Long>();
+
+ // Preemption related variables
+ private final Resource preemptedResources = Resources.clone(Resources.none());
+ private final Set<RMContainer> containersToPreempt = new HashSet<>();
+ private Resource fairshareStarvation = Resources.none();
+ private long lastTimeAtFairShare;
+
+ // minShareStarvation attributed to this application by the leaf queue
+ private Resource minshareStarvation = Resources.none();
// Used to record node reservation by an app.
// Key = RackName, Value = Set of Nodes reserved by app on rack
@@ -107,12 +114,14 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
this.scheduler = scheduler;
+ this.fsQueue = queue;
this.startTime = scheduler.getClock().getTime();
+ this.lastTimeAtFairShare = this.startTime;
this.appPriority = Priority.newInstance(1);
this.resourceWeights = new ResourceWeights();
}
- public ResourceWeights getResourceWeights() {
+ ResourceWeights getResourceWeights() {
return resourceWeights;
}
@@ -123,7 +132,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
return queue.getMetrics();
}
- public void containerCompleted(RMContainer rmContainer,
+ void containerCompleted(RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event) {
try {
writeLock.lock();
@@ -143,6 +152,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Remove from the list of containers
liveContainers.remove(rmContainer.getContainerId());
+ untrackContainerForPreemption(rmContainer);
Resource containerResource = rmContainer.getContainer().getResource();
RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER,
@@ -152,9 +162,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
queue.getMetrics().releaseResources(getUser(), 1, containerResource);
this.attemptResourceUsage.decUsed(containerResource);
- // remove from preemption map if it is completed
- preemptionMap.remove(rmContainer);
-
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
} finally {
@@ -484,7 +491,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @param schedulerKey Scheduler Key
* @param level NodeType
*/
- public void resetAllowedLocalityLevel(
+ void resetAllowedLocalityLevel(
SchedulerRequestKey schedulerKey, NodeType level) {
NodeType old;
try {
@@ -498,57 +505,113 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ " priority " + schedulerKey.getPriority());
}
- // related methods
- public void addPreemption(RMContainer container, long time) {
- assert preemptionMap.get(container) == null;
- try {
- writeLock.lock();
- preemptionMap.put(container, time);
- Resources.addTo(preemptedResources, container.getAllocatedResource());
- } finally {
- writeLock.unlock();
- }
+ @Override
+ public FSLeafQueue getQueue() {
+ Queue queue = super.getQueue();
+ assert queue instanceof FSLeafQueue;
+ return (FSLeafQueue) queue;
}
- public Long getContainerPreemptionTime(RMContainer container) {
- return preemptionMap.get(container);
- }
+ // Preemption related methods
- public Set<RMContainer> getPreemptionContainers() {
- return preemptionMap.keySet();
+ /**
+ * Get overall starvation - fairshare and attributed minshare.
+ *
+ * @return total starvation attributed to this application
+ */
+ Resource getStarvation() {
+ return Resources.add(fairshareStarvation, minshareStarvation);
}
-
- @Override
- public FSLeafQueue getQueue() {
- return (FSLeafQueue)super.getQueue();
+
+ /**
+ * Set the minshare attributed to this application. To be called only from
+ * {@link FSLeafQueue#updateStarvedApps}.
+ *
+ * @param starvation minshare starvation attributed to this app
+ */
+ void setMinshareStarvation(Resource starvation) {
+ this.minshareStarvation = starvation;
}
- public Resource getPreemptedResources() {
- return preemptedResources;
+ /**
+ * Reset the minshare starvation attributed to this application. To be
+ * called only from {@link FSLeafQueue#updateStarvedApps}
+ */
+ void resetMinshareStarvation() {
+ this.minshareStarvation = Resources.none();
}
- public void resetPreemptedResources() {
- preemptedResources = Resources.createResource(0);
- for (RMContainer container : getPreemptionContainers()) {
+ void trackContainerForPreemption(RMContainer container) {
+ containersToPreempt.add(container);
+ synchronized (preemptedResources) {
Resources.addTo(preemptedResources, container.getAllocatedResource());
}
}
- public void clearPreemptedResources() {
- preemptedResources.setMemorySize(0);
- preemptedResources.setVirtualCores(0);
+ private void untrackContainerForPreemption(RMContainer container) {
+ synchronized (preemptedResources) {
+ Resources.subtractFrom(preemptedResources,
+ container.getAllocatedResource());
+ }
+ containersToPreempt.remove(container);
+ }
+
+ Set<RMContainer> getPreemptionContainers() {
+ return containersToPreempt;
+ }
+
+ private Resource getPreemptedResources() {
+ synchronized (preemptedResources) {
+ return preemptedResources;
+ }
+ }
+
+ boolean canContainerBePreempted(RMContainer container) {
+ // Sanity check that the app owns this container
+ if (!getLiveContainersMap().containsKey(container.getContainerId()) &&
+ !newlyAllocatedContainers.contains(container)) {
+ LOG.error("Looking to preempt container " + container +
+ ". Container does not belong to app " + getApplicationId());
+ return false;
+ }
+
+ if (containersToPreempt.contains(container)) {
+ // The container is already under consideration for preemption
+ return false;
+ }
+
+ // Check if any of the parent queues are not preemptable
+ // TODO (YARN-5831): Propagate the "preemptable" flag all the way down to
+ // the app to avoid recursing up every time.
+ for (FSQueue q = getQueue();
+ !q.getQueueName().equals("root");
+ q = q.getParent()) {
+ if (!q.isPreemptable()) {
+ return false;
+ }
+ }
+
+ // Check if the app's allocation will be over its fairshare even
+ // after preempting this container
+ Resource currentUsage = getResourceUsage();
+ Resource fairshare = getFairShare();
+ Resource overFairShareBy = Resources.subtract(currentUsage, fairshare);
+
+ return (Resources.fitsIn(container.getAllocatedResource(),
+ overFairShareBy));
}
/**
* Create and return a container object reflecting an allocation for the
- * given appliction on the given node with the given capability and
+ * given application on the given node with the given capability and
* priority.
+ *
* @param node Node
* @param capability Capability
* @param schedulerKey Scheduler Key
* @return Container
*/
- public Container createContainer(FSSchedulerNode node, Resource capability,
+ private Container createContainer(FSSchedulerNode node, Resource capability,
SchedulerRequestKey schedulerKey) {
NodeId nodeId = node.getRMNode().getNodeID();
@@ -556,12 +619,10 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
getApplicationAttemptId(), getNewContainerId());
// Create the container
- Container container = BuilderUtils.newContainer(containerId, nodeId,
+ return BuilderUtils.newContainer(containerId, nodeId,
node.getRMNode().getHttpAddress(), capability,
schedulerKey.getPriority(), null,
schedulerKey.getAllocationRequestId());
-
- return container;
}
/**
@@ -816,7 +877,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
Collection<SchedulerRequestKey> keysToTry = (reserved) ?
- Arrays.asList(node.getReservedContainer().getReservedSchedulerKey()) :
+ Collections.singletonList(
+ node.getReservedContainer().getReservedSchedulerKey()) :
getSchedulerKeys();
// For each priority, see if we can schedule a node local, rack local
@@ -974,7 +1036,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* Node that the application has an existing reservation on
* @return whether the reservation on the given node is valid.
*/
- public boolean assignReservedContainer(FSSchedulerNode node) {
+ boolean assignReservedContainer(FSSchedulerNode node) {
RMContainer rmContainer = node.getReservedContainer();
SchedulerRequestKey reservedSchedulerKey =
rmContainer.getReservedSchedulerKey();
@@ -1003,17 +1065,43 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
return true;
}
- static class RMContainerComparator implements Comparator<RMContainer>,
- Serializable {
- @Override
- public int compare(RMContainer c1, RMContainer c2) {
- int ret = c1.getContainer().getPriority().compareTo(
- c2.getContainer().getPriority());
- if (ret == 0) {
- return c2.getContainerId().compareTo(c1.getContainerId());
- }
- return ret;
+ /**
+ * Helper method that computes the extent of fairshare fairshareStarvation.
+ */
+ Resource fairShareStarvation() {
+ Resource threshold = Resources.multiply(
+ getFairShare(), fsQueue.getFairSharePreemptionThreshold());
+ Resource starvation = Resources.subtractFrom(threshold, getResourceUsage());
+
+ long now = scheduler.getClock().getTime();
+ boolean starved = Resources.greaterThan(
+ fsQueue.getPolicy().getResourceCalculator(),
+ scheduler.getClusterResource(), starvation, Resources.none());
+
+ if (!starved) {
+ lastTimeAtFairShare = now;
}
+
+ if (starved &&
+ (now - lastTimeAtFairShare > fsQueue.getFairSharePreemptionTimeout())) {
+ this.fairshareStarvation = starvation;
+ } else {
+ this.fairshareStarvation = Resources.none();
+ }
+ return this.fairshareStarvation;
+ }
+
+ ResourceRequest getNextResourceRequest() {
+ return appSchedulingInfo.getNextResourceRequest();
+ }
+
+ /**
+ * Helper method that captures if this app is identified to be starved.
+ * @return true if the app is starved for fairshare, false otherwise
+ */
+ @VisibleForTesting
+ boolean isStarvedForFairShare() {
+ return !Resources.isNone(fairshareStarvation);
}
/* Schedulable methods implementation */
@@ -1045,14 +1133,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
@Override
public Resource getResourceUsage() {
- // Here the getPreemptedResources() always return zero, except in
- // a preemption round
- // In the common case where preempted resource is zero, return the
- // current consumption Resource object directly without calling
- // Resources.subtract which creates a new Resource object for each call.
- return getPreemptedResources().equals(Resources.none()) ?
- getCurrentConsumption() :
- Resources.subtract(getCurrentConsumption(), getPreemptedResources());
+ /*
+ * getResourcesToPreempt() returns zero, except when there are containers
+ * to preempt. Avoid creating an object in the common case.
+ */
+ return getPreemptedResources().equals(Resources.none())
+ ? getCurrentConsumption()
+ : Resources.subtract(getCurrentConsumption(), getPreemptedResources());
}
@Override
@@ -1131,24 +1218,19 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
diagnosticMessageBldr.toString());
}
- /**
- * Preempt a running container according to the priority
+ /*
+ * Overriding to appease findbugs
*/
@Override
- public RMContainer preemptContainer() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("App " + getName() + " is going to preempt a running " +
- "container");
- }
+ public int hashCode() {
+ return super.hashCode();
+ }
- RMContainer toBePreempted = null;
- for (RMContainer container : getLiveContainers()) {
- if (!getPreemptionContainers().contains(container) &&
- (toBePreempted == null ||
- comparator.compare(toBePreempted, container) > 0)) {
- toBePreempted = container;
- }
- }
- return toBePreempted;
+ /*
+ * Overriding to appease findbugs
+ */
+ @Override
+ public boolean equals(Object o) {
+ return super.equals(o);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
new file mode 100644
index 0000000..56bc99c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+/**
+ * Helper class that holds basic information to be passed around
+ * FairScheduler classes. Think of this as a glorified map that holds key
+ * information about the scheduler.
+ */
+public class FSContext {
+ // Preemption-related info
+ private boolean preemptionEnabled = false;
+ private float preemptionUtilizationThreshold;
+ private FSStarvedApps starvedApps;
+
+ public boolean isPreemptionEnabled() {
+ return preemptionEnabled;
+ }
+
+ public void setPreemptionEnabled() {
+ this.preemptionEnabled = true;
+ if (starvedApps == null) {
+ starvedApps = new FSStarvedApps();
+ }
+ }
+
+ public FSStarvedApps getStarvedApps() {
+ return starvedApps;
+ }
+
+ public float getPreemptionUtilizationThreshold() {
+ return preemptionUtilizationThreshold;
+ }
+
+ public void setPreemptionUtilizationThreshold(
+ float preemptionUtilizationThreshold) {
+ this.preemptionUtilizationThreshold = preemptionUtilizationThreshold;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index c393759..343e9c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
import java.util.List;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
@@ -45,16 +44,20 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.util.resource.Resources;
+import static org.apache.hadoop.yarn.util.resource.Resources.none;
+
@Private
@Unstable
public class FSLeafQueue extends FSQueue {
- private static final Log LOG = LogFactory.getLog(
- FSLeafQueue.class.getName());
+ private static final Log LOG = LogFactory.getLog(FSLeafQueue.class.getName());
+ private static final List<FSQueue> EMPTY_LIST = Collections.emptyList();
+
+ private FairScheduler scheduler;
+ private FSContext context;
- private final List<FSAppAttempt> runnableApps = // apps that are runnable
- new ArrayList<FSAppAttempt>();
- private final List<FSAppAttempt> nonRunnableApps =
- new ArrayList<FSAppAttempt>();
+ // apps that are runnable
+ private final List<FSAppAttempt> runnableApps = new ArrayList<>();
+ private final List<FSAppAttempt> nonRunnableApps = new ArrayList<>();
// get a lock with fair distribution for app list updates
private final ReadWriteLock rwl = new ReentrantReadWriteLock(true);
private final Lock readLock = rwl.readLock();
@@ -64,24 +67,23 @@ public class FSLeafQueue extends FSQueue {
// Variables used for preemption
private long lastTimeAtMinShare;
- private long lastTimeAtFairShareThreshold;
-
+
// Track the AM resource usage for this queue
private Resource amResourceUsage;
private final ActiveUsersManager activeUsersManager;
- public static final List<FSQueue> EMPTY_LIST = Collections.emptyList();
public FSLeafQueue(String name, FairScheduler scheduler,
FSParentQueue parent) {
super(name, scheduler, parent);
+ this.scheduler = scheduler;
+ this.context = scheduler.getContext();
this.lastTimeAtMinShare = scheduler.getClock().getTime();
- this.lastTimeAtFairShareThreshold = scheduler.getClock().getTime();
activeUsersManager = new ActiveUsersManager(getMetrics());
amResourceUsage = Resource.newInstance(0, 0);
}
- public void addApp(FSAppAttempt app, boolean runnable) {
+ void addApp(FSAppAttempt app, boolean runnable) {
writeLock.lock();
try {
if (runnable) {
@@ -108,7 +110,7 @@ public class FSLeafQueue extends FSQueue {
* Removes the given app from this queue.
* @return whether or not the app was runnable
*/
- public boolean removeApp(FSAppAttempt app) {
+ boolean removeApp(FSAppAttempt app) {
boolean runnable = false;
// Remove app from runnable/nonRunnable list while holding the write lock
@@ -139,7 +141,7 @@ public class FSLeafQueue extends FSQueue {
* Removes the given app if it is non-runnable and belongs to this queue
* @return true if the app is removed, false otherwise
*/
- public boolean removeNonRunnableApp(FSAppAttempt app) {
+ boolean removeNonRunnableApp(FSAppAttempt app) {
writeLock.lock();
try {
return nonRunnableApps.remove(app);
@@ -148,7 +150,7 @@ public class FSLeafQueue extends FSQueue {
}
}
- public boolean isRunnableApp(FSAppAttempt attempt) {
+ boolean isRunnableApp(FSAppAttempt attempt) {
readLock.lock();
try {
return runnableApps.contains(attempt);
@@ -157,7 +159,7 @@ public class FSLeafQueue extends FSQueue {
}
}
- public boolean isNonRunnableApp(FSAppAttempt attempt) {
+ boolean isNonRunnableApp(FSAppAttempt attempt) {
readLock.lock();
try {
return nonRunnableApps.contains(attempt);
@@ -166,30 +168,8 @@ public class FSLeafQueue extends FSQueue {
}
}
- public void resetPreemptedResources() {
- readLock.lock();
- try {
- for (FSAppAttempt attempt : runnableApps) {
- attempt.resetPreemptedResources();
- }
- } finally {
- readLock.unlock();
- }
- }
-
- public void clearPreemptedResources() {
- readLock.lock();
- try {
- for (FSAppAttempt attempt : runnableApps) {
- attempt.clearPreemptedResources();
- }
- } finally {
- readLock.unlock();
- }
- }
-
- public List<FSAppAttempt> getCopyOfNonRunnableAppSchedulables() {
- List<FSAppAttempt> appsToReturn = new ArrayList<FSAppAttempt>();
+ List<FSAppAttempt> getCopyOfNonRunnableAppSchedulables() {
+ List<FSAppAttempt> appsToReturn = new ArrayList<>();
readLock.lock();
try {
appsToReturn.addAll(nonRunnableApps);
@@ -223,17 +203,78 @@ public class FSLeafQueue extends FSQueue {
}
super.policy = policy;
}
-
+
@Override
- public void recomputeShares() {
+ public void updateInternal(boolean checkStarvation) {
readLock.lock();
try {
policy.computeShares(runnableApps, getFairShare());
+ if (checkStarvation) {
+ updateStarvedApps();
+ }
} finally {
readLock.unlock();
}
}
+ /**
+ * Helper method to identify starved applications. This needs to be called
+ * ONLY from {@link #updateInternal}, after the application shares
+ * are updated.
+ *
+ * A queue can be starving due to fairshare or minshare.
+ *
+ * Minshare is defined only on the queue and not the applications.
+ * Fairshare is defined for both the queue and the applications.
+ *
+ * If this queue is starved due to minshare, we need to identify the most
+ * deserving apps if they themselves are not starved due to fairshare.
+ *
+ * If this queue is starving due to fairshare, there must be at least
+ * one application that is starved. And, even if the queue is not
+ * starved due to fairshare, there might still be starved applications.
+ */
+ private void updateStarvedApps() {
+ // First identify starved applications and track total amount of
+ // starvation (in resources)
+ Resource fairShareStarvation = Resources.clone(none());
+
+ // Fetch apps with unmet demand sorted by fairshare starvation
+ TreeSet<FSAppAttempt> appsWithDemand = fetchAppsWithDemand();
+ for (FSAppAttempt app : appsWithDemand) {
+ Resource appStarvation = app.fairShareStarvation();
+ if (!Resources.equals(Resources.none(), appStarvation)) {
+ context.getStarvedApps().addStarvedApp(app);
+ Resources.addTo(fairShareStarvation, appStarvation);
+ } else {
+ break;
+ }
+ }
+
+ // Compute extent of minshare starvation
+ Resource minShareStarvation = minShareStarvation();
+
+ // Compute minshare starvation that is not subsumed by fairshare starvation
+ Resources.subtractFrom(minShareStarvation, fairShareStarvation);
+
+ // Keep adding apps to the starved list until the unmet demand goes over
+ // the remaining minshare
+ for (FSAppAttempt app : appsWithDemand) {
+ if (Resources.greaterThan(policy.getResourceCalculator(),
+ scheduler.getClusterResource(), minShareStarvation, none())) {
+ Resource appPendingDemand =
+ Resources.subtract(app.getDemand(), app.getResourceUsage());
+ Resources.subtractFrom(minShareStarvation, appPendingDemand);
+ app.setMinshareStarvation(appPendingDemand);
+ context.getStarvedApps().addStarvedApp(app);
+ } else {
+ // Reset minshare starvation in case we had set it in a previous
+ // iteration
+ app.resetMinshareStarvation();
+ }
+ }
+ }
+
@Override
public Resource getDemand() {
return demand;
@@ -256,7 +297,7 @@ public class FSLeafQueue extends FSQueue {
return usage;
}
- public Resource getAmResourceUsage() {
+ Resource getAmResourceUsage() {
return amResourceUsage;
}
@@ -299,7 +340,7 @@ public class FSLeafQueue extends FSQueue {
@Override
public Resource assignContainer(FSSchedulerNode node) {
- Resource assigned = Resources.none();
+ Resource assigned = none();
if (LOG.isDebugEnabled()) {
LOG.debug("Node " + node.getNodeName() + " offered to queue: " +
getName() + " fairShare: " + getFairShare());
@@ -309,26 +350,12 @@ public class FSLeafQueue extends FSQueue {
return assigned;
}
- // Apps that have resource demands.
- TreeSet<FSAppAttempt> pendingForResourceApps =
- new TreeSet<FSAppAttempt>(policy.getComparator());
- readLock.lock();
- try {
- for (FSAppAttempt app : runnableApps) {
- Resource pending = app.getAppAttemptResourceUsage().getPending();
- if (!pending.equals(Resources.none())) {
- pendingForResourceApps.add(app);
- }
- }
- } finally {
- readLock.unlock();
- }
- for (FSAppAttempt sched : pendingForResourceApps) {
+ for (FSAppAttempt sched : fetchAppsWithDemand()) {
if (SchedulerAppUtils.isPlaceBlacklisted(sched, node, LOG)) {
continue;
}
assigned = sched.assignContainer(node);
- if (!assigned.equals(Resources.none())) {
+ if (!assigned.equals(none())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigned container in queue:" + getName() + " " +
"container:" + assigned);
@@ -339,40 +366,21 @@ public class FSLeafQueue extends FSQueue {
return assigned;
}
- @Override
- public RMContainer preemptContainer() {
- RMContainer toBePreempted = null;
-
- // If this queue is not over its fair share, reject
- if (!preemptContainerPreCheck()) {
- return toBePreempted;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Queue " + getName() + " is going to preempt a container " +
- "from its applications.");
- }
-
- // Choose the app that is most over fair share
- Comparator<Schedulable> comparator = policy.getComparator();
- FSAppAttempt candidateSched = null;
+ private TreeSet<FSAppAttempt> fetchAppsWithDemand() {
+ TreeSet<FSAppAttempt> pendingForResourceApps =
+ new TreeSet<>(policy.getComparator());
readLock.lock();
try {
- for (FSAppAttempt sched : runnableApps) {
- if (candidateSched == null ||
- comparator.compare(sched, candidateSched) > 0) {
- candidateSched = sched;
+ for (FSAppAttempt app : runnableApps) {
+ Resource pending = app.getAppAttemptResourceUsage().getPending();
+ if (!pending.equals(none())) {
+ pendingForResourceApps.add(app);
}
}
} finally {
readLock.unlock();
}
-
- // Preempt from the selected app
- if (candidateSched != null) {
- toBePreempted = candidateSched.preemptContainer();
- }
- return toBePreempted;
+ return pendingForResourceApps;
}
@Override
@@ -384,7 +392,7 @@ public class FSLeafQueue extends FSQueue {
public List<QueueUserACLInfo> getQueueUserAclInfo(UserGroupInformation user) {
QueueUserACLInfo userAclInfo =
recordFactory.newRecordInstance(QueueUserACLInfo.class);
- List<QueueACL> operations = new ArrayList<QueueACL>();
+ List<QueueACL> operations = new ArrayList<>();
for (QueueACL operation : QueueACL.values()) {
if (hasAccess(operation, user)) {
operations.add(operation);
@@ -396,23 +404,10 @@ public class FSLeafQueue extends FSQueue {
return Collections.singletonList(userAclInfo);
}
- public long getLastTimeAtMinShare() {
- return lastTimeAtMinShare;
- }
-
private void setLastTimeAtMinShare(long lastTimeAtMinShare) {
this.lastTimeAtMinShare = lastTimeAtMinShare;
}
- public long getLastTimeAtFairShareThreshold() {
- return lastTimeAtFairShareThreshold;
- }
-
- private void setLastTimeAtFairShareThreshold(
- long lastTimeAtFairShareThreshold) {
- this.lastTimeAtFairShareThreshold = lastTimeAtFairShareThreshold;
- }
-
@Override
public int getNumRunnableApps() {
readLock.lock();
@@ -423,7 +418,7 @@ public class FSLeafQueue extends FSQueue {
}
}
- public int getNumNonRunnableApps() {
+ int getNumNonRunnableApps() {
readLock.lock();
try {
return nonRunnableApps.size();
@@ -475,10 +470,11 @@ public class FSLeafQueue extends FSQueue {
/**
* Check whether this queue can run this application master under the
* maxAMShare limit.
- * @param amResource
+ *
+ * @param amResource resources required to run the AM
* @return true if this queue can run
*/
- public boolean canRunAppAM(Resource amResource) {
+ boolean canRunAppAM(Resource amResource) {
if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
return true;
}
@@ -503,7 +499,7 @@ public class FSLeafQueue extends FSQueue {
return Resources.fitsIn(ifRunAMResource, maxAMResource);
}
- public void addAMResourceUsage(Resource amResource) {
+ void addAMResourceUsage(Resource amResource) {
if (amResource != null) {
Resources.addTo(amResourceUsage, amResource);
}
@@ -516,21 +512,8 @@ public class FSLeafQueue extends FSQueue {
}
/**
- * Update the preemption fields for the queue, i.e. the times since last was
- * at its guaranteed share and over its fair share threshold.
- */
- public void updateStarvationStats() {
- long now = scheduler.getClock().getTime();
- if (!isStarvedForMinShare()) {
- setLastTimeAtMinShare(now);
- }
- if (!isStarvedForFairShare()) {
- setLastTimeAtFairShareThreshold(now);
- }
- }
-
- /** Allows setting weight for a dynamically created queue
- * Currently only used for reservation based queues
+ * Allows setting weight for a dynamically created queue.
+ * Currently only used for reservation based queues.
* @param weight queue weight
*/
public void setWeights(float weight) {
@@ -538,37 +521,61 @@ public class FSLeafQueue extends FSQueue {
}
/**
- * Helper method to check if the queue should preempt containers
+ * Helper method to compute the amount of minshare starvation.
*
- * @return true if check passes (can preempt) or false otherwise
+ * @return the extent of minshare starvation
*/
- private boolean preemptContainerPreCheck() {
- return parent.getPolicy().checkIfUsageOverFairShare(getResourceUsage(),
- getFairShare());
+ private Resource minShareStarvation() {
+ // If demand < minshare, we should use demand to determine starvation
+ Resource desiredShare = Resources.min(policy.getResourceCalculator(),
+ scheduler.getClusterResource(), getMinShare(), getDemand());
+
+ Resource starvation = Resources.subtract(desiredShare, getResourceUsage());
+ boolean starved = !Resources.isNone(starvation);
+
+ long now = scheduler.getClock().getTime();
+ if (!starved) {
+ // Record that the queue is not starved
+ setLastTimeAtMinShare(now);
+ }
+
+ if (now - lastTimeAtMinShare < getMinSharePreemptionTimeout()) {
+ // the queue is not starved for the preemption timeout
+ starvation = Resources.clone(Resources.none());
+ }
+
+ return starvation;
}
/**
- * Is a queue being starved for its min share.
+ * Helper method for tests to check if a queue is starved for minShare.
+ * @return whether starved for minshare
*/
@VisibleForTesting
- boolean isStarvedForMinShare() {
- return isStarved(getMinShare());
+ private boolean isStarvedForMinShare() {
+ return !Resources.isNone(minShareStarvation());
}
/**
- * Is a queue being starved for its fair share threshold.
+ * Helper method for tests to check if a queue is starved for fairshare.
+ * @return whether starved for fairshare
*/
@VisibleForTesting
- boolean isStarvedForFairShare() {
- return isStarved(
- Resources.multiply(getFairShare(), getFairSharePreemptionThreshold()));
+ private boolean isStarvedForFairShare() {
+ for (FSAppAttempt app : runnableApps) {
+ if (app.isStarvedForFairShare()) {
+ return true;
+ }
+ }
+ return false;
}
- private boolean isStarved(Resource share) {
- Resource desiredShare = Resources.min(policy.getResourceCalculator(),
- scheduler.getClusterResource(), share, getDemand());
- Resource resourceUsage = getResourceUsage();
- return Resources.lessThan(policy.getResourceCalculator(),
- scheduler.getClusterResource(), resourceUsage, desiredShare);
+ /**
+ * Helper method for tests to check if a queue is starved.
+ * @return whether starved for either minshare or fairshare
+ */
+ @VisibleForTesting
+ boolean isStarved() {
+ return isStarvedForMinShare() || isStarvedForFairShare();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index 53ac8c9..16570aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
import java.util.List;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
@@ -61,7 +60,7 @@ public class FSParentQueue extends FSQueue {
super(name, scheduler, parent);
}
- public void addChildQueue(FSQueue child) {
+ void addChildQueue(FSQueue child) {
writeLock.lock();
try {
childQueues.add(child);
@@ -70,7 +69,7 @@ public class FSParentQueue extends FSQueue {
}
}
- public void removeChildQueue(FSQueue child) {
+ void removeChildQueue(FSQueue child) {
writeLock.lock();
try {
childQueues.remove(child);
@@ -80,20 +79,20 @@ public class FSParentQueue extends FSQueue {
}
@Override
- public void recomputeShares() {
+ public void updateInternal(boolean checkStarvation) {
readLock.lock();
try {
policy.computeShares(childQueues, getFairShare());
for (FSQueue childQueue : childQueues) {
childQueue.getMetrics().setFairShare(childQueue.getFairShare());
- childQueue.recomputeShares();
+ childQueue.updateInternal(checkStarvation);
}
} finally {
readLock.unlock();
}
}
- public void recomputeSteadyShares() {
+ void recomputeSteadyShares() {
readLock.lock();
try {
policy.computeSteadyShares(childQueues, getSteadyFairShare());
@@ -188,7 +187,7 @@ public class FSParentQueue extends FSQueue {
@Override
public List<QueueUserACLInfo> getQueueUserAclInfo(UserGroupInformation user) {
- List<QueueUserACLInfo> userAcls = new ArrayList<QueueUserACLInfo>();
+ List<QueueUserACLInfo> userAcls = new ArrayList<>();
// Add queue acls
userAcls.add(getUserAclInfo(user));
@@ -246,39 +245,6 @@ public class FSParentQueue extends FSQueue {
}
@Override
- public RMContainer preemptContainer() {
- RMContainer toBePreempted = null;
-
- // Find the childQueue which is most over fair share
- FSQueue candidateQueue = null;
- Comparator<Schedulable> comparator = policy.getComparator();
-
- readLock.lock();
- try {
- for (FSQueue queue : childQueues) {
- // Skip selection for non-preemptable queue
- if (!queue.isPreemptable()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("skipping from queue=" + getName()
- + " because it's a non-preemptable queue");
- }
- } else if (candidateQueue == null ||
- comparator.compare(queue, candidateQueue) > 0) {
- candidateQueue = queue;
- }
- }
- } finally {
- readLock.unlock();
- }
-
- // Let the selected queue choose which of its container to preempt
- if (candidateQueue != null) {
- toBePreempted = candidateQueue.preemptContainer();
- }
- return toBePreempted;
- }
-
- @Override
public List<FSQueue> getChildQueues() {
readLock.lock();
try {
@@ -300,8 +266,8 @@ public class FSParentQueue extends FSQueue {
}
super.policy = policy;
}
-
- public void incrementRunnableApps() {
+
+ void incrementRunnableApps() {
writeLock.lock();
try {
runnableApps++;
@@ -310,7 +276,7 @@ public class FSParentQueue extends FSQueue {
}
}
- public void decrementRunnableApps() {
+ void decrementRunnableApps() {
writeLock.lock();
try {
runnableApps--;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
new file mode 100644
index 0000000..3579857
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+
+/**
+ * Thread that handles FairScheduler preemption.
+ */
+class FSPreemptionThread extends Thread {
+ private static final Log LOG = LogFactory.getLog(FSPreemptionThread.class);
+ protected final FSContext context;
+ private final FairScheduler scheduler;
+ private final long warnTimeBeforeKill;
+ private final Timer preemptionTimer;
+
+ FSPreemptionThread(FairScheduler scheduler) {
+ this.scheduler = scheduler;
+ this.context = scheduler.getContext();
+ FairSchedulerConfiguration fsConf = scheduler.getConf();
+ context.setPreemptionEnabled();
+ context.setPreemptionUtilizationThreshold(
+ fsConf.getPreemptionUtilizationThreshold());
+ warnTimeBeforeKill = fsConf.getWaitTimeBeforeKill();
+ preemptionTimer = new Timer("Preemption Timer", true);
+
+ setDaemon(true);
+ setName("FSPreemptionThread");
+ }
+
+ public void run() {
+ while (!Thread.interrupted()) {
+ FSAppAttempt starvedApp;
+ try{
+ starvedApp = context.getStarvedApps().take();
+ if (!Resources.isNone(starvedApp.getStarvation())) {
+ List<RMContainer> containers =
+ identifyContainersToPreempt(starvedApp);
+ if (containers != null) {
+ preemptContainers(containers);
+ }
+ }
+ } catch (InterruptedException e) {
+ LOG.info("Preemption thread interrupted! Exiting.");
+ return;
+ }
+ }
+ }
+
+ /**
+ * Given an app, identify containers to preempt to satisfy the app's next
+ * resource request.
+ *
+ * @param starvedApp starved application for which we are identifying
+ * preemption targets
+ * @return list of containers to preempt to satisfy starvedApp, null if the
+ * app cannot be satisfied by preempting any running containers
+ */
+ private List<RMContainer> identifyContainersToPreempt(
+ FSAppAttempt starvedApp) {
+ List<RMContainer> containers = new ArrayList<>(); // return value
+
+ // Find the nodes that match the next resource request
+ ResourceRequest request = starvedApp.getNextResourceRequest();
+ // TODO (KK): Should we check other resource requests if we can't match
+ // the first one?
+
+ Resource requestCapability = request.getCapability();
+ List<FSSchedulerNode> potentialNodes =
+ scheduler.getNodeTracker().getNodesByResourceName(
+ request.getResourceName());
+
+ // From the potential nodes, pick a node that has enough containers
+ // from apps over their fairshare
+ for (FSSchedulerNode node : potentialNodes) {
+ // Reset containers for the new node being considered.
+ containers.clear();
+
+ // TODO (YARN-5829): Attempt to reserve the node for starved app. The
+ // subsequent if-check needs to be reworked accordingly.
+ FSAppAttempt nodeReservedApp = node.getReservedAppSchedulable();
+ if (nodeReservedApp != null && !nodeReservedApp.equals(starvedApp)) {
+ // This node is already reserved by another app. Let us not consider
+ // this for preemption.
+ continue;
+ }
+
+ // Figure out list of containers to consider
+ List<RMContainer> containersToCheck =
+ node.getCopiedListOfRunningContainers();
+ containersToCheck.removeAll(node.getContainersForPreemption());
+
+ // Initialize potential with unallocated resources
+ Resource potential = Resources.clone(node.getUnallocatedResource());
+ for (RMContainer container : containersToCheck) {
+ FSAppAttempt app =
+ scheduler.getSchedulerApp(container.getApplicationAttemptId());
+
+ if (app.canContainerBePreempted(container)) {
+ // Flag container for preemption
+ containers.add(container);
+ Resources.addTo(potential, container.getAllocatedResource());
+ }
+
+ // Check if we have already identified enough containers
+ if (Resources.fitsIn(requestCapability, potential)) {
+ // Mark the containers as being considered for preemption on the node.
+ // Make sure the containers are subsequently removed by calling
+ // FSSchedulerNode#removeContainerForPreemption.
+ node.addContainersForPreemption(containers);
+ return containers;
+ } else {
+ // TODO (YARN-5829): Unreserve the node for the starved app.
+ }
+ }
+ }
+ return null;
+ }
+
+ private void preemptContainers(List<RMContainer> containers) {
+ // Warn application about containers to be killed
+ for (RMContainer container : containers) {
+ ApplicationAttemptId appAttemptId = container.getApplicationAttemptId();
+ FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId);
+ FSLeafQueue queue = app.getQueue();
+ LOG.info("Preempting container " + container +
+ " from queue " + queue.getName());
+ app.trackContainerForPreemption(container);
+ }
+
+ // Schedule timer task to kill containers
+ preemptionTimer.schedule(
+ new PreemptContainersTask(containers), warnTimeBeforeKill);
+ }
+
+ private class PreemptContainersTask extends TimerTask {
+ private List<RMContainer> containers;
+
+ PreemptContainersTask(List<RMContainer> containers) {
+ this.containers = containers;
+ }
+
+ @Override
+ public void run() {
+ for (RMContainer container : containers) {
+ ContainerStatus status = SchedulerUtils.createPreemptedContainerStatus(
+ container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER);
+
+ LOG.info("Killing container " + container);
+ scheduler.completedContainer(
+ container, status, RMContainerEventType.KILL);
+
+ FSSchedulerNode containerNode = (FSSchedulerNode)
+ scheduler.getNodeTracker().getNode(container.getAllocatedNode());
+ containerNode.removeContainerForPreemption(container);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 5fa2ee1..572b5f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -253,7 +253,7 @@ public abstract class FSQueue implements Queue, Schedulable {
return steadyFairShare;
}
- public void setSteadyFairShare(Resource steadyFairShare) {
+ void setSteadyFairShare(Resource steadyFairShare) {
this.steadyFairShare = steadyFairShare;
metrics.setSteadyFairShare(steadyFairShare);
}
@@ -262,27 +262,27 @@ public abstract class FSQueue implements Queue, Schedulable {
return scheduler.getAllocationConfiguration().hasAccess(name, acl, user);
}
- public long getFairSharePreemptionTimeout() {
+ long getFairSharePreemptionTimeout() {
return fairSharePreemptionTimeout;
}
- public void setFairSharePreemptionTimeout(long fairSharePreemptionTimeout) {
+ void setFairSharePreemptionTimeout(long fairSharePreemptionTimeout) {
this.fairSharePreemptionTimeout = fairSharePreemptionTimeout;
}
- public long getMinSharePreemptionTimeout() {
+ long getMinSharePreemptionTimeout() {
return minSharePreemptionTimeout;
}
- public void setMinSharePreemptionTimeout(long minSharePreemptionTimeout) {
+ void setMinSharePreemptionTimeout(long minSharePreemptionTimeout) {
this.minSharePreemptionTimeout = minSharePreemptionTimeout;
}
- public float getFairSharePreemptionThreshold() {
+ float getFairSharePreemptionThreshold() {
return fairSharePreemptionThreshold;
}
- public void setFairSharePreemptionThreshold(float fairSharePreemptionThreshold) {
+ void setFairSharePreemptionThreshold(float fairSharePreemptionThreshold) {
this.fairSharePreemptionThreshold = fairSharePreemptionThreshold;
}
@@ -292,9 +292,17 @@ public abstract class FSQueue implements Queue, Schedulable {
/**
* Recomputes the shares for all child queues and applications based on this
- * queue's current share
+ * queue's current share, and checks for starvation.
+ *
+ * @param checkStarvation whether to check for fairshare or minshare
+ * starvation on update
*/
- public abstract void recomputeShares();
+ abstract void updateInternal(boolean checkStarvation);
+
+ public void update(Resource fairShare, boolean checkStarvation) {
+ setFairShare(fairShare);
+ updateInternal(checkStarvation);
+ }
/**
* Update the min/fair share preemption timeouts, threshold and preemption
@@ -347,7 +355,7 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @return true if check passes (can assign) or false otherwise
*/
- protected boolean assignContainerPreCheck(FSSchedulerNode node) {
+ boolean assignContainerPreCheck(FSSchedulerNode node) {
if (!Resources.fitsIn(getResourceUsage(), maxShare)
|| node.getReservedContainer() != null) {
return false;
@@ -403,7 +411,7 @@ public abstract class FSQueue implements Queue, Schedulable {
return null;
}
- public boolean fitsInMaxShare(Resource additionalResource) {
+ boolean fitsInMaxShare(Resource additionalResource) {
Resource usagePlusAddition =
Resources.add(getResourceUsage(), additionalResource);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index 024ec67..a27a222 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -29,6 +29,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicat
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+
@Private
@Unstable
public class FSSchedulerNode extends SchedulerNode {
@@ -36,6 +40,8 @@ public class FSSchedulerNode extends SchedulerNode {
private static final Log LOG = LogFactory.getLog(FSSchedulerNode.class);
private FSAppAttempt reservedAppSchedulable;
+ private final Set<RMContainer> containersForPreemption =
+ new ConcurrentSkipListSet<>();
public FSSchedulerNode(RMNode node, boolean usePortForNodeName) {
super(node, usePortForNodeName);
@@ -99,8 +105,36 @@ public class FSSchedulerNode extends SchedulerNode {
this.reservedAppSchedulable = null;
}
- public synchronized FSAppAttempt getReservedAppSchedulable() {
+ synchronized FSAppAttempt getReservedAppSchedulable() {
return reservedAppSchedulable;
}
+ /**
+ * Mark {@code containers} as being considered for preemption so they are
+ * not considered again. A call to this requires a corresponding call to
+ * {@link #removeContainerForPreemption} to ensure we do not mark a
+ * container for preemption and never consider it again and avoid memory
+ * leaks.
+ *
+ * @param containers container to mark
+ */
+ void addContainersForPreemption(Collection<RMContainer> containers) {
+ containersForPreemption.addAll(containers);
+ }
+
+ /**
+ * @return set of containers marked for preemption.
+ */
+ Set<RMContainer> getContainersForPreemption() {
+ return containersForPreemption;
+ }
+
+ /**
+ * Remove container from the set of containers marked for preemption.
+ *
+ * @param container container to remove
+ */
+ void removeContainerForPreemption(RMContainer container) {
+ containersForPreemption.remove(container);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSStarvedApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSStarvedApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSStarvedApps.java
new file mode 100644
index 0000000..4f28e41
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSStarvedApps.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import java.io.Serializable;
+import java.util.Comparator;
+import java.util.concurrent.PriorityBlockingQueue;
+
+/**
+ * Helper class to track starved applications.
+ *
+ * Initially, this uses a blocking queue. We could use other data structures
+ * in the future. This class also has some methods to simplify testing.
+ */
+class FSStarvedApps {
+
+ // List of apps to be processed by the preemption thread.
+ private PriorityBlockingQueue<FSAppAttempt> appsToProcess;
+
+ // App being currently processed. This assumes a single reader.
+ private FSAppAttempt appBeingProcessed;
+
+ FSStarvedApps() {
+ appsToProcess = new PriorityBlockingQueue<>(10, new StarvationComparator());
+ }
+
+ /**
+ * Add a starved application if it is not already added.
+ * @param app application to add
+ */
+ void addStarvedApp(FSAppAttempt app) {
+ if (!app.equals(appBeingProcessed) && !appsToProcess.contains(app)) {
+ appsToProcess.add(app);
+ }
+ }
+
+ /**
+ * Blocking call to fetch the next app to process. The returned app is
+ * tracked until the next call to this method. This tracking assumes a
+ * single reader.
+ *
+ * @return starved application to process
+ * @throws InterruptedException if interrupted while waiting
+ */
+ FSAppAttempt take() throws InterruptedException {
+ // Reset appBeingProcessed before the blocking call
+ appBeingProcessed = null;
+
+ // Blocking call to fetch the next starved application
+ FSAppAttempt app = appsToProcess.take();
+ appBeingProcessed = app;
+ return app;
+ }
+
+ private static class StarvationComparator implements
+ Comparator<FSAppAttempt>, Serializable {
+ private static final long serialVersionUID = 1;
+
+ @Override
+ public int compare(FSAppAttempt app1, FSAppAttempt app2) {
+ int ret = 1;
+ if (Resources.fitsIn(app1.getStarvation(), app2.getStarvation())) {
+ ret = -1;
+ }
+ return ret;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 354f936..17f066b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -24,7 +24,6 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -123,6 +122,7 @@ public class FairScheduler extends
AbstractYarnScheduler<FSAppAttempt, FSSchedulerNode> {
private FairSchedulerConfiguration conf;
+ private FSContext context;
private Resource incrAllocation;
private QueueManager queueMgr;
private boolean usePortForNodeName;
@@ -150,6 +150,9 @@ public class FairScheduler extends
@VisibleForTesting
Thread schedulingThread;
+
+ Thread preemptionThread;
+
// timeout to join when we stop this service
protected final long THREAD_JOIN_TIMEOUT_MS = 1000;
@@ -157,25 +160,6 @@ public class FairScheduler extends
FSQueueMetrics rootMetrics;
FSOpDurations fsOpDurations;
- // Time when we last updated preemption vars
- protected long lastPreemptionUpdateTime;
- // Time we last ran preemptTasksIfNecessary
- private long lastPreemptCheckTime;
-
- // Preemption related variables
- protected boolean preemptionEnabled;
- protected float preemptionUtilizationThreshold;
-
- // How often tasks are preempted
- protected long preemptionInterval;
-
- // ms to wait before force killing stuff (must be longer than a couple
- // of heartbeats to give task-kill commands a chance to act).
- protected long waitTimeBeforeKill;
-
- // Containers whose AMs have been warned that they will be preempted soon.
- private List<RMContainer> warnedContainers = new ArrayList<RMContainer>();
-
private float reservableNodesRatio; // percentage of available nodes
// an app can be reserved on
@@ -211,11 +195,17 @@ public class FairScheduler extends
public FairScheduler() {
super(FairScheduler.class.getName());
+ context = new FSContext();
allocsLoader = new AllocationFileLoaderService();
queueMgr = new QueueManager(this);
maxRunningEnforcer = new MaxRunningAppsEnforcer(this);
}
+ @VisibleForTesting
+ public FSContext getContext() {
+ return context;
+ }
+
public boolean isAtLeastReservationThreshold(
ResourceCalculator resourceCalculator, Resource resource) {
return Resources.greaterThanOrEqual(resourceCalculator,
@@ -296,7 +286,6 @@ public class FairScheduler extends
}
long start = getClock().getTime();
update();
- preemptTasksIfNecessary();
long duration = getClock().getTime() - start;
fsOpDurations.addUpdateThreadRunDuration(duration);
} catch (InterruptedException ie) {
@@ -338,7 +327,6 @@ public class FairScheduler extends
try {
writeLock.lock();
long start = getClock().getTime();
- updateStarvationStats(); // Determine if any queues merit preemption
FSQueue rootQueue = queueMgr.getRootQueue();
@@ -346,214 +334,30 @@ public class FairScheduler extends
rootQueue.updateDemand();
Resource clusterResource = getClusterResource();
- rootQueue.setFairShare(clusterResource);
- // Recursively compute fair shares for all queues
- // and update metrics
- rootQueue.recomputeShares();
+ rootQueue.update(clusterResource, shouldAttemptPreemption());
+
+ // Update metrics
updateRootQueueMetrics();
if (LOG.isDebugEnabled()) {
if (--updatesToSkipForDebug < 0) {
updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY;
- LOG.debug("Cluster Capacity: " + clusterResource + " Allocations: "
- + rootMetrics.getAllocatedResources() + " Availability: "
- + Resource.newInstance(rootMetrics.getAvailableMB(),
- rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue
- .getDemand());
+ LOG.debug("Cluster Capacity: " + clusterResource +
+ " Allocations: " + rootMetrics.getAllocatedResources() +
+ " Availability: " + Resource.newInstance(
+ rootMetrics.getAvailableMB(),
+ rootMetrics.getAvailableVirtualCores()) +
+ " Demand: " + rootQueue.getDemand());
}
- }
-
- long duration = getClock().getTime() - start;
- fsOpDurations.addUpdateCallDuration(duration);
- } finally {
- writeLock.unlock();
- }
- }
- /**
- * Update the preemption fields for all QueueScheduables, i.e. the times since
- * each queue last was at its guaranteed share and over its fair share
- * threshold for each type of task.
- */
- private void updateStarvationStats() {
- lastPreemptionUpdateTime = getClock().getTime();
- for (FSLeafQueue sched : queueMgr.getLeafQueues()) {
- sched.updateStarvationStats();
- }
- }
-
- /**
- * Check for queues that need tasks preempted, either because they have been
- * below their guaranteed share for minSharePreemptionTimeout or they have
- * been below their fair share threshold for the fairSharePreemptionTimeout. If
- * such queues exist, compute how many tasks of each type need to be preempted
- * and then select the right ones using preemptTasks.
- */
- protected void preemptTasksIfNecessary() {
- try {
- writeLock.lock();
- if (!shouldAttemptPreemption()) {
- return;
- }
-
- long curTime = getClock().getTime();
- if (curTime - lastPreemptCheckTime < preemptionInterval) {
- return;
- }
- lastPreemptCheckTime = curTime;
-
- Resource resToPreempt = Resources.clone(Resources.none());
- for (FSLeafQueue sched : queueMgr.getLeafQueues()) {
- Resources.addTo(resToPreempt, resourceDeficit(sched, curTime));
- }
- if (isResourceGreaterThanNone(resToPreempt)) {
- preemptResources(resToPreempt);
+ long duration = getClock().getTime() - start;
+ fsOpDurations.addUpdateCallDuration(duration);
}
} finally {
writeLock.unlock();
}
}
- /**
- * Preempt a quantity of resources. Each round, we start from the root queue,
- * level-by-level, until choosing a candidate application.
- * The policy for prioritizing preemption for each queue depends on its
- * SchedulingPolicy: (1) fairshare/DRF, choose the ChildSchedulable that is
- * most over its fair share; (2) FIFO, choose the childSchedulable that is
- * latest launched.
- * Inside each application, we further prioritize preemption by choosing
- * containers with lowest priority to preempt.
- * We make sure that no queue is placed below its fair share in the process.
- */
- protected void preemptResources(Resource toPreempt) {
- long start = getClock().getTime();
- if (Resources.equals(toPreempt, Resources.none())) {
- return;
- }
-
- // Scan down the list of containers we've already warned and kill them
- // if we need to. Remove any containers from the list that we don't need
- // or that are no longer running.
- Iterator<RMContainer> warnedIter = warnedContainers.iterator();
- while (warnedIter.hasNext()) {
- RMContainer container = warnedIter.next();
- if ((container.getState() == RMContainerState.RUNNING ||
- container.getState() == RMContainerState.ALLOCATED) &&
- isResourceGreaterThanNone(toPreempt)) {
- warnOrKillContainer(container);
- Resources.subtractFrom(toPreempt, container.getContainer().getResource());
- } else {
- warnedIter.remove();
- }
- }
-
- try {
- // Reset preemptedResource for each app
- for (FSLeafQueue queue : getQueueManager().getLeafQueues()) {
- queue.resetPreemptedResources();
- }
-
- while (isResourceGreaterThanNone(toPreempt)) {
- RMContainer container =
- getQueueManager().getRootQueue().preemptContainer();
- if (container == null) {
- break;
- } else {
- warnOrKillContainer(container);
- warnedContainers.add(container);
- Resources.subtractFrom(
- toPreempt, container.getContainer().getResource());
- }
- }
- } finally {
- // Clear preemptedResources for each app
- for (FSLeafQueue queue : getQueueManager().getLeafQueues()) {
- queue.clearPreemptedResources();
- }
- }
-
- long duration = getClock().getTime() - start;
- fsOpDurations.addPreemptCallDuration(duration);
- }
-
- private boolean isResourceGreaterThanNone(Resource toPreempt) {
- return (toPreempt.getMemorySize() > 0) || (toPreempt.getVirtualCores() > 0);
- }
-
- protected void warnOrKillContainer(RMContainer container) {
- ApplicationAttemptId appAttemptId = container.getApplicationAttemptId();
- FSAppAttempt app = getSchedulerApp(appAttemptId);
- FSLeafQueue queue = app.getQueue();
- LOG.info("Preempting container (prio=" + container.getContainer().getPriority() +
- "res=" + container.getContainer().getResource() +
- ") from queue " + queue.getName());
-
- Long time = app.getContainerPreemptionTime(container);
-
- if (time != null) {
- // if we asked for preemption more than maxWaitTimeBeforeKill ms ago,
- // proceed with kill
- if (time + waitTimeBeforeKill < getClock().getTime()) {
- ContainerStatus status =
- SchedulerUtils.createPreemptedContainerStatus(
- container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER);
-
- // TODO: Not sure if this ever actually adds this to the list of cleanup
- // containers on the RMNode (see SchedulerNode.releaseContainer()).
- super.completedContainer(container, status, RMContainerEventType.KILL);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Killing container" + container +
- " (after waiting for preemption for " +
- (getClock().getTime() - time) + "ms)");
- }
- }
- } else {
- // track the request in the FSAppAttempt itself
- app.addPreemption(container, getClock().getTime());
- }
- }
-
- /**
- * Return the resource amount that this queue is allowed to preempt, if any.
- * If the queue has been below its min share for at least its preemption
- * timeout, it should preempt the difference between its current share and
- * this min share. If it has been below its fair share preemption threshold
- * for at least the fairSharePreemptionTimeout, it should preempt enough tasks
- * to get up to its full fair share. If both conditions hold, we preempt the
- * max of the two amounts (this shouldn't happen unless someone sets the
- * timeouts to be identical for some reason).
- */
- protected Resource resourceDeficit(FSLeafQueue sched, long curTime) {
- long minShareTimeout = sched.getMinSharePreemptionTimeout();
- long fairShareTimeout = sched.getFairSharePreemptionTimeout();
- Resource resDueToMinShare = Resources.none();
- Resource resDueToFairShare = Resources.none();
- ResourceCalculator calc = sched.getPolicy().getResourceCalculator();
- Resource clusterResource = getClusterResource();
- if (curTime - sched.getLastTimeAtMinShare() > minShareTimeout) {
- Resource target = Resources.componentwiseMin(
- sched.getMinShare(), sched.getDemand());
- resDueToMinShare = Resources.max(calc, clusterResource,
- Resources.none(), Resources.subtract(target, sched.getResourceUsage()));
- }
- if (curTime - sched.getLastTimeAtFairShareThreshold() > fairShareTimeout) {
- Resource target = Resources.componentwiseMin(
- sched.getFairShare(), sched.getDemand());
- resDueToFairShare = Resources.max(calc, clusterResource,
- Resources.none(), Resources.subtract(target, sched.getResourceUsage()));
- }
- Resource deficit = Resources.max(calc, clusterResource,
- resDueToMinShare, resDueToFairShare);
- if (Resources.greaterThan(calc, clusterResource,
- deficit, Resources.none())) {
- String message = "Should preempt " + deficit + " res for queue "
- + sched.getName() + ": resDueToMinShare = " + resDueToMinShare
- + ", resDueToFairShare = " + resDueToFairShare;
- LOG.info(message);
- }
- return deficit;
- }
-
public RMContainerTokenSecretManager
getContainerTokenSecretManager() {
return rmContext.getContainerTokenSecretManager();
@@ -1198,12 +1002,12 @@ public class FairScheduler extends
* @return true if preemption should be attempted, false otherwise.
*/
private boolean shouldAttemptPreemption() {
- if (preemptionEnabled) {
- Resource clusterResource = getClusterResource();
- return (preemptionUtilizationThreshold < Math.max(
- (float) rootMetrics.getAllocatedMB() / clusterResource.getMemorySize(),
+ if (context.isPreemptionEnabled()) {
+ return (context.getPreemptionUtilizationThreshold() < Math.max(
+ (float) rootMetrics.getAllocatedMB() /
+ getClusterResource().getMemorySize(),
(float) rootMetrics.getAllocatedVirtualCores() /
- clusterResource.getVirtualCores()));
+ getClusterResource().getVirtualCores()));
}
return false;
}
@@ -1391,15 +1195,10 @@ public class FairScheduler extends
rackLocalityThreshold = this.conf.getLocalityThresholdRack();
nodeLocalityDelayMs = this.conf.getLocalityDelayNodeMs();
rackLocalityDelayMs = this.conf.getLocalityDelayRackMs();
- preemptionEnabled = this.conf.getPreemptionEnabled();
- preemptionUtilizationThreshold =
- this.conf.getPreemptionUtilizationThreshold();
assignMultiple = this.conf.getAssignMultiple();
maxAssignDynamic = this.conf.isMaxAssignDynamic();
maxAssign = this.conf.getMaxAssign();
sizeBasedWeight = this.conf.getSizeBasedWeight();
- preemptionInterval = this.conf.getPreemptionInterval();
- waitTimeBeforeKill = this.conf.getWaitTimeBeforeKill();
usePortForNodeName = this.conf.getUsePortForNodeName();
reservableNodesRatio = this.conf.getReservableNodes();
@@ -1437,6 +1236,10 @@ public class FairScheduler extends
schedulingThread.setName("FairSchedulerContinuousScheduling");
schedulingThread.setDaemon(true);
}
+
+ if (this.conf.getPreemptionEnabled()) {
+ createPreemptionThread();
+ }
} finally {
writeLock.unlock();
}
@@ -1453,6 +1256,11 @@ public class FairScheduler extends
}
}
+ @VisibleForTesting
+ protected void createPreemptionThread() {
+ preemptionThread = new FSPreemptionThread(this);
+ }
+
private void updateReservationThreshold() {
Resource newThreshold = Resources.multiply(
getIncrementResourceCapability(),
@@ -1472,6 +1280,9 @@ public class FairScheduler extends
"schedulingThread is null");
schedulingThread.start();
}
+ if (preemptionThread != null) {
+ preemptionThread.start();
+ }
allocsLoader.start();
} finally {
writeLock.unlock();
@@ -1504,6 +1315,10 @@ public class FairScheduler extends
schedulingThread.join(THREAD_JOIN_TIMEOUT_MS);
}
}
+ if (preemptionThread != null) {
+ preemptionThread.interrupt();
+ preemptionThread.join(THREAD_JOIN_TIMEOUT_MS);
+ }
if (allocsLoader != null) {
allocsLoader.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
index 289887f..cf78405 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
@@ -55,50 +55,45 @@ public interface Schedulable {
* Name of job/queue, used for debugging as well as for breaking ties in
* scheduling order deterministically.
*/
- public String getName();
+ String getName();
/**
* Maximum number of resources required by this Schedulable. This is defined as
* number of currently utilized resources + number of unlaunched resources (that
* are either not yet launched or need to be speculated).
*/
- public Resource getDemand();
+ Resource getDemand();
/** Get the aggregate amount of resources consumed by the schedulable. */
- public Resource getResourceUsage();
+ Resource getResourceUsage();
/** Minimum Resource share assigned to the schedulable. */
- public Resource getMinShare();
+ Resource getMinShare();
/** Maximum Resource share assigned to the schedulable. */
- public Resource getMaxShare();
+ Resource getMaxShare();
/** Job/queue weight in fair sharing. */
- public ResourceWeights getWeights();
+ ResourceWeights getWeights();
/** Start time for jobs in FIFO queues; meaningless for QueueSchedulables.*/
- public long getStartTime();
+ long getStartTime();
/** Job priority for jobs in FIFO queues; meaningless for QueueSchedulables. */
- public Priority getPriority();
+ Priority getPriority();
/** Refresh the Schedulable's demand and those of its children if any. */
- public void updateDemand();
+ void updateDemand();
/**
* Assign a container on this node if possible, and return the amount of
* resources assigned.
*/
- public Resource assignContainer(FSSchedulerNode node);
-
- /**
- * Preempt a container from this Schedulable if possible.
- */
- public RMContainer preemptContainer();
+ Resource assignContainer(FSSchedulerNode node);
/** Get the fair share assigned to this Schedulable. */
- public Resource getFairShare();
+ Resource getFairShare();
/** Assign a fair share to this Schedulable. */
- public void setFairShare(Resource fairShare);
+ void setFairShare(Resource fairShare);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/48] hadoop git commit: HADOOP-10776 Open up already widely-used
APIs for delegation-token fetching & renewal to ecosystem projects.
Contributed by Vinod Kumar Vavilapalli
Posted by li...@apache.org.
HADOOP-10776 Open up already widely-used APIs for delegation-token fetching & renewal to ecosystem projects. Contributed by Vinod Kumar Vavilapalli
Cherry picked from 24715cefe80fba531dd16a44ec1cead67c389eb1
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01665e45
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01665e45
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01665e45
Branch: refs/heads/HADOOP-13345
Commit: 01665e456de8d79000ce273dded5ea53aa62965a
Parents: 3df3fa3
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 24 17:51:49 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 24 17:54:10 2016 +0000
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/FileSystem.java | 6 ++++--
.../apache/hadoop/security/AccessControlException.java | 2 +-
.../main/java/org/apache/hadoop/security/Credentials.java | 2 +-
.../java/org/apache/hadoop/security/SecurityUtil.java | 10 ++++++++--
.../org/apache/hadoop/security/UserGroupInformation.java | 6 +++---
.../hadoop/security/authorize/AuthorizationException.java | 2 +-
.../main/java/org/apache/hadoop/security/token/Token.java | 2 +-
.../delegation/AbstractDelegationTokenIdentifier.java | 2 +-
.../delegation/AbstractDelegationTokenSecretManager.java | 2 +-
9 files changed, 21 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01665e45/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index f581f61..55cd97e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -383,7 +383,8 @@ public abstract class FileSystem extends Configured implements Closeable {
* if the filesystem does not implement tokens
* @see SecurityUtil#buildDTServiceName(URI, int)
*/
- @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
public String getCanonicalServiceName() {
return (getChildFileSystems() == null)
? SecurityUtil.buildDTServiceName(getUri(), getDefaultPort())
@@ -617,7 +618,8 @@ public abstract class FileSystem extends Configured implements Closeable {
* @return list of new delegation tokens
* @throws IOException problems obtaining a token
*/
- @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
public Token<?>[] addDelegationTokens(
final String renewer, Credentials credentials) throws IOException {
if (credentials == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01665e45/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AccessControlException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AccessControlException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AccessControlException.java
index b1ab829..d0a3620 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AccessControlException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AccessControlException.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability;
/**
* An exception class for access control related issues.
*/
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceAudience.Public
@InterfaceStability.Evolving
public class AccessControlException extends IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01665e45/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
index 8e12ef1..1283d8f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
@@ -57,7 +57,7 @@ import org.apache.hadoop.security.proto.SecurityProtos.CredentialsProto;
* A class that provides the facilities of reading and writing
* secret keys and Tokens.
*/
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceAudience.Public
@InterfaceStability.Evolving
public class Credentials implements Writable {
private static final Log LOG = LogFactory.getLog(Credentials.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01665e45/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 42abe0e..b7d1ec0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -58,14 +58,20 @@ import sun.net.util.IPAddressUtil;
import com.google.common.annotations.VisibleForTesting;
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+/**
+ * Security Utils.
+ */
+@InterfaceAudience.Public
@InterfaceStability.Evolving
-public class SecurityUtil {
+public final class SecurityUtil {
public static final Log LOG = LogFactory.getLog(SecurityUtil.class);
public static final String HOSTNAME_PATTERN = "_HOST";
public static final String FAILED_TO_GET_UGI_MSG_HEADER =
"Failed to obtain user group information:";
+ private SecurityUtil() {
+ }
+
// controls whether buildTokenService will use an ip or host/ip as given
// by the user
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01665e45/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 82603a4..cf240ff 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -87,7 +87,7 @@ import org.slf4j.LoggerFactory;
* user's username and groups. It supports both the Windows, Unix and Kerberos
* login modules.
*/
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce", "HBase", "Hive", "Oozie"})
+@InterfaceAudience.Public
@InterfaceStability.Evolving
public class UserGroupInformation {
@VisibleForTesting
@@ -840,7 +840,7 @@ public class UserGroupInformation {
/**
* Log in a user using the given subject
- * @parma subject the subject to use when logging in a user, or null to
+ * @param subject the subject to use when logging in a user, or null to
* create a new subject.
* @throws IOException if login fails
*/
@@ -1694,7 +1694,7 @@ public class UserGroupInformation {
}
/**
- * Get the group names for this user. {@ #getGroups(String)} is less
+ * Get the group names for this user. {@link #getGroups()} is less
* expensive alternative when checking for a contained element.
* @return the list of users with the primary group first. If the command
* fails, it returns an empty list.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01665e45/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AuthorizationException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AuthorizationException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AuthorizationException.java
index 07f0321..03f4d99 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AuthorizationException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AuthorizationException.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.security.AccessControlException;
*
* This class <em>does not</em> provide the stack trace for security purposes.
*/
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce", "YARN"})
+@InterfaceAudience.Public
@InterfaceStability.Evolving
public class AuthorizationException extends AccessControlException {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01665e45/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
index 713fb20..99cc8c7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
@@ -504,7 +504,7 @@ public class Token<T extends TokenIdentifier> implements Writable {
* A trivial renewer for token kinds that aren't managed. Sub-classes need
* to implement getKind for their token kind.
*/
- @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+ @InterfaceAudience.Public
@InterfaceStability.Evolving
public static class TrivialRenewer extends TokenRenewer {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01665e45/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
index 0884cfd..6dfe52a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import com.google.common.annotations.VisibleForTesting;
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceAudience.Public
@InterfaceStability.Evolving
public abstract class AbstractDelegationTokenIdentifier
extends TokenIdentifier {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01665e45/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
index 0e311dd..21d3dd6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.util.Time;
import com.google.common.base.Preconditions;
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce", "Hive"})
+@InterfaceAudience.Public
@InterfaceStability.Evolving
public abstract
class AbstractDelegationTokenSecretManager<TokenIdent
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/48] hadoop git commit: YARN-5851. TestContainerManagerSecurity
testContainerManager[1] failed (Contributed by Haibo Chen via Daniel
Templeton)
Posted by li...@apache.org.
YARN-5851. TestContainerManagerSecurity testContainerManager[1] failed (Contributed by Haibo Chen via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd9a96cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd9a96cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd9a96cb
Branch: refs/heads/HADOOP-13345
Commit: dd9a96cb175d63f7c5909cd98f2dc9af267a5864
Parents: 3b9d3ac
Author: Daniel Templeton <te...@apache.org>
Authored: Tue Nov 29 14:22:19 2016 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Tue Nov 29 14:22:19 2016 -0800
----------------------------------------------------------------------
.../apache/hadoop/yarn/server/TestContainerManagerSecurity.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd9a96cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 408c1cc..98cb365 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -105,6 +105,7 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
testRootDir.mkdirs();
httpSpnegoKeytabFile.deleteOnExit();
getKdc().createPrincipal(httpSpnegoKeytabFile, httpSpnegoPrincipal);
+ UserGroupInformation.setConfiguration(conf);
yarnCluster =
new MiniYARNCluster(TestContainerManagerSecurity.class.getName(), 1, 1,
@@ -148,7 +149,6 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
public TestContainerManagerSecurity(Configuration conf) {
conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L);
- UserGroupInformation.setConfiguration(conf);
this.conf = conf;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/48] hadoop git commit: MAPREDUCE-6743. nativetask unit tests need
to provide usable output; fix link errors during mvn test
Posted by li...@apache.org.
MAPREDUCE-6743. nativetask unit tests need to provide usable output; fix link errors during mvn test
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be88d574
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be88d574
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be88d574
Branch: refs/heads/HADOOP-13345
Commit: be88d574a9a49f384d4e80a661b1a7446ba6a92e
Parents: 62b42ef
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Nov 28 15:24:44 2016 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Nov 28 15:24:44 2016 -0800
----------------------------------------------------------------------
.../hadoop-mapreduce-client-nativetask/pom.xml | 10 +-
.../src/CMakeLists.txt | 2 +
.../src/main/native/gtest/gtest-all.cc | 1483 ++++++++---
.../src/main/native/gtest/include/gtest/gtest.h | 2341 +++++++++++++-----
4 files changed, 2892 insertions(+), 944 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be88d574/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 89b23cc..5f33843 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -207,12 +207,16 @@
<phase>test</phase>
<goals><goal>cmake-test</goal></goals>
<configuration>
- <testName>test-native-task</testName>
+ <!-- this should match the xml name without the TEST-part down below -->
+ <testName>nativetask-nttest</testName>
<workingDirectory>${project.build.directory}/native/test</workingDirectory>
<source>${basedir}/src</source>
<binary>${project.build.directory}/native/test/nttest</binary>
- <args><arg>--gtest_filter=-Perf.</arg></args>
- <results>${project.build.directory}/native/test/native-results</results>
+ <args>
+ <arg>--gtest_filter=-Perf.</arg>
+ <arg>--gtest_output=xml:${project.build.directory}/surefire-reports/TEST-nativetask-nttest.xml</arg>
+ </args>
+ <results>${project.build.directory}/surefire-reports</results>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be88d574/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index f878a94..ee02062 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -98,7 +98,9 @@ include_directories(
# add gtest as system library to suppress gcc warnings
include_directories(SYSTEM ${SRC}/gtest/include)
+set(CMAKE_MACOSX_RPATH TRUE)
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
# macosx does not have -lrt
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be88d574/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest-all.cc
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest-all.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest-all.cc
index a9a03b2..4f8c08a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest-all.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest-all.cc
@@ -316,6 +316,8 @@ class GTEST_API_ SingleFailureChecker {
#include <algorithm>
#include <iomanip>
#include <limits>
+#include <list>
+#include <map>
#include <ostream> // NOLINT
#include <sstream>
#include <vector>
@@ -350,6 +352,7 @@ class GTEST_API_ SingleFailureChecker {
#elif GTEST_OS_WINDOWS_MOBILE // We are on Windows CE.
# include <windows.h> // NOLINT
+# undef min
#elif GTEST_OS_WINDOWS // We are on Windows proper.
@@ -372,6 +375,7 @@ class GTEST_API_ SingleFailureChecker {
// cpplint thinks that the header is already included, so we want to
// silence it.
# include <windows.h> // NOLINT
+# undef min
#else
@@ -394,6 +398,8 @@ class GTEST_API_ SingleFailureChecker {
#if GTEST_CAN_STREAM_RESULTS_
# include <arpa/inet.h> // NOLINT
# include <netdb.h> // NOLINT
+# include <sys/socket.h> // NOLINT
+# include <sys/types.h> // NOLINT
#endif
// Indicates that this translation unit is part of Google Test's
@@ -444,7 +450,7 @@ class GTEST_API_ SingleFailureChecker {
// GTEST_IMPLEMENTATION_ is defined to 1 iff the current translation unit is
// part of Google Test's implementation; otherwise it's undefined.
#if !GTEST_IMPLEMENTATION_
-// A user is trying to include this from his code - just say no.
+// If this file is included from the user's code, just say no.
# error "gtest-internal-inl.h is part of Google Test's internal implementation."
# error "It must not be included except by Google Test itself."
#endif // GTEST_IMPLEMENTATION_
@@ -501,6 +507,7 @@ const char kShuffleFlag[] = "shuffle";
const char kStackTraceDepthFlag[] = "stack_trace_depth";
const char kStreamResultToFlag[] = "stream_result_to";
const char kThrowOnFailureFlag[] = "throw_on_failure";
+const char kFlagfileFlag[] = "flagfile";
// A valid random seed must be in [1, kMaxRandomSeed].
const int kMaxRandomSeed = 99999;
@@ -833,6 +840,10 @@ class OsStackTraceGetterInterface {
// CurrentStackTrace() will use to find and hide Google Test stack frames.
virtual void UponLeavingGTest() = 0;
+ // This string is inserted in place of stack frames that are part of
+ // Google Test's implementation.
+ static const char* const kElidedFramesMarker;
+
private:
GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetterInterface);
};
@@ -840,26 +851,12 @@ class OsStackTraceGetterInterface {
// A working implementation of the OsStackTraceGetterInterface interface.
class OsStackTraceGetter : public OsStackTraceGetterInterface {
public:
- OsStackTraceGetter() : caller_frame_(NULL) {}
-
- virtual string CurrentStackTrace(int max_depth, int skip_count)
- GTEST_LOCK_EXCLUDED_(mutex_);
-
- virtual void UponLeavingGTest() GTEST_LOCK_EXCLUDED_(mutex_);
+ OsStackTraceGetter() {}
- // This string is inserted in place of stack frames that are part of
- // Google Test's implementation.
- static const char* const kElidedFramesMarker;
+ virtual string CurrentStackTrace(int max_depth, int skip_count);
+ virtual void UponLeavingGTest();
private:
- Mutex mutex_; // protects all internal state
-
- // We save the stack frame below the frame that calls user code.
- // We do this because the address of the frame immediately below
- // the user code changes between the call to UponLeavingGTest()
- // and any calls to CurrentStackTrace() from within the user code.
- void* caller_frame_;
-
GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetter);
};
@@ -1369,32 +1366,6 @@ GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv);
// platform.
GTEST_API_ std::string GetLastErrnoDescription();
-# if GTEST_OS_WINDOWS
-// Provides leak-safe Windows kernel handle ownership.
-class AutoHandle {
- public:
- AutoHandle() : handle_(INVALID_HANDLE_VALUE) {}
- explicit AutoHandle(HANDLE handle) : handle_(handle) {}
-
- ~AutoHandle() { Reset(); }
-
- HANDLE Get() const { return handle_; }
- void Reset() { Reset(INVALID_HANDLE_VALUE); }
- void Reset(HANDLE handle) {
- if (handle != handle_) {
- if (handle_ != INVALID_HANDLE_VALUE)
- ::CloseHandle(handle_);
- handle_ = handle;
- }
- }
-
- private:
- HANDLE handle_;
-
- GTEST_DISALLOW_COPY_AND_ASSIGN_(AutoHandle);
-};
-# endif // GTEST_OS_WINDOWS
-
// Attempts to parse a string into a positive integer pointed to by the
// number parameter. Returns true if that is possible.
// GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we can use
@@ -1468,7 +1439,7 @@ class TestResultAccessor {
#if GTEST_CAN_STREAM_RESULTS_
// Streams test results to the given port on the given host machine.
-class StreamingListener : public EmptyTestEventListener {
+class GTEST_API_ StreamingListener : public EmptyTestEventListener {
public:
// Abstract base class for writing strings to a socket.
class AbstractSocketWriter {
@@ -1667,6 +1638,12 @@ bool g_help_flag = false;
} // namespace internal
static const char* GetDefaultFilter() {
+#ifdef GTEST_TEST_FILTER_ENV_VAR_
+ const char* const testbridge_test_only = getenv(GTEST_TEST_FILTER_ENV_VAR_);
+ if (testbridge_test_only != NULL) {
+ return testbridge_test_only;
+ }
+#endif // GTEST_TEST_FILTER_ENV_VAR_
return kUniversalFilter;
}
@@ -1767,6 +1744,13 @@ GTEST_DEFINE_bool_(
"if exceptions are enabled or exit the program with a non-zero code "
"otherwise.");
+#if GTEST_USE_OWN_FLAGFILE_FLAG_
+GTEST_DEFINE_string_(
+ flagfile,
+ internal::StringFromGTestEnv("flagfile", ""),
+ "This flag specifies the flagfile to read command-line flags from.");
+#endif // GTEST_USE_OWN_FLAGFILE_FLAG_
+
namespace internal {
// Generates a random number from [0, range), using a Linear
@@ -1791,13 +1775,7 @@ UInt32 Random::Generate(UInt32 range) {
// GTestIsInitialized() returns true iff the user has initialized
// Google Test. Useful for catching the user mistake of not initializing
// Google Test before calling RUN_ALL_TESTS().
-//
-// A user must call testing::InitGoogleTest() to initialize Google
-// Test. g_init_gtest_count is set to the number of times
-// InitGoogleTest() has been called. We don't protect this variable
-// under a mutex as it is only accessed in the main thread.
-GTEST_API_ int g_init_gtest_count = 0;
-static bool GTestIsInitialized() { return g_init_gtest_count != 0; }
+static bool GTestIsInitialized() { return GetArgvs().size() > 0; }
// Iterates over a vector of TestCases, keeping a running sum of the
// results of calling a given int-returning method on each.
@@ -1853,8 +1831,16 @@ void AssertHelper::operator=(const Message& message) const {
// Mutex for linked pointers.
GTEST_API_ GTEST_DEFINE_STATIC_MUTEX_(g_linked_ptr_mutex);
-// Application pathname gotten in InitGoogleTest.
-std::string g_executable_path;
+// A copy of all command line arguments. Set by InitGoogleTest().
+::std::vector<testing::internal::string> g_argvs;
+
+const ::std::vector<testing::internal::string>& GetArgvs() {
+#if defined(GTEST_CUSTOM_GET_ARGVS_)
+ return GTEST_CUSTOM_GET_ARGVS_();
+#else // defined(GTEST_CUSTOM_GET_ARGVS_)
+ return g_argvs;
+#endif // defined(GTEST_CUSTOM_GET_ARGVS_)
+}
// Returns the current application's name, removing directory path if that
// is present.
@@ -1862,9 +1848,9 @@ FilePath GetCurrentExecutableName() {
FilePath result;
#if GTEST_OS_WINDOWS
- result.Set(FilePath(g_executable_path).RemoveExtension("exe"));
+ result.Set(FilePath(GetArgvs()[0]).RemoveExtension("exe"));
#else
- result.Set(FilePath(g_executable_path));
+ result.Set(FilePath(GetArgvs()[0]));
#endif // GTEST_OS_WINDOWS
return result.RemoveDirectoryName();
@@ -2256,8 +2242,12 @@ int UnitTestImpl::test_to_run_count() const {
// CurrentOsStackTraceExceptTop(1), Foo() will be included in the
// trace but Bar() and CurrentOsStackTraceExceptTop() won't.
std::string UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) {
- (void)skip_count;
- return "";
+ return os_stack_trace_getter()->CurrentStackTrace(
+ static_cast<int>(GTEST_FLAG(stack_trace_depth)),
+ skip_count + 1
+ // Skips the user-specified number of frames plus this function
+ // itself.
+ ); // NOLINT
}
// Returns the current time in milliseconds.
@@ -2286,21 +2276,13 @@ TimeInMillis GetTimeInMillis() {
#elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_
__timeb64 now;
-# ifdef _MSC_VER
-
// MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996
// (deprecated function) there.
// TODO(kenton@google.com): Use GetTickCount()? Or use
// SystemTimeToFileTime()
-# pragma warning(push) // Saves the current warning state.
-# pragma warning(disable:4996) // Temporarily disables warning 4996.
- _ftime64(&now);
-# pragma warning(pop) // Restores the warning state.
-# else
-
+ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996)
_ftime64(&now);
-
-# endif // _MSC_VER
+ GTEST_DISABLE_MSC_WARNINGS_POP_()
return static_cast<TimeInMillis>(now.time) * 1000 + now.millitm;
#elif GTEST_HAS_GETTIMEOFDAY_
@@ -2385,6 +2367,23 @@ static void StreamWideCharsToMessage(const wchar_t* wstr, size_t length,
#endif // GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING
+void SplitString(const ::std::string& str, char delimiter,
+ ::std::vector< ::std::string>* dest) {
+ ::std::vector< ::std::string> parsed;
+ ::std::string::size_type pos = 0;
+ while (::testing::internal::AlwaysTrue()) {
+ const ::std::string::size_type colon = str.find(delimiter, pos);
+ if (colon == ::std::string::npos) {
+ parsed.push_back(str.substr(pos));
+ break;
+ } else {
+ parsed.push_back(str.substr(pos, colon - pos));
+ pos = colon + 1;
+ }
+ }
+ dest->swap(parsed);
+}
+
} // namespace internal
// Constructs an empty Message.
@@ -2440,6 +2439,13 @@ AssertionResult::AssertionResult(const AssertionResult& other)
static_cast< ::std::string*>(NULL)) {
}
+// Swaps two AssertionResults.
+void AssertionResult::swap(AssertionResult& other) {
+ using std::swap;
+ swap(success_, other.success_);
+ swap(message_, other.message_);
+}
+
// Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE.
AssertionResult AssertionResult::operator!() const {
AssertionResult negation(!success_);
@@ -2466,6 +2472,276 @@ AssertionResult AssertionFailure(const Message& message) {
namespace internal {
+namespace edit_distance {
+std::vector<EditType> CalculateOptimalEdits(const std::vector<size_t>& left,
+ const std::vector<size_t>& right) {
+ std::vector<std::vector<double> > costs(
+ left.size() + 1, std::vector<double>(right.size() + 1));
+ std::vector<std::vector<EditType> > best_move(
+ left.size() + 1, std::vector<EditType>(right.size() + 1));
+
+ // Populate for empty right.
+ for (size_t l_i = 0; l_i < costs.size(); ++l_i) {
+ costs[l_i][0] = static_cast<double>(l_i);
+ best_move[l_i][0] = kRemove;
+ }
+ // Populate for empty left.
+ for (size_t r_i = 1; r_i < costs[0].size(); ++r_i) {
+ costs[0][r_i] = static_cast<double>(r_i);
+ best_move[0][r_i] = kAdd;
+ }
+
+ for (size_t l_i = 0; l_i < left.size(); ++l_i) {
+ for (size_t r_i = 0; r_i < right.size(); ++r_i) {
+ if (left[l_i] == right[r_i]) {
+ // Found a match. Consume it.
+ costs[l_i + 1][r_i + 1] = costs[l_i][r_i];
+ best_move[l_i + 1][r_i + 1] = kMatch;
+ continue;
+ }
+
+ const double add = costs[l_i + 1][r_i];
+ const double remove = costs[l_i][r_i + 1];
+ const double replace = costs[l_i][r_i];
+ if (add < remove && add < replace) {
+ costs[l_i + 1][r_i + 1] = add + 1;
+ best_move[l_i + 1][r_i + 1] = kAdd;
+ } else if (remove < add && remove < replace) {
+ costs[l_i + 1][r_i + 1] = remove + 1;
+ best_move[l_i + 1][r_i + 1] = kRemove;
+ } else {
+ // We make replace a little more expensive than add/remove to lower
+ // their priority.
+ costs[l_i + 1][r_i + 1] = replace + 1.00001;
+ best_move[l_i + 1][r_i + 1] = kReplace;
+ }
+ }
+ }
+
+ // Reconstruct the best path. We do it in reverse order.
+ std::vector<EditType> best_path;
+ for (size_t l_i = left.size(), r_i = right.size(); l_i > 0 || r_i > 0;) {
+ EditType move = best_move[l_i][r_i];
+ best_path.push_back(move);
+ l_i -= move != kAdd;
+ r_i -= move != kRemove;
+ }
+ std::reverse(best_path.begin(), best_path.end());
+ return best_path;
+}
+
+namespace {
+
+// Helper class to convert string into ids with deduplication.
+class InternalStrings {
+ public:
+ size_t GetId(const std::string& str) {
+ IdMap::iterator it = ids_.find(str);
+ if (it != ids_.end()) return it->second;
+ size_t id = ids_.size();
+ return ids_[str] = id;
+ }
+
+ private:
+ typedef std::map<std::string, size_t> IdMap;
+ IdMap ids_;
+};
+
+} // namespace
+
+std::vector<EditType> CalculateOptimalEdits(
+ const std::vector<std::string>& left,
+ const std::vector<std::string>& right) {
+ std::vector<size_t> left_ids, right_ids;
+ {
+ InternalStrings intern_table;
+ for (size_t i = 0; i < left.size(); ++i) {
+ left_ids.push_back(intern_table.GetId(left[i]));
+ }
+ for (size_t i = 0; i < right.size(); ++i) {
+ right_ids.push_back(intern_table.GetId(right[i]));
+ }
+ }
+ return CalculateOptimalEdits(left_ids, right_ids);
+}
+
+namespace {
+
+// Helper class that holds the state for one hunk and prints it out to the
+// stream.
+// It reorders adds/removes when possible to group all removes before all
+// adds. It also adds the hunk header before printint into the stream.
+class Hunk {
+ public:
+ Hunk(size_t left_start, size_t right_start)
+ : left_start_(left_start),
+ right_start_(right_start),
+ adds_(),
+ removes_(),
+ common_() {}
+
+ void PushLine(char edit, const char* line) {
+ switch (edit) {
+ case ' ':
+ ++common_;
+ FlushEdits();
+ hunk_.push_back(std::make_pair(' ', line));
+ break;
+ case '-':
+ ++removes_;
+ hunk_removes_.push_back(std::make_pair('-', line));
+ break;
+ case '+':
+ ++adds_;
+ hunk_adds_.push_back(std::make_pair('+', line));
+ break;
+ }
+ }
+
+ void PrintTo(std::ostream* os) {
+ PrintHeader(os);
+ FlushEdits();
+ for (std::list<std::pair<char, const char*> >::const_iterator it =
+ hunk_.begin();
+ it != hunk_.end(); ++it) {
+ *os << it->first << it->second << "\n";
+ }
+ }
+
+ bool has_edits() const { return adds_ || removes_; }
+
+ private:
+ void FlushEdits() {
+ hunk_.splice(hunk_.end(), hunk_removes_);
+ hunk_.splice(hunk_.end(), hunk_adds_);
+ }
+
+ // Print a unified diff header for one hunk.
+ // The format is
+ // "@@ -<left_start>,<left_length> +<right_start>,<right_length> @@"
+ // where the left/right parts are ommitted if unnecessary.
+ void PrintHeader(std::ostream* ss) const {
+ *ss << "@@ ";
+ if (removes_) {
+ *ss << "-" << left_start_ << "," << (removes_ + common_);
+ }
+ if (removes_ && adds_) {
+ *ss << " ";
+ }
+ if (adds_) {
+ *ss << "+" << right_start_ << "," << (adds_ + common_);
+ }
+ *ss << " @@\n";
+ }
+
+ size_t left_start_, right_start_;
+ size_t adds_, removes_, common_;
+ std::list<std::pair<char, const char*> > hunk_, hunk_adds_, hunk_removes_;
+};
+
+} // namespace
+
+// Create a list of diff hunks in Unified diff format.
+// Each hunk has a header generated by PrintHeader above plus a body with
+// lines prefixed with ' ' for no change, '-' for deletion and '+' for
+// addition.
+// 'context' represents the desired unchanged prefix/suffix around the diff.
+// If two hunks are close enough that their contexts overlap, then they are
+// joined into one hunk.
+std::string CreateUnifiedDiff(const std::vector<std::string>& left,
+ const std::vector<std::string>& right,
+ size_t context) {
+ const std::vector<EditType> edits = CalculateOptimalEdits(left, right);
+
+ size_t l_i = 0, r_i = 0, edit_i = 0;
+ std::stringstream ss;
+ while (edit_i < edits.size()) {
+ // Find first edit.
+ while (edit_i < edits.size() && edits[edit_i] == kMatch) {
+ ++l_i;
+ ++r_i;
+ ++edit_i;
+ }
+
+ // Find the first line to include in the hunk.
+ const size_t prefix_context = std::min(l_i, context);
+ Hunk hunk(l_i - prefix_context + 1, r_i - prefix_context + 1);
+ for (size_t i = prefix_context; i > 0; --i) {
+ hunk.PushLine(' ', left[l_i - i].c_str());
+ }
+
+ // Iterate the edits until we found enough suffix for the hunk or the input
+ // is over.
+ size_t n_suffix = 0;
+ for (; edit_i < edits.size(); ++edit_i) {
+ if (n_suffix >= context) {
+ // Continue only if the next hunk is very close.
+ std::vector<EditType>::const_iterator it = edits.begin() + edit_i;
+ while (it != edits.end() && *it == kMatch) ++it;
+ if (it == edits.end() || (it - edits.begin()) - edit_i >= context) {
+ // There is no next edit or it is too far away.
+ break;
+ }
+ }
+
+ EditType edit = edits[edit_i];
+ // Reset count when a non match is found.
+ n_suffix = edit == kMatch ? n_suffix + 1 : 0;
+
+ if (edit == kMatch || edit == kRemove || edit == kReplace) {
+ hunk.PushLine(edit == kMatch ? ' ' : '-', left[l_i].c_str());
+ }
+ if (edit == kAdd || edit == kReplace) {
+ hunk.PushLine('+', right[r_i].c_str());
+ }
+
+ // Advance indices, depending on edit type.
+ l_i += edit != kAdd;
+ r_i += edit != kRemove;
+ }
+
+ if (!hunk.has_edits()) {
+ // We are done. We don't want this hunk.
+ break;
+ }
+
+ hunk.PrintTo(&ss);
+ }
+ return ss.str();
+}
+
+} // namespace edit_distance
+
+namespace {
+
+// The string representation of the values received in EqFailure() are already
+// escaped. Split them on escaped '\n' boundaries. Leave all other escaped
+// characters the same.
+std::vector<std::string> SplitEscapedString(const std::string& str) {
+ std::vector<std::string> lines;
+ size_t start = 0, end = str.size();
+ if (end > 2 && str[0] == '"' && str[end - 1] == '"') {
+ ++start;
+ --end;
+ }
+ bool escaped = false;
+ for (size_t i = start; i + 1 < end; ++i) {
+ if (escaped) {
+ escaped = false;
+ if (str[i] == 'n') {
+ lines.push_back(str.substr(start, i - start - 1));
+ start = i + 1;
+ }
+ } else {
+ escaped = str[i] == '\\';
+ }
+ }
+ lines.push_back(str.substr(start, end - start));
+ return lines;
+}
+
+} // namespace
+
// Constructs and returns the message for an equality assertion
// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure.
//
@@ -2473,31 +2749,42 @@ namespace internal {
// and their values, as strings. For example, for ASSERT_EQ(foo, bar)
// where foo is 5 and bar is 6, we have:
//
-// expected_expression: "foo"
-// actual_expression: "bar"
-// expected_value: "5"
-// actual_value: "6"
+// lhs_expression: "foo"
+// rhs_expression: "bar"
+// lhs_value: "5"
+// rhs_value: "6"
//
// The ignoring_case parameter is true iff the assertion is a
-// *_STRCASEEQ*. When it's true, the string " (ignoring case)" will
+// *_STRCASEEQ*. When it's true, the string "Ignoring case" will
// be inserted into the message.
-AssertionResult EqFailure(const char* expected_expression,
- const char* actual_expression,
- const std::string& expected_value,
- const std::string& actual_value,
+AssertionResult EqFailure(const char* lhs_expression,
+ const char* rhs_expression,
+ const std::string& lhs_value,
+ const std::string& rhs_value,
bool ignoring_case) {
Message msg;
- msg << "Value of: " << actual_expression;
- if (actual_value != actual_expression) {
- msg << "\n Actual: " << actual_value;
+ msg << " Expected: " << lhs_expression;
+ if (lhs_value != lhs_expression) {
+ msg << "\n Which is: " << lhs_value;
+ }
+ msg << "\nTo be equal to: " << rhs_expression;
+ if (rhs_value != rhs_expression) {
+ msg << "\n Which is: " << rhs_value;
}
- msg << "\nExpected: " << expected_expression;
if (ignoring_case) {
- msg << " (ignoring case)";
+ msg << "\nIgnoring case";
}
- if (expected_value != expected_expression) {
- msg << "\nWhich is: " << expected_value;
+
+ if (!lhs_value.empty() && !rhs_value.empty()) {
+ const std::vector<std::string> lhs_lines =
+ SplitEscapedString(lhs_value);
+ const std::vector<std::string> rhs_lines =
+ SplitEscapedString(rhs_value);
+ if (lhs_lines.size() > 1 || rhs_lines.size() > 1) {
+ msg << "\nWith diff:\n"
+ << edit_distance::CreateUnifiedDiff(lhs_lines, rhs_lines);
+ }
}
return AssertionFailure() << msg;
@@ -2595,18 +2882,18 @@ namespace internal {
// The helper function for {ASSERT|EXPECT}_EQ with int or enum
// arguments.
-AssertionResult CmpHelperEQ(const char* expected_expression,
- const char* actual_expression,
- BiggestInt expected,
- BiggestInt actual) {
- if (expected == actual) {
+AssertionResult CmpHelperEQ(const char* lhs_expression,
+ const char* rhs_expression,
+ BiggestInt lhs,
+ BiggestInt rhs) {
+ if (lhs == rhs) {
return AssertionSuccess();
}
- return EqFailure(expected_expression,
- actual_expression,
- FormatForComparisonFailureMessage(expected, actual),
- FormatForComparisonFailureMessage(actual, expected),
+ return EqFailure(lhs_expression,
+ rhs_expression,
+ FormatForComparisonFailureMessage(lhs, rhs),
+ FormatForComparisonFailureMessage(rhs, lhs),
false);
}
@@ -2645,34 +2932,34 @@ GTEST_IMPL_CMP_HELPER_(GT, > )
#undef GTEST_IMPL_CMP_HELPER_
// The helper function for {ASSERT|EXPECT}_STREQ.
-AssertionResult CmpHelperSTREQ(const char* expected_expression,
- const char* actual_expression,
- const char* expected,
- const char* actual) {
- if (String::CStringEquals(expected, actual)) {
+AssertionResult CmpHelperSTREQ(const char* lhs_expression,
+ const char* rhs_expression,
+ const char* lhs,
+ const char* rhs) {
+ if (String::CStringEquals(lhs, rhs)) {
return AssertionSuccess();
}
- return EqFailure(expected_expression,
- actual_expression,
- PrintToString(expected),
- PrintToString(actual),
+ return EqFailure(lhs_expression,
+ rhs_expression,
+ PrintToString(lhs),
+ PrintToString(rhs),
false);
}
// The helper function for {ASSERT|EXPECT}_STRCASEEQ.
-AssertionResult CmpHelperSTRCASEEQ(const char* expected_expression,
- const char* actual_expression,
- const char* expected,
- const char* actual) {
- if (String::CaseInsensitiveCStringEquals(expected, actual)) {
+AssertionResult CmpHelperSTRCASEEQ(const char* lhs_expression,
+ const char* rhs_expression,
+ const char* lhs,
+ const char* rhs) {
+ if (String::CaseInsensitiveCStringEquals(lhs, rhs)) {
return AssertionSuccess();
}
- return EqFailure(expected_expression,
- actual_expression,
- PrintToString(expected),
- PrintToString(actual),
+ return EqFailure(lhs_expression,
+ rhs_expression,
+ PrintToString(lhs),
+ PrintToString(rhs),
true);
}
@@ -3027,18 +3314,18 @@ bool String::WideCStringEquals(const wchar_t * lhs, const wchar_t * rhs) {
}
// Helper function for *_STREQ on wide strings.
-AssertionResult CmpHelperSTREQ(const char* expected_expression,
- const char* actual_expression,
- const wchar_t* expected,
- const wchar_t* actual) {
- if (String::WideCStringEquals(expected, actual)) {
+AssertionResult CmpHelperSTREQ(const char* lhs_expression,
+ const char* rhs_expression,
+ const wchar_t* lhs,
+ const wchar_t* rhs) {
+ if (String::WideCStringEquals(lhs, rhs)) {
return AssertionSuccess();
}
- return EqFailure(expected_expression,
- actual_expression,
- PrintToString(expected),
- PrintToString(actual),
+ return EqFailure(lhs_expression,
+ rhs_expression,
+ PrintToString(lhs),
+ PrintToString(rhs),
false);
}
@@ -3371,14 +3658,15 @@ int TestResult::test_property_count() const {
// Creates a Test object.
-// The c'tor saves the values of all Google Test flags.
+// The c'tor saves the states of all flags.
Test::Test()
- : gtest_flag_saver_(new internal::GTestFlagSaver) {
+ : gtest_flag_saver_(new GTEST_FLAG_SAVER_) {
}
-// The d'tor restores the values of all Google Test flags.
+// The d'tor restores the states of all flags. The actual work is
+// done by the d'tor of the gtest_flag_saver_ field, and thus not
+// visible here.
Test::~Test() {
- delete gtest_flag_saver_;
}
// Sets up the test fixture.
@@ -3447,8 +3735,8 @@ bool Test::HasSameFixtureClass() {
const bool this_is_TEST = this_fixture_id == internal::GetTestTypeId();
if (first_is_TEST || this_is_TEST) {
- // The user mixed TEST and TEST_F in this test case - we'll tell
- // him/her how to fix it.
+ // Both TEST and TEST_F appear in same test case, which is incorrect.
+ // Tell the user how to fix this.
// Gets the name of the TEST and the name of the TEST_F. Note
// that first_is_TEST and this_is_TEST cannot both be true, as
@@ -3468,8 +3756,8 @@ bool Test::HasSameFixtureClass() {
<< "want to change the TEST to TEST_F or move it to another test\n"
<< "case.";
} else {
- // The user defined two fixture classes with the same name in
- // two namespaces - we'll tell him/her how to fix it.
+ // Two fixture classes with the same name appear in two different
+ // namespaces, which is not allowed. Tell the user how to fix this.
ADD_FAILURE()
<< "All tests in the same test case must use the same test fixture\n"
<< "class. However, in test case "
@@ -3662,12 +3950,14 @@ TestInfo::TestInfo(const std::string& a_test_case_name,
const std::string& a_name,
const char* a_type_param,
const char* a_value_param,
+ internal::CodeLocation a_code_location,
internal::TypeId fixture_class_id,
internal::TestFactoryBase* factory)
: test_case_name_(a_test_case_name),
name_(a_name),
type_param_(a_type_param ? new std::string(a_type_param) : NULL),
value_param_(a_value_param ? new std::string(a_value_param) : NULL),
+ location_(a_code_location),
fixture_class_id_(fixture_class_id),
should_run_(false),
is_disabled_(false),
@@ -3691,6 +3981,7 @@ namespace internal {
// this is not a typed or a type-parameterized test.
// value_param: text representation of the test's value parameter,
// or NULL if this is not a value-parameterized test.
+// code_location: code location where the test is defined
// fixture_class_id: ID of the test fixture class
// set_up_tc: pointer to the function that sets up the test case
// tear_down_tc: pointer to the function that tears down the test case
@@ -3702,20 +3993,21 @@ TestInfo* MakeAndRegisterTestInfo(
const char* name,
const char* type_param,
const char* value_param,
+ CodeLocation code_location,
TypeId fixture_class_id,
SetUpTestCaseFunc set_up_tc,
TearDownTestCaseFunc tear_down_tc,
TestFactoryBase* factory) {
TestInfo* const test_info =
new TestInfo(test_case_name, name, type_param, value_param,
- fixture_class_id, factory);
+ code_location, fixture_class_id, factory);
GetUnitTestImpl()->AddTestInfo(set_up_tc, tear_down_tc, test_info);
return test_info;
}
#if GTEST_HAS_PARAM_TEST
void ReportInvalidTestCaseType(const char* test_case_name,
- const char* file, int line) {
+ CodeLocation code_location) {
Message errors;
errors
<< "Attempted redefinition of test case " << test_case_name << ".\n"
@@ -3727,7 +4019,9 @@ void ReportInvalidTestCaseType(const char* test_case_name,
<< "probably rename one of the classes to put the tests into different\n"
<< "test cases.";
- fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(),
+ fprintf(stderr, "%s %s",
+ FormatFileLocation(code_location.file.c_str(),
+ code_location.line).c_str(),
errors.GetString().c_str());
}
#endif // GTEST_HAS_PARAM_TEST
@@ -4038,7 +4332,8 @@ enum GTestColor {
COLOR_YELLOW
};
-#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE
+#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \
+ !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
// Returns the character attribute for the given color.
WORD GetColorAttribute(GTestColor color) {
@@ -4083,6 +4378,10 @@ bool ShouldUseColor(bool stdout_is_tty) {
String::CStringEquals(term, "xterm-256color") ||
String::CStringEquals(term, "screen") ||
String::CStringEquals(term, "screen-256color") ||
+ String::CStringEquals(term, "tmux") ||
+ String::CStringEquals(term, "tmux-256color") ||
+ String::CStringEquals(term, "rxvt-unicode") ||
+ String::CStringEquals(term, "rxvt-unicode-256color") ||
String::CStringEquals(term, "linux") ||
String::CStringEquals(term, "cygwin");
return stdout_is_tty && term_supports_color;
@@ -4106,8 +4405,9 @@ void ColoredPrintf(GTestColor color, const char* fmt, ...) {
va_list args;
va_start(args, fmt);
-#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS || GTEST_OS_IOS
- const bool use_color = false;
+#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS || \
+ GTEST_OS_IOS || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT
+ const bool use_color = AlwaysFalse();
#else
static const bool in_color_mode =
ShouldUseColor(posix::IsATTY(posix::FileNo(stdout)) != 0);
@@ -4121,7 +4421,8 @@ void ColoredPrintf(GTestColor color, const char* fmt, ...) {
return;
}
-#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE
+#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \
+ !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);
// Gets the current text color.
@@ -4696,34 +4997,39 @@ std::string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters(
// Formats the given time in milliseconds as seconds.
std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) {
::std::stringstream ss;
- ss << ms/1000.0;
+ ss << (static_cast<double>(ms) * 1e-3);
return ss.str();
}
-// Converts the given epoch time in milliseconds to a date string in the ISO
-// 8601 format, without the timezone information.
-std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms) {
- // Using non-reentrant version as localtime_r is not portable.
- time_t seconds = static_cast<time_t>(ms / 1000);
-#ifdef _MSC_VER
-# pragma warning(push) // Saves the current warning state.
-# pragma warning(disable:4996) // Temporarily disables warning 4996
- // (function or variable may be unsafe).
- const struct tm* const time_struct = localtime(&seconds); // NOLINT
-# pragma warning(pop) // Restores the warning state again.
+static bool PortableLocaltime(time_t seconds, struct tm* out) {
+#if defined(_MSC_VER)
+ return localtime_s(out, &seconds) == 0;
+#elif defined(__MINGW32__) || defined(__MINGW64__)
+ // MINGW <time.h> provides neither localtime_r nor localtime_s, but uses
+ // Windows' localtime(), which has a thread-local tm buffer.
+ struct tm* tm_ptr = localtime(&seconds); // NOLINT
+ if (tm_ptr == NULL)
+ return false;
+ *out = *tm_ptr;
+ return true;
#else
- const struct tm* const time_struct = localtime(&seconds); // NOLINT
+ return localtime_r(&seconds, out) != NULL;
#endif
- if (time_struct == NULL)
- return ""; // Invalid ms value
+}
+// Converts the given epoch time in milliseconds to a date string in the ISO
+// 8601 format, without the timezone information.
+std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms) {
+ struct tm time_struct;
+ if (!PortableLocaltime(static_cast<time_t>(ms / 1000), &time_struct))
+ return "";
// YYYY-MM-DDThh:mm:ss
- return StreamableToString(time_struct->tm_year + 1900) + "-" +
- String::FormatIntWidth2(time_struct->tm_mon + 1) + "-" +
- String::FormatIntWidth2(time_struct->tm_mday) + "T" +
- String::FormatIntWidth2(time_struct->tm_hour) + ":" +
- String::FormatIntWidth2(time_struct->tm_min) + ":" +
- String::FormatIntWidth2(time_struct->tm_sec);
+ return StreamableToString(time_struct.tm_year + 1900) + "-" +
+ String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" +
+ String::FormatIntWidth2(time_struct.tm_mday) + "T" +
+ String::FormatIntWidth2(time_struct.tm_hour) + ":" +
+ String::FormatIntWidth2(time_struct.tm_min) + ":" +
+ String::FormatIntWidth2(time_struct.tm_sec);
}
// Streams an XML CDATA section, escaping invalid CDATA sequences as needed.
@@ -4986,26 +5292,15 @@ ScopedTrace::~ScopedTrace()
// class OsStackTraceGetter
-// Returns the current OS stack trace as an std::string. Parameters:
-//
-// max_depth - the maximum number of stack frames to be included
-// in the trace.
-// skip_count - the number of top frames to be skipped; doesn't count
-// against max_depth.
-//
-string OsStackTraceGetter::CurrentStackTrace(int /* max_depth */,
- int /* skip_count */)
- GTEST_LOCK_EXCLUDED_(mutex_) {
- return "";
-}
+const char* const OsStackTraceGetterInterface::kElidedFramesMarker =
+ "... " GTEST_NAME_ " internal frames ...";
-void OsStackTraceGetter::UponLeavingGTest()
- GTEST_LOCK_EXCLUDED_(mutex_) {
+string OsStackTraceGetter::CurrentStackTrace(int /*max_depth*/,
+ int /*skip_count*/) {
+ return "";
}
-const char* const
-OsStackTraceGetter::kElidedFramesMarker =
- "... " GTEST_NAME_ " internal frames ...";
+void OsStackTraceGetter::UponLeavingGTest() {}
// A helper class that creates the premature-exit file in its
// constructor and deletes the file in its destructor.
@@ -5296,7 +5591,7 @@ void UnitTest::AddTestPartResult(
// with another testing framework) and specify the former on the
// command line for debugging.
if (GTEST_FLAG(break_on_failure)) {
-#if GTEST_OS_WINDOWS
+#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
// Using DebugBreak on Windows allows gtest to still break into a debugger
// when a failure happens and both the --gtest_break_on_failure and
// the --gtest_catch_exceptions flags are specified.
@@ -5374,7 +5669,7 @@ int UnitTest::Run() {
// process. In either case the user does not want to see pop-up dialogs
// about crashes - they are expected.
if (impl()->catch_exceptions() || in_death_test_child_process) {
-# if !GTEST_OS_WINDOWS_MOBILE
+# if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
// SetErrorMode doesn't exist on CE.
SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT |
SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX);
@@ -5477,17 +5772,10 @@ namespace internal {
UnitTestImpl::UnitTestImpl(UnitTest* parent)
: parent_(parent),
-#ifdef _MSC_VER
-# pragma warning(push) // Saves the current warning state.
-# pragma warning(disable:4355) // Temporarily disables warning 4355
- // (using this in initializer).
+ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4355 /* using this in initializer */)
default_global_test_part_result_reporter_(this),
default_per_thread_test_part_result_reporter_(this),
-# pragma warning(pop) // Restores the warning state again.
-#else
- default_global_test_part_result_reporter_(this),
- default_per_thread_test_part_result_reporter_(this),
-#endif // _MSC_VER
+ GTEST_DISABLE_MSC_WARNINGS_POP_()
global_test_part_result_repoter_(
&default_global_test_part_result_reporter_),
per_thread_test_part_result_reporter_(
@@ -5598,6 +5886,11 @@ void UnitTestImpl::PostFlagParsingInit() {
if (!post_flag_parse_init_performed_) {
post_flag_parse_init_performed_ = true;
+#if defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_)
+ // Register to send notifications about key process state changes.
+ listeners()->Append(new GTEST_CUSTOM_TEST_EVENT_LISTENER_());
+#endif // defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_)
+
#if GTEST_HAS_DEATH_TEST
InitDeathTestSubprocessControlInfo();
SuppressTestEventsIfInSubprocess();
@@ -5731,6 +6024,11 @@ bool UnitTestImpl::RunAllTests() {
#if GTEST_HAS_DEATH_TEST
in_subprocess_for_death_test = (internal_run_death_test_flag_.get() != NULL);
+# if defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_)
+ if (in_subprocess_for_death_test) {
+ GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_();
+ }
+# endif // defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_)
#endif // GTEST_HAS_DEATH_TEST
const bool should_shard = ShouldShard(kTestTotalShards, kTestShardIndex,
@@ -6067,7 +6365,11 @@ void UnitTestImpl::set_os_stack_trace_getter(
// getter, and returns it.
OsStackTraceGetterInterface* UnitTestImpl::os_stack_trace_getter() {
if (os_stack_trace_getter_ == NULL) {
+#ifdef GTEST_OS_STACK_TRACE_GETTER_
+ os_stack_trace_getter_ = new GTEST_OS_STACK_TRACE_GETTER_;
+#else
os_stack_trace_getter_ = new OsStackTraceGetter;
+#endif // GTEST_OS_STACK_TRACE_GETTER_
}
return os_stack_trace_getter_;
@@ -6366,6 +6668,58 @@ static const char kColorEncodedHelpMessage[] =
"(not one in your own code or tests), please report it to\n"
"@G<" GTEST_DEV_EMAIL_ ">@D.\n";
+bool ParseGoogleTestFlag(const char* const arg) {
+ return ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag,
+ >EST_FLAG(also_run_disabled_tests)) ||
+ ParseBoolFlag(arg, kBreakOnFailureFlag,
+ >EST_FLAG(break_on_failure)) ||
+ ParseBoolFlag(arg, kCatchExceptionsFlag,
+ >EST_FLAG(catch_exceptions)) ||
+ ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) ||
+ ParseStringFlag(arg, kDeathTestStyleFlag,
+ >EST_FLAG(death_test_style)) ||
+ ParseBoolFlag(arg, kDeathTestUseFork,
+ >EST_FLAG(death_test_use_fork)) ||
+ ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) ||
+ ParseStringFlag(arg, kInternalRunDeathTestFlag,
+ >EST_FLAG(internal_run_death_test)) ||
+ ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) ||
+ ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) ||
+ ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) ||
+ ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) ||
+ ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) ||
+ ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) ||
+ ParseInt32Flag(arg, kStackTraceDepthFlag,
+ >EST_FLAG(stack_trace_depth)) ||
+ ParseStringFlag(arg, kStreamResultToFlag,
+ >EST_FLAG(stream_result_to)) ||
+ ParseBoolFlag(arg, kThrowOnFailureFlag,
+ >EST_FLAG(throw_on_failure));
+}
+
+#if GTEST_USE_OWN_FLAGFILE_FLAG_
+void LoadFlagsFromFile(const std::string& path) {
+ FILE* flagfile = posix::FOpen(path.c_str(), "r");
+ if (!flagfile) {
+ fprintf(stderr,
+ "Unable to open file \"%s\"\n",
+ GTEST_FLAG(flagfile).c_str());
+ fflush(stderr);
+ exit(EXIT_FAILURE);
+ }
+ std::string contents(ReadEntireFile(flagfile));
+ posix::FClose(flagfile);
+ std::vector<std::string> lines;
+ SplitString(contents, '\n', &lines);
+ for (size_t i = 0; i < lines.size(); ++i) {
+ if (lines[i].empty())
+ continue;
+ if (!ParseGoogleTestFlag(lines[i].c_str()))
+ g_help_flag = true;
+ }
+}
+#endif // GTEST_USE_OWN_FLAGFILE_FLAG_
+
// Parses the command line for Google Test flags, without initializing
// other parts of Google Test. The type parameter CharType can be
// instantiated to either char or wchar_t.
@@ -6379,35 +6733,24 @@ void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) {
using internal::ParseInt32Flag;
using internal::ParseStringFlag;
- // Do we see a Google Test flag?
- if (ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag,
- >EST_FLAG(also_run_disabled_tests)) ||
- ParseBoolFlag(arg, kBreakOnFailureFlag,
- >EST_FLAG(break_on_failure)) ||
- ParseBoolFlag(arg, kCatchExceptionsFlag,
- >EST_FLAG(catch_exceptions)) ||
- ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) ||
- ParseStringFlag(arg, kDeathTestStyleFlag,
- >EST_FLAG(death_test_style)) ||
- ParseBoolFlag(arg, kDeathTestUseFork,
- >EST_FLAG(death_test_use_fork)) ||
- ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) ||
- ParseStringFlag(arg, kInternalRunDeathTestFlag,
- >EST_FLAG(internal_run_death_test)) ||
- ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) ||
- ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) ||
- ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) ||
- ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) ||
- ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) ||
- ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) ||
- ParseInt32Flag(arg, kStackTraceDepthFlag,
- >EST_FLAG(stack_trace_depth)) ||
- ParseStringFlag(arg, kStreamResultToFlag,
- >EST_FLAG(stream_result_to)) ||
- ParseBoolFlag(arg, kThrowOnFailureFlag,
- >EST_FLAG(throw_on_failure))
- ) {
- // Yes. Shift the remainder of the argv list left by one. Note
+ bool remove_flag = false;
+ if (ParseGoogleTestFlag(arg)) {
+ remove_flag = true;
+#if GTEST_USE_OWN_FLAGFILE_FLAG_
+ } else if (ParseStringFlag(arg, kFlagfileFlag, >EST_FLAG(flagfile))) {
+ LoadFlagsFromFile(GTEST_FLAG(flagfile));
+ remove_flag = true;
+#endif // GTEST_USE_OWN_FLAGFILE_FLAG_
+ } else if (arg_string == "--help" || arg_string == "-h" ||
+ arg_string == "-?" || arg_string == "/?" ||
+ HasGoogleTestFlagPrefix(arg)) {
+ // Both help flag and unrecognized Google Test flags (excluding
+ // internal ones) trigger help display.
+ g_help_flag = true;
+ }
+
+ if (remove_flag) {
+ // Shift the remainder of the argv list left by one. Note
// that argv has (*argc + 1) elements, the last one always being
// NULL. The following loop moves the trailing NULL element as
// well.
@@ -6421,12 +6764,6 @@ void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) {
// We also need to decrement the iterator as we just removed
// an element.
i--;
- } else if (arg_string == "--help" || arg_string == "-h" ||
- arg_string == "-?" || arg_string == "/?" ||
- HasGoogleTestFlagPrefix(arg)) {
- // Both help flag and unrecognized Google Test flags (excluding
- // internal ones) trigger help display.
- g_help_flag = true;
}
}
@@ -6453,24 +6790,16 @@ void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv) {
// wchar_t.
template <typename CharType>
void InitGoogleTestImpl(int* argc, CharType** argv) {
- g_init_gtest_count++;
-
// We don't want to run the initialization code twice.
- if (g_init_gtest_count != 1) return;
+ if (GTestIsInitialized()) return;
if (*argc <= 0) return;
- internal::g_executable_path = internal::StreamableToString(argv[0]);
-
-#if GTEST_HAS_DEATH_TEST
-
g_argvs.clear();
for (int i = 0; i != *argc; i++) {
g_argvs.push_back(StreamableToString(argv[i]));
}
-#endif // GTEST_HAS_DEATH_TEST
-
ParseGoogleTestFlagsOnly(argc, argv);
GetUnitTestImpl()->PostFlagParsingInit();
}
@@ -6487,13 +6816,21 @@ void InitGoogleTestImpl(int* argc, CharType** argv) {
//
// Calling the function for the second time has no user-visible effect.
void InitGoogleTest(int* argc, char** argv) {
+#if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
+ GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv);
+#else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
internal::InitGoogleTestImpl(argc, argv);
+#endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
}
// This overloaded version can be used in Windows programs compiled in
// UNICODE mode.
void InitGoogleTest(int* argc, wchar_t** argv) {
+#if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
+ GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv);
+#else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
internal::InitGoogleTestImpl(argc, argv);
+#endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
}
} // namespace testing
@@ -6563,9 +6900,9 @@ void InitGoogleTest(int* argc, wchar_t** argv) {
// Indicates that this translation unit is part of Google Test's
// implementation. It must come before gtest-internal-inl.h is
-// included, or there will be a compiler error. This trick is to
-// prevent a user from accidentally including gtest-internal-inl.h in
-// his code.
+// included, or there will be a compiler error. This trick exists to
+// prevent the accidental inclusion of gtest-internal-inl.h in the
+// user's code.
#define GTEST_IMPLEMENTATION_ 1
#undef GTEST_IMPLEMENTATION_
@@ -6614,7 +6951,9 @@ namespace internal {
// Valid only for fast death tests. Indicates the code is running in the
// child process of a fast style death test.
+# if !GTEST_OS_WINDOWS
static bool g_in_fast_death_test_child = false;
+# endif
// Returns a Boolean value indicating whether the caller is currently
// executing in the context of the death test child process. Tools such as
@@ -6663,6 +7002,14 @@ KilledBySignal::KilledBySignal(int signum) : signum_(signum) {
// KilledBySignal function-call operator.
bool KilledBySignal::operator()(int exit_status) const {
+# if defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_)
+ {
+ bool result;
+ if (GTEST_KILLED_BY_SIGNAL_OVERRIDE_(signum_, exit_status, &result)) {
+ return result;
+ }
+ }
+# endif // defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_)
return WIFSIGNALED(exit_status) && WTERMSIG(exit_status) == signum_;
}
# endif // !GTEST_OS_WINDOWS
@@ -7369,6 +7716,11 @@ class ExecDeathTest : public ForkingDeathTest {
static ::std::vector<testing::internal::string>
GetArgvsForDeathTestChildProcess() {
::std::vector<testing::internal::string> args = GetInjectableArgvs();
+# if defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_)
+ ::std::vector<testing::internal::string> extra_args =
+ GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_();
+ args.insert(args.end(), extra_args.begin(), extra_args.end());
+# endif // defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_)
return args;
}
// The name of the file in which the death test is located.
@@ -7479,6 +7831,8 @@ void StackLowerThanAddress(const void* ptr, bool* result) {
*result = (&dummy < ptr);
}
+// Make sure AddressSanitizer does not tamper with the stack here.
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
bool StackGrowsDown() {
int dummy;
bool result;
@@ -7696,26 +8050,6 @@ bool DefaultDeathTestFactory::Create(const char* statement, const RE* regex,
return true;
}
-// Splits a given string on a given delimiter, populating a given
-// vector with the fields. GTEST_HAS_DEATH_TEST implies that we have
-// ::std::string, so we can use it here.
-static void SplitString(const ::std::string& str, char delimiter,
- ::std::vector< ::std::string>* dest) {
- ::std::vector< ::std::string> parsed;
- ::std::string::size_type pos = 0;
- while (::testing::internal::AlwaysTrue()) {
- const ::std::string::size_type colon = str.find(delimiter, pos);
- if (colon == ::std::string::npos) {
- parsed.push_back(str.substr(pos));
- break;
- } else {
- parsed.push_back(str.substr(pos, colon - pos));
- pos = colon + 1;
- }
- }
- dest->swap(parsed);
-}
-
# if GTEST_OS_WINDOWS
// Recreates the pipe and event handles from the provided parameters,
// signals the event, and returns a file descriptor wrapped around the pipe
@@ -7904,7 +8238,6 @@ namespace internal {
// of them.
const char kPathSeparator = '\\';
const char kAlternatePathSeparator = '/';
-const char kPathSeparatorString[] = "\\";
const char kAlternatePathSeparatorString[] = "/";
# if GTEST_OS_WINDOWS_MOBILE
// Windows CE doesn't have a current directory. You should not use
@@ -7918,7 +8251,6 @@ const char kCurrentDirectoryString[] = ".\\";
# endif // GTEST_OS_WINDOWS_MOBILE
#else
const char kPathSeparator = '/';
-const char kPathSeparatorString[] = "/";
const char kCurrentDirectoryString[] = "./";
#endif // GTEST_OS_WINDOWS
@@ -7933,7 +8265,7 @@ static bool IsPathSeparator(char c) {
// Returns the current working directory, or "" if unsuccessful.
FilePath FilePath::GetCurrentDir() {
-#if GTEST_OS_WINDOWS_MOBILE
+#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT
// Windows CE doesn't have a current directory, so we just return
// something reasonable.
return FilePath(kCurrentDirectoryString);
@@ -7942,7 +8274,14 @@ FilePath FilePath::GetCurrentDir() {
return FilePath(_getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd);
#else
char cwd[GTEST_PATH_MAX_ + 1] = { '\0' };
- return FilePath(getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd);
+ char* result = getcwd(cwd, sizeof(cwd));
+# if GTEST_OS_NACL
+ // getcwd will likely fail in NaCl due to the sandbox, so return something
+ // reasonable. The user may have provided a shim implementation for getcwd,
+ // however, so fallback only when failure is detected.
+ return FilePath(result == NULL ? kCurrentDirectoryString : cwd);
+# endif // GTEST_OS_NACL
+ return FilePath(result == NULL ? "" : cwd);
#endif // GTEST_OS_WINDOWS_MOBILE
}
@@ -8250,15 +8589,16 @@ void FilePath::Normalize() {
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
+#include <fstream>
-#if GTEST_OS_WINDOWS_MOBILE
-# include <windows.h> // For TerminateProcess()
-#elif GTEST_OS_WINDOWS
+#if GTEST_OS_WINDOWS
+# include <windows.h>
# include <io.h>
# include <sys/stat.h>
+# include <map> // Used in ThreadLocal.
#else
# include <unistd.h>
-#endif // GTEST_OS_WINDOWS_MOBILE
+#endif // GTEST_OS_WINDOWS
#if GTEST_OS_MAC
# include <mach/mach_init.h>
@@ -8268,15 +8608,21 @@ void FilePath::Normalize() {
#if GTEST_OS_QNX
# include <devctl.h>
+# include <fcntl.h>
# include <sys/procfs.h>
#endif // GTEST_OS_QNX
+#if GTEST_OS_AIX
+# include <procinfo.h>
+# include <sys/types.h>
+#endif // GTEST_OS_AIX
+
// Indicates that this translation unit is part of Google Test's
// implementation. It must come before gtest-internal-inl.h is
-// included, or there will be a compiler error. This trick is to
-// prevent a user from accidentally including gtest-internal-inl.h in
-// his code.
+// included, or there will be a compiler error. This trick exists to
+// prevent the accidental inclusion of gtest-internal-inl.h in the
+// user's code.
#define GTEST_IMPLEMENTATION_ 1
#undef GTEST_IMPLEMENTATION_
@@ -8292,10 +8638,31 @@ const int kStdOutFileno = STDOUT_FILENO;
const int kStdErrFileno = STDERR_FILENO;
#endif // _MSC_VER
-#if GTEST_OS_MAC
+#if GTEST_OS_LINUX
+
+namespace {
+template <typename T>
+T ReadProcFileField(const string& filename, int field) {
+ std::string dummy;
+ std::ifstream file(filename.c_str());
+ while (field-- > 0) {
+ file >> dummy;
+ }
+ T output = 0;
+ file >> output;
+ return output;
+}
+} // namespace
+
+// Returns the number of active threads, or 0 when there is an error.
+size_t GetThreadCount() {
+ const string filename =
+ (Message() << "/proc/" << getpid() << "/stat").GetString();
+ return ReadProcFileField<int>(filename, 19);
+}
+
+#elif GTEST_OS_MAC
-// Returns the number of threads running in the process, or 0 to indicate that
-// we cannot detect it.
size_t GetThreadCount() {
const task_t task = mach_task_self();
mach_msg_type_number_t thread_count;
@@ -8333,6 +8700,19 @@ size_t GetThreadCount() {
}
}
+#elif GTEST_OS_AIX
+
+size_t GetThreadCount() {
+ struct procentry64 entry;
+ pid_t pid = getpid();
+ int status = getprocs64(&entry, sizeof(entry), NULL, 0, &pid, 1);
+ if (status == 1) {
+ return entry.pi_thcount;
+ } else {
+ return 0;
+ }
+}
+
#else
size_t GetThreadCount() {
@@ -8341,7 +8721,390 @@ size_t GetThreadCount() {
return 0;
}
-#endif // GTEST_OS_MAC
+#endif // GTEST_OS_LINUX
+
+#if GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS
+
+void SleepMilliseconds(int n) {
+ ::Sleep(n);
+}
+
+AutoHandle::AutoHandle()
+ : handle_(INVALID_HANDLE_VALUE) {}
+
+AutoHandle::AutoHandle(Handle handle)
+ : handle_(handle) {}
+
+AutoHandle::~AutoHandle() {
+ Reset();
+}
+
+AutoHandle::Handle AutoHandle::Get() const {
+ return handle_;
+}
+
+void AutoHandle::Reset() {
+ Reset(INVALID_HANDLE_VALUE);
+}
+
+void AutoHandle::Reset(HANDLE handle) {
+ // Resetting with the same handle we already own is invalid.
+ if (handle_ != handle) {
+ if (IsCloseable()) {
+ ::CloseHandle(handle_);
+ }
+ handle_ = handle;
+ } else {
+ GTEST_CHECK_(!IsCloseable())
+ << "Resetting a valid handle to itself is likely a programmer error "
+ "and thus not allowed.";
+ }
+}
+
+bool AutoHandle::IsCloseable() const {
+ // Different Windows APIs may use either of these values to represent an
+ // invalid handle.
+ return handle_ != NULL && handle_ != INVALID_HANDLE_VALUE;
+}
+
+Notification::Notification()
+ : event_(::CreateEvent(NULL, // Default security attributes.
+ TRUE, // Do not reset automatically.
+ FALSE, // Initially unset.
+ NULL)) { // Anonymous event.
+ GTEST_CHECK_(event_.Get() != NULL);
+}
+
+void Notification::Notify() {
+ GTEST_CHECK_(::SetEvent(event_.Get()) != FALSE);
+}
+
+void Notification::WaitForNotification() {
+ GTEST_CHECK_(
+ ::WaitForSingleObject(event_.Get(), INFINITE) == WAIT_OBJECT_0);
+}
+
+Mutex::Mutex()
+ : owner_thread_id_(0),
+ type_(kDynamic),
+ critical_section_init_phase_(0),
+ critical_section_(new CRITICAL_SECTION) {
+ ::InitializeCriticalSection(critical_section_);
+}
+
+Mutex::~Mutex() {
+ // Static mutexes are leaked intentionally. It is not thread-safe to try
+ // to clean them up.
+ // TODO(yukawa): Switch to Slim Reader/Writer (SRW) Locks, which requires
+ // nothing to clean it up but is available only on Vista and later.
+ // http://msdn.microsoft.com/en-us/library/windows/desktop/aa904937.aspx
+ if (type_ == kDynamic) {
+ ::DeleteCriticalSection(critical_section_);
+ delete critical_section_;
+ critical_section_ = NULL;
+ }
+}
+
+void Mutex::Lock() {
+ ThreadSafeLazyInit();
+ ::EnterCriticalSection(critical_section_);
+ owner_thread_id_ = ::GetCurrentThreadId();
+}
+
+void Mutex::Unlock() {
+ ThreadSafeLazyInit();
+ // We don't protect writing to owner_thread_id_ here, as it's the
+ // caller's responsibility to ensure that the current thread holds the
+ // mutex when this is called.
+ owner_thread_id_ = 0;
+ ::LeaveCriticalSection(critical_section_);
+}
+
+// Does nothing if the current thread holds the mutex. Otherwise, crashes
+// with high probability.
+void Mutex::AssertHeld() {
+ ThreadSafeLazyInit();
+ GTEST_CHECK_(owner_thread_id_ == ::GetCurrentThreadId())
+ << "The current thread is not holding the mutex @" << this;
+}
+
+// Initializes owner_thread_id_ and critical_section_ in static mutexes.
+void Mutex::ThreadSafeLazyInit() {
+ // Dynamic mutexes are initialized in the constructor.
+ if (type_ == kStatic) {
+ switch (
+ ::InterlockedCompareExchange(&critical_section_init_phase_, 1L, 0L)) {
+ case 0:
+ // If critical_section_init_phase_ was 0 before the exchange, we
+ // are the first to test it and need to perform the initialization.
+ owner_thread_id_ = 0;
+ critical_section_ = new CRITICAL_SECTION;
+ ::InitializeCriticalSection(critical_section_);
+ // Updates the critical_section_init_phase_ to 2 to signal
+ // initialization complete.
+ GTEST_CHECK_(::InterlockedCompareExchange(
+ &critical_section_init_phase_, 2L, 1L) ==
+ 1L);
+ break;
+ case 1:
+ // Somebody else is already initializing the mutex; spin until they
+ // are done.
+ while (::InterlockedCompareExchange(&critical_section_init_phase_,
+ 2L,
+ 2L) != 2L) {
+ // Possibly yields the rest of the thread's time slice to other
+ // threads.
+ ::Sleep(0);
+ }
+ break;
+
+ case 2:
+ break; // The mutex is already initialized and ready for use.
+
+ default:
+ GTEST_CHECK_(false)
+ << "Unexpected value of critical_section_init_phase_ "
+ << "while initializing a static mutex.";
+ }
+ }
+}
+
+namespace {
+
+class ThreadWithParamSupport : public ThreadWithParamBase {
+ public:
+ static HANDLE CreateThread(Runnable* runnable,
+ Notification* thread_can_start) {
+ ThreadMainParam* param = new ThreadMainParam(runnable, thread_can_start);
+ DWORD thread_id;
+ // TODO(yukawa): Consider to use _beginthreadex instead.
+ HANDLE thread_handle = ::CreateThread(
+ NULL, // Default security.
+ 0, // Default stack size.
+ &ThreadWithParamSupport::ThreadMain,
+ param, // Parameter to ThreadMainStatic
+ 0x0, // Default creation flags.
+ &thread_id); // Need a valid pointer for the call to work under Win98.
+ GTEST_CHECK_(thread_handle != NULL) << "CreateThread failed with error "
+ << ::GetLastError() << ".";
+ if (thread_handle == NULL) {
+ delete param;
+ }
+ return thread_handle;
+ }
+
+ private:
+ struct ThreadMainParam {
+ ThreadMainParam(Runnable* runnable, Notification* thread_can_start)
+ : runnable_(runnable),
+ thread_can_start_(thread_can_start) {
+ }
+ scoped_ptr<Runnable> runnable_;
+ // Does not own.
+ Notification* thread_can_start_;
+ };
+
+ static DWORD WINAPI ThreadMain(void* ptr) {
+ // Transfers ownership.
+ scoped_ptr<ThreadMainParam> param(static_cast<ThreadMainParam*>(ptr));
+ if (param->thread_can_start_ != NULL)
+ param->thread_can_start_->WaitForNotification();
+ param->runnable_->Run();
+ return 0;
+ }
+
+ // Prohibit instantiation.
+ ThreadWithParamSupport();
+
+ GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParamSupport);
+};
+
+} // namespace
+
+ThreadWithParamBase::ThreadWithParamBase(Runnable *runnable,
+ Notification* thread_can_start)
+ : thread_(ThreadWithParamSupport::CreateThread(runnable,
+ thread_can_start)) {
+}
+
+ThreadWithParamBase::~ThreadWithParamBase() {
+ Join();
+}
+
+void ThreadWithParamBase::Join() {
+ GTEST_CHECK_(::WaitForSingleObject(thread_.Get(), INFINITE) == WAIT_OBJECT_0)
+ << "Failed to join the thread with error " << ::GetLastError() << ".";
+}
+
+// Maps a thread to a set of ThreadIdToThreadLocals that have values
+// instantiated on that thread and notifies them when the thread exits. A
+// ThreadLocal instance is expected to persist until all threads it has
+// values on have terminated.
+class ThreadLocalRegistryImpl {
+ public:
+ // Registers thread_local_instance as having value on the current thread.
+ // Returns a value that can be used to identify the thread from other threads.
+ static ThreadLocalValueHolderBase* GetValueOnCurrentThread(
+ const ThreadLocalBase* thread_local_instance) {
+ DWORD current_thread = ::GetCurrentThreadId();
+ MutexLock lock(&mutex_);
+ ThreadIdToThreadLocals* const thread_to_thread_locals =
+ GetThreadLocalsMapLocked();
+ ThreadIdToThreadLocals::iterator thread_local_pos =
+ thread_to_thread_locals->find(current_thread);
+ if (thread_local_pos == thread_to_thread_locals->end()) {
+ thread_local_pos = thread_to_thread_locals->insert(
+ std::make_pair(current_thread, ThreadLocalValues())).first;
+ StartWatcherThreadFor(current_thread);
+ }
+ ThreadLocalValues& thread_local_values = thread_local_pos->second;
+ ThreadLocalValues::iterator value_pos =
+ thread_local_values.find(thread_local_instance);
+ if (value_pos == thread_local_values.end()) {
+ value_pos =
+ thread_local_values
+ .insert(std::make_pair(
+ thread_local_instance,
+ linked_ptr<ThreadLocalValueHolderBase>(
+ thread_local_instance->NewValueForCurrentThread())))
+ .first;
+ }
+ return value_pos->second.get();
+ }
+
+ static void OnThreadLocalDestroyed(
+ const ThreadLocalBase* thread_local_instance) {
+ std::vector<linked_ptr<ThreadLocalValueHolderBase> > value_holders;
+ // Clean up the ThreadLocalValues data structure while holding the lock, but
+ // defer the destruction of the ThreadLocalValueHolderBases.
+ {
+ MutexLock lock(&mutex_);
+ ThreadIdToThreadLocals* const thread_to_thread_locals =
+ GetThreadLocalsMapLocked();
+ for (ThreadIdToThreadLocals::iterator it =
+ thread_to_thread_locals->begin();
+ it != thread_to_thread_locals->end();
+ ++it) {
+ ThreadLocalValues& thread_local_values = it->second;
+ ThreadLocalValues::iterator value_pos =
+ thread_local_values.find(thread_local_instance);
+ if (value_pos != thread_local_values.end()) {
+ value_holders.push_back(value_pos->second);
+ thread_local_values.erase(value_pos);
+ // This 'if' can only be successful at most once, so theoretically we
+ // could break out of the loop here, but we don't bother doing so.
+ }
+ }
+ }
+ // Outside the lock, let the destructor for 'value_holders' deallocate the
+ // ThreadLocalValueHolderBases.
+ }
+
+ static void OnThreadExit(DWORD thread_id) {
+ GTEST_CHECK_(thread_id != 0) << ::GetLastError();
+ std::vector<linked_ptr<ThreadLocalValueHolderBase> > value_holders;
+ // Clean up the ThreadIdToThreadLocals data structure while holding the
+ // lock, but defer the destruction of the ThreadLocalValueHolderBases.
+ {
+ MutexLock lock(&mutex_);
+ ThreadIdToThreadLocals* const thread_to_thread_locals =
+ GetThreadLocalsMapLocked();
+ ThreadIdToThreadLocals::iterator thread_local_pos =
+ thread_to_thread_locals->find(thread_id);
+ if (thread_local_pos != thread_to_thread_locals->end()) {
+ ThreadLocalValues& thread_local_values = thread_local_pos->second;
+ for (ThreadLocalValues::iterator value_pos =
+ thread_local_values.begin();
+ value_pos != thread_local_values.end();
+ ++value_pos) {
+ value_holders.push_back(value_pos->second);
+ }
+ thread_to_thread_locals->erase(thread_local_pos);
+ }
+ }
+ // Outside the lock, let the destructor for 'value_holders' deallocate the
+ // ThreadLocalValueHolderBases.
+ }
+
+ private:
+ // In a particular thread, maps a ThreadLocal object to its value.
+ typedef std::map<const ThreadLocalBase*,
+ linked_ptr<ThreadLocalValueHolderBase> > ThreadLocalValues;
+ // Stores all ThreadIdToThreadLocals having values in a thread, indexed by
+ // thread's ID.
+ typedef std::map<DWORD, ThreadLocalValues> ThreadIdToThreadLocals;
+
+ // Holds the thread id and thread handle that we pass from
+ // StartWatcherThreadFor to WatcherThreadFunc.
+ typedef std::pair<DWORD, HANDLE> ThreadIdAndHandle;
+
+ static void StartWatcherThreadFor(DWORD thread_id) {
+ // The returned handle will be kept in thread_map and closed by
+ // watcher_thread in WatcherThreadFunc.
+ HANDLE thread = ::OpenThread(SYNCHRONIZE | THREAD_QUERY_INFORMATION,
+ FALSE,
+ thread_id);
+ GTEST_CHECK_(thread != NULL);
+ // We need to to pass a valid thread ID pointer into CreateThread for it
+ // to work correctly under Win98.
+ DWORD watcher_thread_id;
+ HANDLE watcher_thread = ::CreateThread(
+ NULL, // Default security.
+ 0, // Default stack size
+ &ThreadLocalRegistryImpl::WatcherThreadFunc,
+ reinterpret_cast<LPVOID>(new ThreadIdAndHandle(thread_id, thread)),
+ CREATE_SUSPENDED,
+ &watcher_thread_id);
+ GTEST_CHECK_(watcher_thread != NULL);
+ // Give the watcher thread the same priority as ours to avoid being
+ // blocked by it.
+ ::SetThreadPriority(watcher_thread,
+ ::GetThreadPriority(::GetCurrentThread()));
+ ::ResumeThread(watcher_thread);
+ ::CloseHandle(watcher_thread);
+ }
+
+ // Monitors exit from a given thread and notifies those
+ // ThreadIdToThreadLocals about thread termination.
+ static DWORD WINAPI WatcherThreadFunc(LPVOID param) {
+ const ThreadIdAndHandle* tah =
+ reinterpret_cast<const ThreadIdAndHandle*>(param);
+ GTEST_CHECK_(
+ ::WaitForSingleObject(tah->second, INFINITE) == WAIT_OBJECT_0);
+ OnThreadExit(tah->first);
+ ::CloseHandle(tah->second);
+ delete tah;
+ return 0;
+ }
+
+ // Returns map of thread local instances.
+ static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() {
+ mutex_.AssertHeld();
+ static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals;
+ return map;
+ }
+
+ // Protects access to GetThreadLocalsMapLocked() and its return value.
+ static Mutex mutex_;
+ // Protects access to GetThreadMapLocked() and its return value.
+ static Mutex thread_map_mutex_;
+};
+
+Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex);
+Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex);
+
+ThreadLocalValueHolderBase* ThreadLocalRegistry::GetValueOnCurrentThread(
+ const ThreadLocalBase* thread_local_instance) {
+ return ThreadLocalRegistryImpl::GetValueOnCurrentThread(
+ thread_local_instance);
+}
+
+void ThreadLocalRegistry::OnThreadLocalDestroyed(
+ const ThreadLocalBase* thread_local_instance) {
+ ThreadLocalRegistryImpl::OnThreadLocalDestroyed(thread_local_instance);
+}
+
+#endif // GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS
#if GTEST_USES_POSIX_RE
@@ -8691,7 +9454,6 @@ GTEST_API_ ::std::string FormatCompilerIndependentFileLocation(
return file_name + ":" + StreamableToString(line);
}
-
GTestLog::GTestLog(GTestLogSeverity severity, const char* file, int line)
: severity_(severity) {
const char* const marker =
@@ -8712,10 +9474,7 @@ GTestLog::~GTestLog() {
}
// Disable Microsoft deprecation warnings for POSIX functions called from
// this class (creat, dup, dup2, and close)
-#ifdef _MSC_VER
-# pragma warning(push)
-# pragma warning(disable: 4996)
-#endif // _MSC_VER
+GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996)
#if GTEST_HAS_STREAM_REDIRECTION
@@ -8791,12 +9550,6 @@ class CapturedStream {
}
private:
- // Reads the entire content of a file as an std::string.
- static std::string ReadEntireFile(FILE* file);
-
- // Returns the size (in bytes) of a file.
- static size_t GetFileSize(FILE* file);
-
const int fd_; // A stream to capture.
int uncaptured_fd_;
// Name of the temporary file holding the stderr output.
@@ -8805,38 +9558,7 @@ class CapturedStream {
GTEST_DISALLOW_COPY_AND_ASSIGN_(CapturedStream);
};
-// Returns the size (in bytes) of a file.
-size_t CapturedStream::GetFileSize(FILE* file) {
- fseek(file, 0, SEEK_END);
- return static_cast<size_t>(ftell(file));
-}
-
-// Reads the entire content of a file as a string.
-std::string CapturedStream::ReadEntireFile(FILE* file) {
- const size_t file_size = GetFileSize(file);
- char* const buffer = new char[file_size];
-
- size_t bytes_last_read = 0; // # of bytes read in the last fread()
- size_t bytes_read = 0; // # of bytes read so far
-
- fseek(file, 0, SEEK_SET);
-
- // Keeps reading the file until we cannot read further or the
- // pre-determined file size is reached.
- do {
- bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file);
- bytes_read += bytes_last_read;
- } while (bytes_last_read > 0 && bytes_read < file_size);
-
- const std::string content(buffer, bytes_read);
- delete[] buffer;
-
- return content;
-}
-
-# ifdef _MSC_VER
-# pragma warning(pop)
-# endif // _MSC_VER
+GTEST_DISABLE_MSC_WARNINGS_POP_()
static CapturedStream* g_captured_stderr = NULL;
static CapturedStream* g_captured_stdout = NULL;
@@ -8882,10 +9604,52 @@ std::string GetCapturedStderr() {
#endif // GTEST_HAS_STREAM_REDIRECTION
-#if GTEST_HAS_DEATH_TEST
+std::string TempDir() {
+#if GTEST_OS_WINDOWS_MOBILE
+ return "\\temp\\";
+#elif GTEST_OS_WINDOWS
+ const char* temp_dir = posix::GetEnv("TEMP");
+ if (temp_dir == NULL || temp_dir[0] == '\0')
+ return "\\temp\\";
+ else if (temp_dir[strlen(temp_dir) - 1] == '\\')
+ return temp_dir;
+ else
+ return std::string(temp_dir) + "\\";
+#elif GTEST_OS_LINUX_ANDROID
+ return "/sdcard/";
+#else
+ return "/tmp/";
+#endif // GTEST_OS_WINDOWS_MOBILE
+}
-// A copy of all command line arguments. Set by InitGoogleTest().
-::std::vector<testing::internal::string> g_argvs;
+size_t GetFileSize(FILE* file) {
+ fseek(file, 0, SEEK_END);
+ return static_cast<size_t>(ftell(file));
+}
+
+std::string ReadEntireFile(FILE* file) {
+ const size_t file_size = GetFileSize(file);
+ char* const buffer = new char[file_size];
+
+ size_t bytes_last_read = 0; // # of bytes read in the last fread()
+ size_t bytes_read = 0; // # of bytes read so far
+
+ fseek(file, 0, SEEK_SET);
+
+ // Keeps reading the file until we cannot read further or the
+ // pre-determined file size is reached.
+ do {
+ bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file);
+ bytes_read += bytes_last_read;
+ } while (bytes_last_read > 0 && bytes_read < file_size);
+
+ const std::string content(buffer, bytes_read);
+ delete[] buffer;
+
+ return content;
+}
+
+#if GTEST_HAS_DEATH_TEST
static const ::std::vector<testing::internal::string>* g_injected_test_argvs =
NULL; // Owned.
@@ -8900,7 +9664,7 @@ const ::std::vector<testing::internal::string>& GetInjectableArgvs() {
if (g_injected_test_argvs != NULL) {
return *g_injected_test_argvs;
}
- return g_argvs;
+ return GetArgvs();
}
#endif // GTEST_HAS_DEATH_TEST
@@ -8974,6 +9738,9 @@ bool ParseInt32(const Message& src_text, const char* str, Int32* value) {
//
// The value is considered true iff it's not "0".
bool BoolFromGTestEnv(const char* flag, bool default_value) {
+#if defined(GTEST_GET_BOOL_FROM_ENV_)
+ return GTEST_GET_BOOL_FROM_ENV_(flag, default_value);
+#endif // defined(GTEST_GET_BOOL_FROM_ENV_)
const std::string env_var = FlagToEnvVar(flag);
const char* const string_value = posix::GetEnv(env_var.c_str());
return string_value == NULL ?
@@ -8984,6 +9751,9 @@ bool BoolFromGTestEnv(const char* flag, bool default_value) {
// variable corresponding to the given flag; if it isn't set or
// doesn't represent a valid 32-bit integer, returns default_value.
Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) {
+#if defined(GTEST_GET_INT32_FROM_ENV_)
+ return GTEST_GET_INT32_FROM_ENV_(flag, default_value);
+#endif // defined(GTEST_GET_INT32_FROM_ENV_)
const std::string env_var = FlagToEnvVar(flag);
const char* const string_value = posix::GetEnv(env_var.c_str());
if (string_value == NULL) {
@@ -9005,10 +9775,33 @@ Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) {
// Reads and returns the string environment variable corresponding to
// the given flag; if it's not set, returns default_value.
-const char* StringFromGTestEnv(const char* flag, const char* default_value) {
+std::string StringFromGTestEnv(const char* flag, const char* default_value) {
+#if defined(GTEST_GET_STRING_FROM_ENV_)
+ return GTEST_GET_STRING_FROM_ENV_(flag, default_value);
+#endif // defined(GTEST_GET_STRING_FROM_ENV_)
const std::string env_var = FlagToEnvVar(flag);
- const char* const value = posix::GetEnv(env_var.c_str());
- return value == NULL ? default_value : value;
+ const char* value = posix::GetEnv(env_var.c_str());
+ if (value != NULL) {
+ return value;
+ }
+
+ // As a special case for the 'output' flag, if GTEST_OUTPUT is not
+ // set, we look for XML_OUTPUT_FILE, which is set by the Bazel build
+ // system. The value of XML_OUTPUT_FILE is a filename without the
+ // "xml:" prefix of GTEST_OUTPUT.
+ //
+ // The net priority order after flag processing is thus:
+ // --gtest_output command line flag
+ // GTEST_OUTPUT environment variable
+ // XML_OUTPUT_FILE environment variable
+ // 'default_value'
+ if (strcmp(flag, "output") == 0) {
+ value = posix::GetEnv("XML_OUTPUT_FILE");
+ if (value != NULL) {
+ return std::string("xml:") + value;
+ }
+ }
+ return default_value;
}
} // namespace internal
@@ -9059,6 +9852,7 @@ const char* StringFromGTestEnv(const char* flag, const char* default_value) {
#include <ctype.h>
#include <stdio.h>
+#include <cwchar>
#include <ostream> // NOLINT
#include <string>
@@ -9069,6 +9863,9 @@ namespace {
using ::std::ostream;
// Prints a segment of bytes in the given object.
+GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
void PrintByteSegmentInObjectTo(const unsigned char* obj_bytes, size_t start,
size_t count, ostream* os) {
char text[5] = "";
@@ -9265,6 +10062,9 @@ void PrintTo(wchar_t wc, ostream* os) {
// The array starts at begin, the length is len, it may include '\0' characters
// and may not be NUL-terminated.
template <typename CharType>
+GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
static void PrintCharsAsStringTo(
const CharType* begin, size_t len, ostream* os) {
const char* const kQuoteBegin = sizeof(CharType) == 1 ? "\"" : "L\"";
@@ -9286,6 +10086,9 @@ static void PrintCharsAsStringTo(
// Prints a (const) char/wchar_t array of 'len' elements, starting at address
// 'begin'. CharType must be either char or wchar_t.
template <typename CharType>
+GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
static void UniversalPrintCharArray(
const CharType* begin, size_t len, ostream* os) {
// The code
@@ -9342,7 +10145,7 @@ void PrintTo(const wchar_t* s, ostream* os) {
*os << "NULL";
} else {
*os << ImplicitCast_<const void*>(s) << " pointing to ";
- PrintCharsAsStringTo(s, wcslen(s), os);
+ PrintCharsAsStringTo(s, std::wcslen(s), os);
}
}
#endif // wchar_t is native
@@ -9410,9 +10213,9 @@ void PrintWideStringTo(const ::std::wstring& s, ostream* os) {
// Indicates that this translation unit is part of Google Test's
// implementation. It must come before gtest-internal-inl.h is
-// included, or there will be a compiler error. This trick is to
-// prevent a user from accidentally including gtest-internal-inl.h in
-// his code.
+// included, or there will be a compiler error. This trick exists to
+// prevent the accidental inclusion of gtest-internal-inl.h in the
+// user's code.
#define GTEST_IMPLEMENTATION_ 1
#undef GTEST_IMPLEMENTATION_
@@ -9527,33 +10330,41 @@ static const char* SkipSpaces(const char* str) {
return str;
}
+static std::vector<std::string> SplitIntoTestNames(const char* src) {
+ std::vector<std::string> name_vec;
+ src = SkipSpaces(src);
+ for (; src != NULL; src = SkipComma(src)) {
+ name_vec.push_back(StripTrailingSpaces(GetPrefixUntilComma(src)));
+ }
+ return name_vec;
+}
+
// Verifies that registered_tests match the test names in
-// defined_test_names_; returns registered_tests if successful, or
+// registered_tests_; returns registered_tests if successful, or
// aborts the program otherwise.
const char* TypedTestCasePState::VerifyRegisteredTestNames(
const char* file, int line, const char* registered_tests) {
- typedef ::std::set<const char*>::const_iterator DefinedTestIter;
+ typedef RegisteredTestsMap::const_iterator RegisteredTestIter;
registered_ = true;
- // Skip initial whitespace in registered_tests since some
- // preprocessors prefix stringizied literals with whitespace.
- registered_tests = SkipSpaces(registered_tests);
+ std::vector<std::string> name_vec = SplitIntoTestNames(registered_tests);
Message errors;
- ::std::set<std::string> tests;
- for (const char* names = registered_tests; names != NULL;
- names = SkipComma(names)) {
- const std::string name = GetPrefixUntilComma(names);
+
+ std::set<std::string> tests;
+ for (std::vector<std::string>::const_iterator name_it = name_vec.begin();
+ name_it != name_vec.end(); ++name_it) {
+ const std::string& name = *name_it;
if (tests.count(name) != 0) {
errors << "Test " << name << " is listed more than once.\n";
continue;
}
bool found = false;
- for (DefinedTestIter it = defined_test_names_.begin();
- it != defined_test_names_.end();
+ for (RegisteredTestIter it = registered_tests_.begin();
+ it != registered_tests_.end();
++it) {
- if (name == *it) {
+ if (name == it->first) {
found = true;
break;
}
@@ -9567,11 +10378,11 @@ const char* TypedTestCasePState::VerifyRegisteredTestNames(
}
}
- for (DefinedTestIter it = defined_test_names_.begin();
- it != defined_test_names_.end();
+ for (RegisteredTestIter it = registered_tests_.begin();
+ it != registered_tests_.end();
++it) {
- if (tests.count(*it) == 0) {
- errors << "You forgot to list test " << *it << ".\n";
+ if (tests.count(it->first) == 0) {
+ errors << "You forgot to list test " << it->first << ".\n";
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/48] hadoop git commit: YARN-5774. MR Job stuck in ACCEPTED status
without any progress in Fair Scheduler if set
yarn.scheduler.minimum-allocation-mb to 0. (Contributed by Yufei Gu via
Daniel Templeton)
Posted by li...@apache.org.
YARN-5774. MR Job stuck in ACCEPTED status without any progress in Fair Scheduler
if set yarn.scheduler.minimum-allocation-mb to 0. (Contributed by Yufei Gu via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25f9872b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25f9872b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25f9872b
Branch: refs/heads/HADOOP-13345
Commit: 25f9872be63423ada6a18481eaad2888e731fdac
Parents: 00096dc
Author: Daniel Templeton <te...@apache.org>
Authored: Tue Nov 29 09:40:49 2016 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Tue Nov 29 09:40:49 2016 -0800
----------------------------------------------------------------------
.../api/records/AbstractResourceRequest.java | 55 ++++++++++++++
.../yarn/api/records/ResourceRequest.java | 19 +----
.../api/records/UpdateContainerRequest.java | 18 +----
.../resource/DefaultResourceCalculator.java | 19 +++--
.../resource/DominantResourceCalculator.java | 25 ++++++-
.../yarn/util/resource/ResourceCalculator.java | 17 +----
.../util/resource/TestResourceCalculator.java | 79 ++++++++++++++++++++
.../server/resourcemanager/RMAppManager.java | 6 +-
.../server/resourcemanager/RMServerUtils.java | 9 +--
.../scheduler/AbstractYarnScheduler.java | 20 +++++
.../scheduler/SchedulerUtils.java | 59 +++------------
.../scheduler/YarnScheduler.java | 8 ++
.../scheduler/capacity/CapacityScheduler.java | 4 +-
.../scheduler/fair/FairScheduler.java | 40 ++++++++--
.../scheduler/fifo/FifoScheduler.java | 4 +-
.../scheduler/TestSchedulerUtils.java | 22 +++---
16 files changed, 258 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
new file mode 100644
index 0000000..819a607
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * {@code AbstractResourceRequest} represents a generic resource request made
+ * by an application to the {@code ResourceManager}.
+ * <p>
+ * It includes:
+ * <ul>
+ * <li>{@link Resource} capability required for each request.</li>
+ * </ul>
+ *
+ * @see Resource
+ */
+@Public
+@Unstable
+public abstract class AbstractResourceRequest {
+
+ /**
+ * Set the <code>Resource</code> capability of the request
+ * @param capability <code>Resource</code> capability of the request
+ */
+ @Public
+ @Stable
+ public abstract void setCapability(Resource capability);
+
+ /**
+ * Get the <code>Resource</code> capability of the request.
+ * @return <code>Resource</code> capability of the request
+ */
+ @Public
+ @Stable
+ public abstract Resource getCapability();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index be2c783..ce85b21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -58,7 +58,8 @@ import org.apache.hadoop.yarn.util.Records;
*/
@Public
@Stable
-public abstract class ResourceRequest implements Comparable<ResourceRequest> {
+public abstract class ResourceRequest extends AbstractResourceRequest
+ implements Comparable<ResourceRequest>{
@Public
@Stable
@@ -346,22 +347,6 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
public abstract void setResourceName(String resourceName);
/**
- * Get the <code>Resource</code> capability of the request.
- * @return <code>Resource</code> capability of the request
- */
- @Public
- @Stable
- public abstract Resource getCapability();
-
- /**
- * Set the <code>Resource</code> capability of the request
- * @param capability <code>Resource</code> capability of the request
- */
- @Public
- @Stable
- public abstract void setCapability(Resource capability);
-
- /**
* Get the number of containers required with the given specifications.
* @return number of containers required with the given specifications
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
index ef39f5c..200dea3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
@@ -62,7 +62,7 @@ import org.apache.hadoop.yarn.util.Records;
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public abstract class UpdateContainerRequest {
+public abstract class UpdateContainerRequest extends AbstractResourceRequest {
@InterfaceAudience.Public
@InterfaceStability.Unstable
@@ -128,22 +128,6 @@ public abstract class UpdateContainerRequest {
public abstract void setContainerId(ContainerId containerId);
/**
- * Get the <code>Resource</code> capability of the container.
- * @return <code>Resource</code> capability of the container
- */
- @InterfaceAudience.Public
- @InterfaceStability.Unstable
- public abstract Resource getCapability();
-
- /**
- * Set the <code>Resource</code> capability of the container.
- * @param capability <code>Resource</code> capability of the container
- */
- @InterfaceAudience.Public
- @InterfaceStability.Unstable
- public abstract void setCapability(Resource capability);
-
- /**
* Get the target <code>ExecutionType</code> of the container.
* @return <code>ExecutionType</code> of the container
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index fe78de2..42c45ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.util.resource;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -24,7 +26,9 @@ import org.apache.hadoop.yarn.api.records.Resource;
@Private
@Unstable
public class DefaultResourceCalculator extends ResourceCalculator {
-
+ private static final Log LOG =
+ LogFactory.getLog(DefaultResourceCalculator.class);
+
@Override
public int compare(Resource unused, Resource lhs, Resource rhs) {
// Only consider memory
@@ -64,6 +68,13 @@ public class DefaultResourceCalculator extends ResourceCalculator {
@Override
public Resource normalize(Resource r, Resource minimumResource,
Resource maximumResource, Resource stepFactor) {
+ if (stepFactor.getMemorySize() == 0) {
+ LOG.error("Memory cannot be allocated in increments of zero. Assuming " +
+ minimumResource.getMemorySize() + "MB increment size. "
+ + "Please ensure the scheduler configuration is correct.");
+ stepFactor = minimumResource;
+ }
+
long normalizedMemory = Math.min(
roundUp(
Math.max(r.getMemorySize(), minimumResource.getMemorySize()),
@@ -73,12 +84,6 @@ public class DefaultResourceCalculator extends ResourceCalculator {
}
@Override
- public Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource) {
- return normalize(r, minimumResource, maximumResource, minimumResource);
- }
-
- @Override
public Resource roundUp(Resource r, Resource stepFactor) {
return Resources.createResource(
roundUp(r.getMemorySize(), stepFactor.getMemorySize())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 2695811..9f1c8d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.util.resource;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -45,7 +47,9 @@ import org.apache.hadoop.yarn.api.records.Resource;
@Private
@Unstable
public class DominantResourceCalculator extends ResourceCalculator {
-
+ private static final Log LOG =
+ LogFactory.getLog(DominantResourceCalculator.class);
+
@Override
public int compare(Resource clusterResource, Resource lhs, Resource rhs) {
@@ -152,6 +156,25 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource normalize(Resource r, Resource minimumResource,
Resource maximumResource, Resource stepFactor) {
+ if (stepFactor.getMemorySize() == 0 || stepFactor.getVirtualCores() == 0) {
+ Resource step = Resources.clone(stepFactor);
+ if (stepFactor.getMemorySize() == 0) {
+ LOG.error("Memory cannot be allocated in increments of zero. Assuming "
+ + minimumResource.getMemorySize() + "MB increment size. "
+ + "Please ensure the scheduler configuration is correct.");
+ step.setMemorySize(minimumResource.getMemorySize());
+ }
+
+ if (stepFactor.getVirtualCores() == 0) {
+ LOG.error("VCore cannot be allocated in increments of zero. Assuming "
+ + minimumResource.getVirtualCores() + "VCores increment size. "
+ + "Please ensure the scheduler configuration is correct.");
+ step.setVirtualCores(minimumResource.getVirtualCores());
+ }
+
+ stepFactor = step;
+ }
+
long normalizedMemory = Math.min(
roundUp(
Math.max(r.getMemorySize(), minimumResource.getMemorySize()),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index a7df9c5..50ce04c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -99,22 +99,7 @@ public abstract class ResourceCalculator {
/**
* Normalize resource <code>r</code> given the base
* <code>minimumResource</code> and verify against max allowed
- * <code>maximumResource</code>
- *
- * @param r resource
- * @param minimumResource step-factor
- * @param maximumResource the upper bound of the resource to be allocated
- * @return normalized resource
- */
- public Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource) {
- return normalize(r, minimumResource, maximumResource, minimumResource);
- }
-
- /**
- * Normalize resource <code>r</code> given the base
- * <code>minimumResource</code> and verify against max allowed
- * <code>maximumResource</code> using a step factor for hte normalization.
+ * <code>maximumResource</code> using a step factor for the normalization.
*
* @param r resource
* @param minimumResource minimum value
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
index 0654891..b123b05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
@@ -150,4 +150,83 @@ public class TestResourceCalculator {
Resources.min(resourceCalculator, clusterResource, lhs, rhs));
}
+ /**
+ * Test resource normalization.
+ */
+ @Test(timeout = 10000)
+ public void testNormalize() {
+ // requested resources value cannot be an arbitrary number.
+ Resource ask = Resource.newInstance(1111, 2);
+ Resource min = Resource.newInstance(1024, 1);
+ Resource max = Resource.newInstance(8 * 1024, 8);
+ Resource increment = Resource.newInstance(1024, 4);
+ if (resourceCalculator instanceof DefaultResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ } else if (resourceCalculator instanceof DominantResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ Assert.assertEquals(4, result.getVirtualCores());
+ }
+
+ // if resources asked are less than minimum resource, then normalize it to
+ // minimum resource.
+ ask = Resource.newInstance(512, 0);
+ min = Resource.newInstance(2 * 1024, 2);
+ max = Resource.newInstance(8 * 1024, 8);
+ increment = Resource.newInstance(1024, 1);
+ if (resourceCalculator instanceof DefaultResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ } else if (resourceCalculator instanceof DominantResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ Assert.assertEquals(2, result.getVirtualCores());
+ }
+
+ // if resources asked are larger than maximum resource, then normalize it to
+ // maximum resources.
+ ask = Resource.newInstance(9 * 1024, 9);
+ min = Resource.newInstance(2 * 1024, 2);
+ max = Resource.newInstance(8 * 1024, 8);
+ increment = Resource.newInstance(1024, 1);
+ if (resourceCalculator instanceof DefaultResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(8 * 1024, result.getMemorySize());
+ } else if (resourceCalculator instanceof DominantResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(8 * 1024, result.getMemorySize());
+ Assert.assertEquals(8, result.getVirtualCores());
+ }
+
+ // if increment is 0, use minimum resource as the increment resource.
+ ask = Resource.newInstance(1111, 2);
+ min = Resource.newInstance(2 * 1024, 2);
+ max = Resource.newInstance(8 * 1024, 8);
+ increment = Resource.newInstance(0, 0);
+ if (resourceCalculator instanceof DefaultResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ } else if (resourceCalculator instanceof DominantResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ Assert.assertEquals(2, result.getVirtualCores());
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index b5f02fc..ce3da06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -460,11 +460,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
throw e;
}
- SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(),
- scheduler.getClusterResource(),
- scheduler.getMinimumResourceCapability(),
- scheduler.getMaximumResourceCapability(),
- scheduler.getMinimumResourceCapability());
+ scheduler.normalizeRequest(amReq);
return amReq;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 45a1f30..a0cdf68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -73,7 +73,6 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@@ -304,13 +303,7 @@ public class RMServerUtils {
return false;
}
ResourceScheduler scheduler = rmContext.getScheduler();
- ResourceCalculator rc = scheduler.getResourceCalculator();
- Resource targetResource = Resources.normalize(rc, request.getCapability(),
- scheduler.getMinimumResourceCapability(),
- scheduler.getMaximumResourceCapability(),
- scheduler.getMinimumResourceCapability());
- // Update normalized target resource
- request.setCapability(targetResource);
+ scheduler.normalizeRequest(request);
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 94af4dd..4818ea3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -1024,4 +1025,23 @@ public abstract class AbstractYarnScheduler
}
}
+ @Override
+ public void normalizeRequest(AbstractResourceRequest ask) {
+ SchedulerUtils.normalizeRequest(ask,
+ getResourceCalculator(),
+ getMinimumResourceCapability(),
+ getMaximumResourceCapability(),
+ getMinimumResourceCapability());
+ }
+
+ /**
+ * Normalize a list of resource requests.
+ *
+ * @param asks resource requests
+ */
+ protected void normalizeRequests(List<ResourceRequest> asks) {
+ for (ResourceRequest ask: asks) {
+ normalizeRequest(ask);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c999e26..6f905b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.io.IOException;
-import java.util.List;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
@@ -33,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
@@ -124,72 +124,31 @@ public class SchedulerUtils {
}
/**
- * Utility method to normalize a list of resource requests, by insuring that
- * the memory for each request is a multiple of minMemory and is not zero.
- */
- public static void normalizeRequests(
- List<ResourceRequest> asks,
- ResourceCalculator resourceCalculator,
- Resource clusterResource,
- Resource minimumResource,
- Resource maximumResource) {
- for (ResourceRequest ask : asks) {
- normalizeRequest(
- ask, resourceCalculator, clusterResource, minimumResource,
- maximumResource, minimumResource);
- }
- }
-
- /**
* Utility method to normalize a resource request, by insuring that the
* requested memory is a multiple of minMemory and is not zero.
*/
public static void normalizeRequest(
ResourceRequest ask,
ResourceCalculator resourceCalculator,
- Resource clusterResource,
Resource minimumResource,
Resource maximumResource) {
- Resource normalized =
- Resources.normalize(
- resourceCalculator, ask.getCapability(), minimumResource,
- maximumResource, minimumResource);
- ask.setCapability(normalized);
- }
-
- /**
- * Utility method to normalize a list of resource requests, by insuring that
- * the memory for each request is a multiple of minMemory and is not zero.
- */
- public static void normalizeRequests(
- List<ResourceRequest> asks,
- ResourceCalculator resourceCalculator,
- Resource clusterResource,
- Resource minimumResource,
- Resource maximumResource,
- Resource incrementResource) {
- for (ResourceRequest ask : asks) {
- normalizeRequest(
- ask, resourceCalculator, clusterResource, minimumResource,
- maximumResource, incrementResource);
- }
+ normalizeRequest(ask, resourceCalculator,
+ minimumResource, maximumResource, minimumResource);
}
/**
* Utility method to normalize a resource request, by insuring that the
- * requested memory is a multiple of minMemory and is not zero.
+ * requested memory is a multiple of increment resource and is not zero.
*/
public static void normalizeRequest(
- ResourceRequest ask,
- ResourceCalculator resourceCalculator,
- Resource clusterResource,
+ AbstractResourceRequest ask,
+ ResourceCalculator resourceCalculator,
Resource minimumResource,
Resource maximumResource,
Resource incrementResource) {
- Resource normalized =
- Resources.normalize(
- resourceCalculator, ask.getCapability(), minimumResource,
- maximumResource, incrementResource);
+ Resource normalized = Resources.normalize(
+ resourceCalculator, ask.getCapability(), minimumResource,
+ maximumResource, incrementResource);
ask.setCapability(normalized);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 62ab7f4..7167384 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -368,4 +369,11 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
* @return SchedulerNode corresponds to nodeId
*/
SchedulerNode getSchedulerNode(NodeId nodeId);
+
+ /**
+ * Normalize a resource request.
+ *
+ * @param request the resource request to be normalized
+ */
+ void normalizeRequest(AbstractResourceRequest request);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index aea2e4e..cc8b3b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1119,9 +1119,7 @@ public class CapacityScheduler extends
decreaseContainers(decreaseRequests, application);
// Sanity check for new allocation requests
- SchedulerUtils.normalizeRequests(ask, getResourceCalculator(),
- getClusterResource(), getMinimumResourceCapability(),
- getMaximumResourceCapability());
+ normalizeRequests(ask);
Allocation allocation;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 17f066b..4285e29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -212,12 +213,12 @@ public class FairScheduler extends
getClusterResource(), resource, reservationThreshold);
}
- private void validateConf(Configuration conf) {
+ private void validateConf(FairSchedulerConfiguration config) {
// validate scheduler memory allocation setting
- int minMem = conf.getInt(
+ int minMem = config.getInt(
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
- int maxMem = conf.getInt(
+ int maxMem = config.getInt(
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
@@ -232,11 +233,19 @@ public class FairScheduler extends
+ "the minimum allocation value.");
}
+ long incrementMem = config.getIncrementAllocation().getMemorySize();
+ if (incrementMem <= 0) {
+ throw new YarnRuntimeException("Invalid resource scheduler memory"
+ + " allocation configuration: "
+ + FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB
+ + "=" + incrementMem + ". Values must be greater than 0.");
+ }
+
// validate scheduler vcores allocation setting
- int minVcores = conf.getInt(
+ int minVcores = config.getInt(
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
- int maxVcores = conf.getInt(
+ int maxVcores = config.getInt(
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
@@ -250,6 +259,14 @@ public class FairScheduler extends
+ "and the maximum allocation value must be greater than or equal to"
+ "the minimum allocation value.");
}
+
+ int incrementVcore = config.getIncrementAllocation().getVirtualCores();
+ if (incrementVcore <= 0) {
+ throw new YarnRuntimeException("Invalid resource scheduler vcores"
+ + " allocation configuration: "
+ + FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES
+ + "=" + incrementVcore + ". Values must be greater than 0.");
+ }
}
public FairSchedulerConfiguration getConf() {
@@ -768,6 +785,15 @@ public class FairScheduler extends
}
@Override
+ public void normalizeRequest(AbstractResourceRequest ask) {
+ SchedulerUtils.normalizeRequest(ask,
+ DOMINANT_RESOURCE_CALCULATOR,
+ minimumAllocation,
+ getMaximumResourceCapability(),
+ incrAllocation);
+ }
+
+ @Override
public Allocation allocate(ApplicationAttemptId appAttemptId,
List<ResourceRequest> ask, List<ContainerId> release,
List<String> blacklistAdditions, List<String> blacklistRemovals,
@@ -783,9 +809,7 @@ public class FairScheduler extends
}
// Sanity check
- SchedulerUtils.normalizeRequests(ask, DOMINANT_RESOURCE_CALCULATOR,
- getClusterResource(), minimumAllocation, getMaximumResourceCapability(),
- incrAllocation);
+ normalizeRequests(ask);
// Record container allocation start time
application.recordContainerRequestTime(getClock().getTime());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 5ccde19..b5122c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -335,9 +335,7 @@ public class FifoScheduler extends
}
// Sanity check
- SchedulerUtils.normalizeRequests(ask, resourceCalculator,
- getClusterResource(), minimumAllocation,
- getMaximumResourceCapability());
+ normalizeRequests(ask);
// Release containers
releaseContainers(release, application);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index 63f97c5..98e1e1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -112,37 +112,37 @@ public class TestSchedulerUtils {
// case negative memory
ask.setCapability(Resources.createResource(-1024));
- SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
+ SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource,
maxResource);
assertEquals(minMemory, ask.getCapability().getMemorySize());
// case zero memory
ask.setCapability(Resources.createResource(0));
- SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
+ SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource,
maxResource);
assertEquals(minMemory, ask.getCapability().getMemorySize());
// case memory is a multiple of minMemory
ask.setCapability(Resources.createResource(2 * minMemory));
- SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
+ SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource,
maxResource);
assertEquals(2 * minMemory, ask.getCapability().getMemorySize());
// case memory is not a multiple of minMemory
ask.setCapability(Resources.createResource(minMemory + 10));
- SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
+ SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource,
maxResource);
assertEquals(2 * minMemory, ask.getCapability().getMemorySize());
// case memory is equal to max allowed
ask.setCapability(Resources.createResource(maxMemory));
- SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
+ SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource,
maxResource);
assertEquals(maxMemory, ask.getCapability().getMemorySize());
// case memory is just less than max
ask.setCapability(Resources.createResource(maxMemory - 10));
- SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
+ SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource,
maxResource);
assertEquals(maxMemory, ask.getCapability().getMemorySize());
@@ -150,14 +150,14 @@ public class TestSchedulerUtils {
maxResource = Resources.createResource(maxMemory - 10, 0);
ask.setCapability(Resources.createResource(maxMemory - 100));
// multiple of minMemory > maxMemory, then reduce to maxMemory
- SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
+ SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource,
maxResource);
assertEquals(maxResource.getMemorySize(), ask.getCapability().getMemorySize());
// ask is more than max
maxResource = Resources.createResource(maxMemory, 0);
ask.setCapability(Resources.createResource(maxMemory + 100));
- SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
+ SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource,
maxResource);
assertEquals(maxResource.getMemorySize(), ask.getCapability().getMemorySize());
}
@@ -175,13 +175,13 @@ public class TestSchedulerUtils {
// case negative memory/vcores
ask.setCapability(Resources.createResource(-1024, -1));
SchedulerUtils.normalizeRequest(
- ask, resourceCalculator, clusterResource, minResource, maxResource);
+ ask, resourceCalculator, minResource, maxResource);
assertEquals(minResource, ask.getCapability());
// case zero memory/vcores
ask.setCapability(Resources.createResource(0, 0));
SchedulerUtils.normalizeRequest(
- ask, resourceCalculator, clusterResource, minResource, maxResource);
+ ask, resourceCalculator, minResource, maxResource);
assertEquals(minResource, ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
assertEquals(1024, ask.getCapability().getMemorySize());
@@ -189,7 +189,7 @@ public class TestSchedulerUtils {
// case non-zero memory & zero cores
ask.setCapability(Resources.createResource(1536, 0));
SchedulerUtils.normalizeRequest(
- ask, resourceCalculator, clusterResource, minResource, maxResource);
+ ask, resourceCalculator, minResource, maxResource);
assertEquals(Resources.createResource(2048, 1), ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
assertEquals(2048, ask.getCapability().getMemorySize());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/48] hadoop git commit: HADOOP-13828. Implement
getFileChecksum(path,
length) for ViewFileSystem. Contributed by Manoj Govindassamy.
Posted by li...@apache.org.
HADOOP-13828. Implement getFileChecksum(path, length) for ViewFileSystem. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2b1ff02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2b1ff02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2b1ff02
Branch: refs/heads/HADOOP-13345
Commit: a2b1ff0257bde26d1f64454e97bc1225294a30b9
Parents: 5d5614f
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Nov 28 11:54:43 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Nov 28 11:54:43 2016 -0800
----------------------------------------------------------------------
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 6 ++++
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 9 ++++++
.../fs/viewfs/TestViewFileSystemHdfs.java | 29 ++++++++++++++++++++
3 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2b1ff02/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index 9f61af6..272433f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -222,6 +222,12 @@ class ChRootedFileSystem extends FilterFileSystem {
}
@Override
+ public FileChecksum getFileChecksum(final Path f, final long length)
+ throws IOException {
+ return super.getFileChecksum(fullPath(f), length);
+ }
+
+ @Override
public FileStatus getFileStatus(final Path f)
throws IOException {
return super.getFileStatus(fullPath(f));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2b1ff02/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 9061b2a..ed1bda2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -350,6 +350,15 @@ public class ViewFileSystem extends FileSystem {
return res.targetFileSystem.getFileChecksum(res.remainingPath);
}
+ @Override
+ public FileChecksum getFileChecksum(final Path f, final long length)
+ throws AccessControlException, FileNotFoundException,
+ IOException {
+ InodeTree.ResolveResult<FileSystem> res =
+ fsState.resolve(getUriPath(f), true);
+ return res.targetFileSystem.getFileChecksum(res.remainingPath, length);
+ }
+
private static FileStatus fixFileStatus(FileStatus orig,
Path qualified) throws IOException {
// FileStatus#getPath is a fully qualified path relative to the root of
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2b1ff02/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
index 0e420d0..58b77f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FsConstants;
@@ -218,4 +220,31 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
DFSTestUtil.FsShellRun("-df /", 0, null, newConf);
DFSTestUtil.FsShellRun("-df", 0, null, newConf);
}
+
+ @Test
+ public void testFileChecksum() throws IOException {
+ ViewFileSystem viewFs = (ViewFileSystem) fsView;
+ Path mountDataRootPath = new Path("/data");
+ String fsTargetFileName = "debug.log";
+ Path fsTargetFilePath = new Path(targetTestRoot, "data/debug.log");
+ Path mountDataFilePath = new Path(mountDataRootPath, fsTargetFileName);
+
+ fileSystemTestHelper.createFile(fsTarget, fsTargetFilePath);
+ FileStatus fileStatus = viewFs.getFileStatus(mountDataFilePath);
+ long fileLength = fileStatus.getLen();
+
+ FileChecksum fileChecksumViaViewFs =
+ viewFs.getFileChecksum(mountDataFilePath);
+ FileChecksum fileChecksumViaTargetFs =
+ fsTarget.getFileChecksum(fsTargetFilePath);
+ Assert.assertTrue("File checksum not matching!",
+ fileChecksumViaViewFs.equals(fileChecksumViaTargetFs));
+
+ fileChecksumViaViewFs =
+ viewFs.getFileChecksum(mountDataFilePath, fileLength / 2);
+ fileChecksumViaTargetFs =
+ fsTarget.getFileChecksum(fsTargetFilePath, fileLength / 2);
+ Assert.assertTrue("File checksum not matching!",
+ fileChecksumViaViewFs.equals(fileChecksumViaTargetFs));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/48] hadoop git commit: HADOOP-13823. s3a rename: fail if dest
file exists. Contributed by Steve Loughran
Posted by li...@apache.org.
HADOOP-13823. s3a rename: fail if dest file exists. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d60a60be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d60a60be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d60a60be
Branch: refs/heads/HADOOP-13345
Commit: d60a60be8aa450c44d3be69d26c88025e253ac0c
Parents: be88d57
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Nov 28 16:30:29 2016 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Nov 28 16:30:29 2016 -0800
----------------------------------------------------------------------
.../hadoop/fs/s3a/RenameFailedException.java | 70 +++++++++++++++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 89 +++++++++++++-------
.../fs/s3a/ITestS3AFileSystemContract.java | 15 ----
.../src/test/resources/contract/s3a.xml | 2 +-
4 files changed, 130 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d60a60be/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
new file mode 100644
index 0000000..6c88f3b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+
+/**
+ * Error to indicate that a specific rename failed.
+ * The exit code defines the exit code to be returned in the {@code rename()}
+ * call.
+ * Target path is set to destination.
+ */
+public class RenameFailedException extends PathIOException {
+
+ /**
+ * Exit code to be returned.
+ */
+ private boolean exitCode = false;
+
+ public RenameFailedException(String src, String dest, Throwable cause) {
+ super(src, cause);
+ setOperation("rename");
+ setTargetPath(dest);
+ }
+
+ public RenameFailedException(String src, String dest, String error) {
+ super(src, error);
+ setOperation("rename");
+ setTargetPath(dest);
+ }
+
+ public RenameFailedException(Path src, Path optionalDest, String error) {
+ super(src.toString(), error);
+ setOperation("rename");
+ if (optionalDest != null) {
+ setTargetPath(optionalDest.toString());
+ }
+ }
+
+ public boolean getExitCode() {
+ return exitCode;
+ }
+
+ /**
+ * Set the exit code.
+ * @param code exit code to raise
+ * @return the exception
+ */
+ public RenameFailedException withExitCode(boolean code) {
+ this.exitCode = code;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d60a60be/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 925abbc..f1cde4e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -630,10 +630,12 @@ public class S3AFileSystem extends FileSystem {
* there is no Progressable passed in, this can time out jobs.
*
* Note: This implementation differs with other S3 drivers. Specifically:
+ * <pre>
* Fails if src is a file and dst is a directory.
* Fails if src is a directory and dst is a file.
* Fails if the parent of dst does not exist or is a file.
* Fails if dst is a directory that is not empty.
+ * </pre>
*
* @param src path to be renamed
* @param dst new path after rename
@@ -645,58 +647,86 @@ public class S3AFileSystem extends FileSystem {
return innerRename(src, dst);
} catch (AmazonClientException e) {
throw translateException("rename(" + src +", " + dst + ")", src, e);
+ } catch (RenameFailedException e) {
+ LOG.debug(e.getMessage());
+ return e.getExitCode();
+ } catch (FileNotFoundException e) {
+ LOG.debug(e.toString());
+ return false;
}
}
/**
* The inner rename operation. See {@link #rename(Path, Path)} for
* the description of the operation.
+ * This operation throws an exception on any failure which needs to be
+ * reported and downgraded to a failure. That is: if a rename
* @param src path to be renamed
* @param dst new path after rename
- * @return true if rename is successful
+ * @throws RenameFailedException if some criteria for a state changing
+ * rename was not met. This means work didn't happen; it's not something
+ * which is reported upstream to the FileSystem APIs, for which the semantics
+ * of "false" are pretty vague.
+ * @throws FileNotFoundException there's no source file.
* @throws IOException on IO failure.
* @throws AmazonClientException on failures inside the AWS SDK
*/
- private boolean innerRename(Path src, Path dst) throws IOException,
- AmazonClientException {
+ private boolean innerRename(Path src, Path dst)
+ throws RenameFailedException, FileNotFoundException, IOException,
+ AmazonClientException {
LOG.debug("Rename path {} to {}", src, dst);
incrementStatistic(INVOCATION_RENAME);
String srcKey = pathToKey(src);
String dstKey = pathToKey(dst);
- if (srcKey.isEmpty() || dstKey.isEmpty()) {
- LOG.debug("rename: source {} or dest {}, is empty", srcKey, dstKey);
- return false;
+ if (srcKey.isEmpty()) {
+ throw new RenameFailedException(src, dst, "source is root directory");
}
-
- S3AFileStatus srcStatus;
- try {
- srcStatus = getFileStatus(src);
- } catch (FileNotFoundException e) {
- LOG.error("rename: src not found {}", src);
- return false;
+ if (dstKey.isEmpty()) {
+ throw new RenameFailedException(src, dst, "dest is root directory");
}
+ // get the source file status; this raises a FNFE if there is no source
+ // file.
+ S3AFileStatus srcStatus = getFileStatus(src);
+
if (srcKey.equals(dstKey)) {
- LOG.debug("rename: src and dst refer to the same file or directory: {}",
+ LOG.debug("rename: src and dest refer to the same file or directory: {}",
dst);
- return srcStatus.isFile();
+ throw new RenameFailedException(src, dst,
+ "source and dest refer to the same file or directory")
+ .withExitCode(srcStatus.isFile());
}
S3AFileStatus dstStatus = null;
try {
dstStatus = getFileStatus(dst);
-
- if (srcStatus.isDirectory() && dstStatus.isFile()) {
- LOG.debug("rename: src {} is a directory and dst {} is a file",
- src, dst);
- return false;
+ // if there is no destination entry, an exception is raised.
+ // hence this code sequence can assume that there is something
+ // at the end of the path; the only detail being what it is and
+ // whether or not it can be the destination of the rename.
+ if (srcStatus.isDirectory()) {
+ if (dstStatus.isFile()) {
+ throw new RenameFailedException(src, dst,
+ "source is a directory and dest is a file")
+ .withExitCode(srcStatus.isFile());
+ } else if (!dstStatus.isEmptyDirectory()) {
+ throw new RenameFailedException(src, dst,
+ "Destination is a non-empty directory")
+ .withExitCode(false);
+ }
+ // at this point the destination is an empty directory
+ } else {
+ // source is a file. The destination must be a directory,
+ // empty or not
+ if (dstStatus.isFile()) {
+ throw new RenameFailedException(src, dst,
+ "Cannot rename onto an existing file")
+ .withExitCode(false);
+ }
}
- if (dstStatus.isDirectory() && !dstStatus.isEmptyDirectory()) {
- return false;
- }
} catch (FileNotFoundException e) {
LOG.debug("rename: destination path {} not found", dst);
// Parent must exist
@@ -705,12 +735,12 @@ public class S3AFileSystem extends FileSystem {
try {
S3AFileStatus dstParentStatus = getFileStatus(dst.getParent());
if (!dstParentStatus.isDirectory()) {
- return false;
+ throw new RenameFailedException(src, dst,
+ "destination parent is not a directory");
}
} catch (FileNotFoundException e2) {
- LOG.debug("rename: destination path {} has no parent {}",
- dst, parent);
- return false;
+ throw new RenameFailedException(src, dst,
+ "destination has no parent ");
}
}
}
@@ -745,9 +775,8 @@ public class S3AFileSystem extends FileSystem {
//Verify dest is not a child of the source directory
if (dstKey.startsWith(srcKey)) {
- LOG.debug("cannot rename a directory {}" +
- " to a subdirectory of self: {}", srcKey, dstKey);
- return false;
+ throw new RenameFailedException(srcKey, dstKey,
+ "cannot rename a directory to a subdirectory o fitself ");
}
List<DeleteObjectsRequest.KeyVersion> keysToDelete = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d60a60be/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
index 0eb601b..f39ec81 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
@@ -89,21 +89,6 @@ public class ITestS3AFileSystemContract extends FileSystemContractBaseTest {
}
@Override
- public void testRenameFileAsExistingFile() throws Exception {
- if (!renameSupported()) {
- return;
- }
-
- Path src = path("/test/hadoop/file");
- createFile(src);
- Path dst = path("/test/new/newfile");
- createFile(dst);
- // s3 doesn't support rename option
- // rename-overwrites-dest is always allowed.
- rename(src, dst, true, false, true);
- }
-
- @Override
public void testRenameDirectoryAsExistingDirectory() throws Exception {
if (!renameSupported()) {
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d60a60be/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
index a534f0a..fe0af66 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
@@ -114,7 +114,7 @@
<property>
<name>fs.contract.rename-overwrites-dest</name>
- <value>true</value>
+ <value>false</value>
</property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/48] hadoop git commit: YARN-4752. Improved preemption in
FairScheduler. (kasha)
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
index 2cbe507..36ee685 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
@@ -17,1467 +17,259 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerRequestKey;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
- .TestUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
-import org.apache.hadoop.yarn.util.ControlledClock;
-import org.apache.hadoop.yarn.util.resource.Resources;
-
import org.junit.After;
-import org.junit.Assert;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import java.util.Arrays;
+import java.util.Collection;
+/**
+ * Tests to verify fairshare and minshare preemption, using parameterization.
+ */
+@RunWith(Parameterized.class)
public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
- private final static String ALLOC_FILE = new File(TEST_DIR,
- TestFairSchedulerPreemption.class.getName() + ".xml").getAbsolutePath();
+ private static final File ALLOC_FILE = new File(TEST_DIR, "test-queues");
- private ControlledClock clock;
+ // Node Capacity = NODE_CAPACITY_MULTIPLE * (1 GB or 1 vcore)
+ private static final int NODE_CAPACITY_MULTIPLE = 4;
- private static class StubbedFairScheduler extends FairScheduler {
- public long lastPreemptMemory = -1;
+ private final boolean fairsharePreemption;
- @Override
- protected void preemptResources(Resource toPreempt) {
- lastPreemptMemory = toPreempt.getMemorySize();
- }
+ // App that takes up the entire cluster
+ private FSAppAttempt greedyApp;
- public void resetLastPreemptResources() {
- lastPreemptMemory = -1;
- }
+ // Starving app that is expected to instigate preemption
+ private FSAppAttempt starvingApp;
+
+ @Parameterized.Parameters
+ public static Collection<Boolean[]> getParameters() {
+ return Arrays.asList(new Boolean[][] {
+ {true}, {false}});
}
- public Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, StubbedFairScheduler.class,
- ResourceScheduler.class);
- conf.setBoolean(FairSchedulerConfiguration.PREEMPTION, true);
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- return conf;
+ public TestFairSchedulerPreemption(Boolean fairshare) throws IOException {
+ fairsharePreemption = fairshare;
+ writeAllocFile();
}
@Before
- public void setup() throws IOException {
- conf = createConfiguration();
- clock = new ControlledClock();
+ public void setup() {
+ createConfiguration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ ALLOC_FILE.getAbsolutePath());
+ conf.setBoolean(FairSchedulerConfiguration.PREEMPTION, true);
+ conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
+ conf.setInt(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 0);
}
@After
public void teardown() {
- if (resourceManager != null) {
- resourceManager.stop();
- resourceManager = null;
- }
+ ALLOC_FILE.delete();
conf = null;
- }
-
- private void startResourceManagerWithStubbedFairScheduler(float utilizationThreshold) {
- conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD,
- utilizationThreshold);
- resourceManager = new MockRM(conf);
- resourceManager.start();
-
- assertTrue(
- resourceManager.getResourceScheduler() instanceof StubbedFairScheduler);
- scheduler = (FairScheduler)resourceManager.getResourceScheduler();
-
- scheduler.setClock(clock);
- scheduler.updateInterval = 60 * 1000;
- }
-
- // YARN-4648: The starting code for ResourceManager mock is originated from
- // TestFairScheduler. It should be keep as it was to guarantee no changing
- // behaviour of ResourceManager preemption.
- private void startResourceManagerWithRealFairScheduler() {
- scheduler = new FairScheduler();
- conf = new YarnConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
- ResourceScheduler.class);
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
- conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
- 1024);
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240);
- conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false);
- conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
- conf.setFloat(
- FairSchedulerConfiguration
- .RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE,
- TEST_RESERVATION_THRESHOLD);
-
- resourceManager = new MockRM(conf);
-
- // TODO: This test should really be using MockRM. For now starting stuff
- // that is needed at a bare minimum.
- ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
- resourceManager.getRMContext().getStateStore().start();
-
- // to initialize the master key
- resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
-
- scheduler.setRMContext(resourceManager.getRMContext());
- }
-
- private void stopResourceManager() {
- if (scheduler != null) {
- scheduler.stop();
- scheduler = null;
- }
if (resourceManager != null) {
resourceManager.stop();
resourceManager = null;
}
- QueueMetrics.clearQueueMetrics();
- DefaultMetricsSystem.shutdown();
- }
-
- private void registerNodeAndSubmitApp(
- int memory, int vcores, int appContainers, int appMemory) {
- RMNode node1 = MockNodes.newNodeInfo(
- 1, Resources.createResource(memory, vcores), 1, "node1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
-
- assertEquals("Incorrect amount of resources in the cluster",
- memory, scheduler.rootMetrics.getAvailableMB());
- assertEquals("Incorrect amount of resources in the cluster",
- vcores, scheduler.rootMetrics.getAvailableVirtualCores());
-
- createSchedulingRequest(appMemory, "queueA", "user1", appContainers);
- scheduler.update();
- // Sufficient node check-ins to fully schedule containers
- for (int i = 0; i < 3; i++) {
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
- scheduler.handle(nodeUpdate1);
- }
- assertEquals("app1's request is not met",
- memory - appContainers * appMemory,
- scheduler.rootMetrics.getAvailableMB());
}
- @Test
- public void testPreemptionWithFreeResources() throws Exception {
+ private void writeAllocFile() throws IOException {
+ /*
+ * Queue hierarchy:
+ * root
+ * |--- preemptable
+ * |--- child-1
+ * |--- child-2
+ * |--- nonpreemptible
+ * |--- child-1
+ * |--- child-2
+ */
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("<maxResources>0mb,0vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>1</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>1</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.print("<defaultMinSharePreemptionTimeout>5</defaultMinSharePreemptionTimeout>");
- out.print("<fairSharePreemptionTimeout>10</fairSharePreemptionTimeout>");
- out.println("</allocations>");
- out.close();
-
- startResourceManagerWithStubbedFairScheduler(0f);
- // Create node with 4GB memory and 4 vcores
- registerNodeAndSubmitApp(4 * 1024, 4, 2, 1024);
-
- // Verify submitting another request triggers preemption
- createSchedulingRequest(1024, "queueB", "user1", 1, 1);
- scheduler.update();
- clock.tickSec(6);
-
- ((StubbedFairScheduler) scheduler).resetLastPreemptResources();
- scheduler.preemptTasksIfNecessary();
- assertEquals("preemptResources() should have been called", 1024,
- ((StubbedFairScheduler) scheduler).lastPreemptMemory);
-
- resourceManager.stop();
-
- startResourceManagerWithStubbedFairScheduler(0.8f);
- // Create node with 4GB memory and 4 vcores
- registerNodeAndSubmitApp(4 * 1024, 4, 3, 1024);
- // Verify submitting another request doesn't trigger preemption
- createSchedulingRequest(1024, "queueB", "user1", 1, 1);
- scheduler.update();
- clock.tickSec(6);
+ out.println("<queue name=\"preemptable\">");
+ writePreemptionParams(out);
- ((StubbedFairScheduler) scheduler).resetLastPreemptResources();
- scheduler.preemptTasksIfNecessary();
- assertEquals("preemptResources() should not have been called", -1,
- ((StubbedFairScheduler) scheduler).lastPreemptMemory);
+ // Child-1
+ out.println("<queue name=\"child-1\">");
+ writeResourceParams(out);
+ out.println("</queue>");
- resourceManager.stop();
+ // Child-2
+ out.println("<queue name=\"child-2\">");
+ writeResourceParams(out);
+ out.println("</queue>");
- startResourceManagerWithStubbedFairScheduler(0.7f);
- // Create node with 4GB memory and 4 vcores
- registerNodeAndSubmitApp(4 * 1024, 4, 3, 1024);
+ out.println("</queue>"); // end of preemptable queue
- // Verify submitting another request triggers preemption
- createSchedulingRequest(1024, "queueB", "user1", 1, 1);
- scheduler.update();
- clock.tickSec(6);
+ // Queue with preemption disallowed
+ out.println("<queue name=\"nonpreemptable\">");
+ out.println("<allowPreemptionFrom>false" +
+ "</allowPreemptionFrom>");
+ writePreemptionParams(out);
- ((StubbedFairScheduler) scheduler).resetLastPreemptResources();
- scheduler.preemptTasksIfNecessary();
- assertEquals("preemptResources() should have been called", 1024,
- ((StubbedFairScheduler) scheduler).lastPreemptMemory);
- }
+ // Child-1
+ out.println("<queue name=\"child-1\">");
+ writeResourceParams(out);
+ out.println("</queue>");
- @Test (timeout = 5000)
- /**
- * Make sure containers are chosen to be preempted in the correct order.
- */
- public void testChoiceOfPreemptedContainers() throws Exception {
- startResourceManagerWithRealFairScheduler();
- conf.setLong(FairSchedulerConfiguration.PREEMPTION_INTERVAL, 5000);
- conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10000);
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE + ".allocation.file", ALLOC_FILE);
- conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
+ // Child-2
+ out.println("<queue name=\"child-2\">");
+ writeResourceParams(out);
+ out.println("</queue>");
- ControlledClock clock = new ControlledClock();
- scheduler.setClock(clock);
+ out.println("</queue>"); // end of nonpreemptable queue
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>.25</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>.25</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queueC\">");
- out.println("<weight>.25</weight>");
- out.println("</queue>");
- out.println("<queue name=\"default\">");
- out.println("<weight>.25</weight>");
- out.println("</queue>");
out.println("</allocations>");
out.close();
- scheduler.init(conf);
- scheduler.start();
- scheduler.reinitialize(conf, resourceManager.getRMContext());
-
- // Create two nodes
- RMNode node1 =
- MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 1,
- "127.0.0.1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
-
- RMNode node2 =
- MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 2,
- "127.0.0.2");
- NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
- scheduler.handle(nodeEvent2);
-
- // Queue A and B each request two applications
- ApplicationAttemptId app1 =
- createSchedulingRequest(1 * 1024, 1, "queueA", "user1", 1, 1);
- createSchedulingRequestExistingApplication(1 * 1024, 1, 2, app1);
- ApplicationAttemptId app2 =
- createSchedulingRequest(1 * 1024, 1, "queueA", "user1", 1, 3);
- createSchedulingRequestExistingApplication(1 * 1024, 1, 4, app2);
-
- ApplicationAttemptId app3 =
- createSchedulingRequest(1 * 1024, 1, "queueB", "user1", 1, 1);
- createSchedulingRequestExistingApplication(1 * 1024, 1, 2, app3);
- ApplicationAttemptId app4 =
- createSchedulingRequest(1 * 1024, 1, "queueB", "user1", 1, 3);
- createSchedulingRequestExistingApplication(1 * 1024, 1, 4, app4);
-
- scheduler.update();
-
- scheduler.getQueueManager().getLeafQueue("queueA", true)
- .setPolicy(SchedulingPolicy.parse("fifo"));
- scheduler.getQueueManager().getLeafQueue("queueB", true)
- .setPolicy(SchedulingPolicy.parse("fair"));
+ assertTrue("Allocation file does not exist, not running the test",
+ ALLOC_FILE.exists());
+ }
- // Sufficient node check-ins to fully schedule containers
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
- NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2);
- for (int i = 0; i < 4; i++) {
- scheduler.handle(nodeUpdate1);
- scheduler.handle(nodeUpdate2);
+ private void writePreemptionParams(PrintWriter out) {
+ if (fairsharePreemption) {
+ out.println("<fairSharePreemptionThreshold>1" +
+ "</fairSharePreemptionThreshold>");
+ out.println("<fairSharePreemptionTimeout>0" +
+ "</fairSharePreemptionTimeout>");
+ } else {
+ out.println("<minSharePreemptionTimeout>0" +
+ "</minSharePreemptionTimeout>");
}
+ }
- assertEquals(2, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(2, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(2, scheduler.getSchedulerApp(app3).getLiveContainers().size());
- assertEquals(2, scheduler.getSchedulerApp(app4).getLiveContainers().size());
-
- // Now new requests arrive from queueC and default
- createSchedulingRequest(1 * 1024, 1, "queueC", "user1", 1, 1);
- createSchedulingRequest(1 * 1024, 1, "queueC", "user1", 1, 1);
- createSchedulingRequest(1 * 1024, 1, "default", "user1", 1, 1);
- createSchedulingRequest(1 * 1024, 1, "default", "user1", 1, 1);
- scheduler.update();
-
- // We should be able to claw back one container from queueA and queueB each.
- scheduler.preemptResources(Resources.createResource(2 * 1024));
- assertEquals(2, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(2, scheduler.getSchedulerApp(app3).getLiveContainers().size());
-
- // First verify we are adding containers to preemption list for the app.
- // For queueA (fifo), app2 is selected.
- // For queueB (fair), app4 is selected.
- assertTrue("App2 should have container to be preempted",
- !Collections.disjoint(
- scheduler.getSchedulerApp(app2).getLiveContainers(),
- scheduler.getSchedulerApp(app2).getPreemptionContainers()));
- assertTrue("App4 should have container to be preempted",
- !Collections.disjoint(
- scheduler.getSchedulerApp(app2).getLiveContainers(),
- scheduler.getSchedulerApp(app2).getPreemptionContainers()));
-
- // Pretend 15 seconds have passed
- clock.tickSec(15);
-
- // Trigger a kill by insisting we want containers back
- scheduler.preemptResources(Resources.createResource(2 * 1024));
-
- // At this point the containers should have been killed (since we are not simulating AM)
- assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
- // Inside each app, containers are sorted according to their priorities.
- // Containers with priority 4 are preempted for app2 and app4.
- Set<RMContainer> set = new HashSet<RMContainer>();
- for (RMContainer container :
- scheduler.getSchedulerApp(app2).getLiveContainers()) {
- if (container.getAllocatedSchedulerKey().getPriority().getPriority() ==
- 4) {
- set.add(container);
- }
- }
- for (RMContainer container :
- scheduler.getSchedulerApp(app4).getLiveContainers()) {
- if (container.getAllocatedSchedulerKey().getPriority().getPriority() ==
- 4) {
- set.add(container);
- }
+ private void writeResourceParams(PrintWriter out) {
+ if (!fairsharePreemption) {
+ out.println("<minResources>4096mb,4vcores</minResources>");
}
- assertTrue("Containers with priority=4 in app2 and app4 should be " +
- "preempted.", set.isEmpty());
-
- // Trigger a kill by insisting we want containers back
- scheduler.preemptResources(Resources.createResource(2 * 1024));
-
- // Pretend 15 seconds have passed
- clock.tickSec(15);
-
- // We should be able to claw back another container from A and B each.
- // For queueA (fifo), continue preempting from app2.
- // For queueB (fair), even app4 has a lowest priority container with p=4, it
- // still preempts from app3 as app3 is most over fair share.
- scheduler.preemptResources(Resources.createResource(2 * 1024));
-
- assertEquals(2, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(0, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app3).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
-
- // Now A and B are below fair share, so preemption shouldn't do anything
- scheduler.preemptResources(Resources.createResource(2 * 1024));
- assertTrue("App1 should have no container to be preempted",
- scheduler.getSchedulerApp(app1).getPreemptionContainers().isEmpty());
- assertTrue("App2 should have no container to be preempted",
- scheduler.getSchedulerApp(app2).getPreemptionContainers().isEmpty());
- assertTrue("App3 should have no container to be preempted",
- scheduler.getSchedulerApp(app3).getPreemptionContainers().isEmpty());
- assertTrue("App4 should have no container to be preempted",
- scheduler.getSchedulerApp(app4).getPreemptionContainers().isEmpty());
- stopResourceManager();
}
- @Test
- public void testPreemptionIsNotDelayedToNextRound() throws Exception {
- startResourceManagerWithRealFairScheduler();
-
- conf.setLong(FairSchedulerConfiguration.PREEMPTION_INTERVAL, 5000);
- conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10000);
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
-
- ControlledClock clock = new ControlledClock();
- scheduler.setClock(clock);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>8</weight>");
- out.println("<queue name=\"queueA1\" />");
- out.println("<queue name=\"queueA2\" />");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>2</weight>");
- out.println("</queue>");
- out.println("<defaultFairSharePreemptionTimeout>10</defaultFairSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>");
- out.println("</allocations>");
- out.close();
-
- scheduler.init(conf);
- scheduler.start();
- scheduler.reinitialize(conf, resourceManager.getRMContext());
-
- // Add a node of 8G
- RMNode node1 = MockNodes.newNodeInfo(1,
- Resources.createResource(8 * 1024, 8), 1, "127.0.0.1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
-
- // Run apps in queueA.A1 and queueB
- ApplicationAttemptId app1 = createSchedulingRequest(1 * 1024, 1,
- "queueA.queueA1", "user1", 7, 1);
- // createSchedulingRequestExistingApplication(1 * 1024, 1, 2, app1);
- ApplicationAttemptId app2 = createSchedulingRequest(1 * 1024, 1, "queueB",
- "user2", 1, 1);
+ private void setupCluster() throws IOException {
+ resourceManager = new MockRM(conf);
+ resourceManager.start();
+ scheduler = (FairScheduler) resourceManager.getResourceScheduler();
- scheduler.update();
+ // Create and add two nodes to the cluster
+ addNode(NODE_CAPACITY_MULTIPLE * 1024, NODE_CAPACITY_MULTIPLE);
+ addNode(NODE_CAPACITY_MULTIPLE * 1024, NODE_CAPACITY_MULTIPLE);
+ }
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
- for (int i = 0; i < 8; i++) {
- scheduler.handle(nodeUpdate1);
+ private void sendEnoughNodeUpdatesToAssignFully() {
+ for (RMNode node : rmNodes) {
+ NodeUpdateSchedulerEvent nodeUpdateSchedulerEvent =
+ new NodeUpdateSchedulerEvent(node);
+ for (int i = 0; i < NODE_CAPACITY_MULTIPLE; i++) {
+ scheduler.handle(nodeUpdateSchedulerEvent);
+ }
}
-
- // verify if the apps got the containers they requested
- assertEquals(7, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size());
-
- // Now submit an app in queueA.queueA2
- ApplicationAttemptId app3 = createSchedulingRequest(1 * 1024, 1,
- "queueA.queueA2", "user3", 7, 1);
- scheduler.update();
-
- // Let 11 sec pass
- clock.tickSec(11);
-
- scheduler.update();
- Resource toPreempt = scheduler.resourceDeficit(scheduler.getQueueManager()
- .getLeafQueue("queueA.queueA2", false), clock.getTime());
- assertEquals(3277, toPreempt.getMemorySize());
-
- // verify if the 3 containers required by queueA2 are preempted in the same
- // round
- scheduler.preemptResources(toPreempt);
- assertEquals(3, scheduler.getSchedulerApp(app1).getPreemptionContainers()
- .size());
- stopResourceManager();
}
- @Test (timeout = 5000)
/**
- * Tests the timing of decision to preempt tasks.
+ * Submit application to {@code queue1} and take over the entire cluster.
+ * Submit application with larger containers to {@code queue2} that
+ * requires preemption from the first application.
+ *
+ * @param queue1 first queue
+ * @param queue2 second queue
+ * @throws InterruptedException if interrupted while waiting
*/
- public void testPreemptionDecision() throws Exception {
- startResourceManagerWithRealFairScheduler();
-
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- ControlledClock clock = new ControlledClock();
- scheduler.setClock(clock);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("<maxResources>0mb,0vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueC\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueD\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<defaultMinSharePreemptionTimeout>5</defaultMinSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionTimeout>10</defaultFairSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>");
- out.println("</allocations>");
- out.close();
-
- scheduler.init(conf);
- scheduler.start();
- scheduler.reinitialize(conf, resourceManager.getRMContext());
-
- // Create four nodes
- RMNode node1 =
- MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 2), 1,
- "127.0.0.1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
-
- RMNode node2 =
- MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 2), 2,
- "127.0.0.2");
- NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
- scheduler.handle(nodeEvent2);
-
- RMNode node3 =
- MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 2), 3,
- "127.0.0.3");
- NodeAddedSchedulerEvent nodeEvent3 = new NodeAddedSchedulerEvent(node3);
- scheduler.handle(nodeEvent3);
-
- // Queue A and B each request three containers
- ApplicationAttemptId app1 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 1);
- ApplicationAttemptId app2 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 2);
- ApplicationAttemptId app3 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 3);
-
- ApplicationAttemptId app4 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 1);
- ApplicationAttemptId app5 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 2);
- ApplicationAttemptId app6 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 3);
-
- scheduler.update();
-
- // Sufficient node check-ins to fully schedule containers
- for (int i = 0; i < 2; i++) {
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
- scheduler.handle(nodeUpdate1);
-
- NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2);
- scheduler.handle(nodeUpdate2);
-
- NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3);
- scheduler.handle(nodeUpdate3);
- }
-
- // Now new requests arrive from queues C and D
- ApplicationAttemptId app7 =
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 1);
- ApplicationAttemptId app8 =
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 2);
- ApplicationAttemptId app9 =
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 3);
-
- ApplicationAttemptId app10 =
- createSchedulingRequest(1 * 1024, "queueD", "user1", 1, 1);
- ApplicationAttemptId app11 =
- createSchedulingRequest(1 * 1024, "queueD", "user1", 1, 2);
- ApplicationAttemptId app12 =
- createSchedulingRequest(1 * 1024, "queueD", "user1", 1, 3);
-
+ private void submitApps(String queue1, String queue2)
+ throws InterruptedException {
+ // Create an app that takes up all the resources on the cluster
+ ApplicationAttemptId appAttemptId1
+ = createSchedulingRequest(1024, 1, queue1, "default",
+ NODE_CAPACITY_MULTIPLE * rmNodes.size());
+ greedyApp = scheduler.getSchedulerApp(appAttemptId1);
scheduler.update();
+ sendEnoughNodeUpdatesToAssignFully();
+ assertEquals(8, greedyApp.getLiveContainers().size());
- FSLeafQueue schedC =
- scheduler.getQueueManager().getLeafQueue("queueC", true);
- FSLeafQueue schedD =
- scheduler.getQueueManager().getLeafQueue("queueD", true);
+ // Create an app that takes up all the resources on the cluster
+ ApplicationAttemptId appAttemptId2
+ = createSchedulingRequest(2048, 2, queue2, "default",
+ NODE_CAPACITY_MULTIPLE * rmNodes.size() / 2);
+ starvingApp = scheduler.getSchedulerApp(appAttemptId2);
- assertTrue(Resources.equals(
- Resources.none(), scheduler.resourceDeficit(schedC, clock.getTime())));
- assertTrue(Resources.equals(
- Resources.none(), scheduler.resourceDeficit(schedD, clock.getTime())));
- // After minSharePreemptionTime has passed, they should want to preempt min
- // share.
- clock.tickSec(6);
- assertEquals(
- 1024, scheduler.resourceDeficit(schedC, clock.getTime()).getMemorySize());
- assertEquals(
- 1024, scheduler.resourceDeficit(schedD, clock.getTime()).getMemorySize());
+ // Sleep long enough to pass
+ Thread.sleep(10);
- // After fairSharePreemptionTime has passed, they should want to preempt
- // fair share.
scheduler.update();
- clock.tickSec(6);
- assertEquals(
- 1536 , scheduler.resourceDeficit(schedC, clock.getTime()).getMemorySize());
- assertEquals(
- 1536, scheduler.resourceDeficit(schedD, clock.getTime()).getMemorySize());
- stopResourceManager();
}
- @Test
-/**
- * Tests the timing of decision to preempt tasks.
- */
- public void testPreemptionDecisionWithDRF() throws Exception {
- startResourceManagerWithRealFairScheduler();
-
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- ControlledClock clock = new ControlledClock();
- scheduler.setClock(clock);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("<maxResources>0mb,0vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,1vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,2vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueC\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,3vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueD\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,2vcores</minResources>");
- out.println("</queue>");
- out.println("<defaultMinSharePreemptionTimeout>5</defaultMinSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionTimeout>10</defaultFairSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>");
- out.println("<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
-
- scheduler.init(conf);
- scheduler.start();
- scheduler.reinitialize(conf, resourceManager.getRMContext());
-
- // Create four nodes
- RMNode node1 =
- MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 4), 1,
- "127.0.0.1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
-
- RMNode node2 =
- MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 4), 2,
- "127.0.0.2");
- NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
- scheduler.handle(nodeEvent2);
-
- RMNode node3 =
- MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 4), 3,
- "127.0.0.3");
- NodeAddedSchedulerEvent nodeEvent3 = new NodeAddedSchedulerEvent(node3);
- scheduler.handle(nodeEvent3);
-
- // Queue A and B each request three containers
- ApplicationAttemptId app1 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 1);
- ApplicationAttemptId app2 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 2);
- ApplicationAttemptId app3 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 3);
-
- ApplicationAttemptId app4 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 1);
- ApplicationAttemptId app5 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 2);
- ApplicationAttemptId app6 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 3);
-
- scheduler.update();
-
- // Sufficient node check-ins to fully schedule containers
- for (int i = 0; i < 2; i++) {
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
- scheduler.handle(nodeUpdate1);
-
- NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2);
- scheduler.handle(nodeUpdate2);
-
- NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3);
- scheduler.handle(nodeUpdate3);
+ private void verifyPreemption() throws InterruptedException {
+ // Sleep long enough for four containers to be preempted. Note that the
+ // starved app must be queued four times for containers to be preempted.
+ for (int i = 0; i < 10000; i++) {
+ if (greedyApp.getLiveContainers().size() == 4) {
+ break;
+ }
+ Thread.sleep(10);
}
- // Now new requests arrive from queues C and D
- ApplicationAttemptId app7 =
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 1);
- ApplicationAttemptId app8 =
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 2);
- ApplicationAttemptId app9 =
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 3);
-
- ApplicationAttemptId app10 =
- createSchedulingRequest(1 * 1024, "queueD", "user1", 2, 1);
- ApplicationAttemptId app11 =
- createSchedulingRequest(1 * 1024, "queueD", "user1", 2, 2);
- ApplicationAttemptId app12 =
- createSchedulingRequest(1 * 1024, "queueD", "user1", 2, 3);
-
- scheduler.update();
-
- FSLeafQueue schedC =
- scheduler.getQueueManager().getLeafQueue("queueC", true);
- FSLeafQueue schedD =
- scheduler.getQueueManager().getLeafQueue("queueD", true);
+ // Verify the right amount of containers are preempted from greedyApp
+ assertEquals(4, greedyApp.getLiveContainers().size());
- assertTrue(Resources.equals(
- Resources.none(), scheduler.resourceDeficit(schedC, clock.getTime())));
- assertTrue(Resources.equals(
- Resources.none(), scheduler.resourceDeficit(schedD, clock.getTime())));
-
- // Test :
- // 1) whether componentWise min works as expected.
- // 2) DRF calculator is used
-
- // After minSharePreemptionTime has passed, they should want to preempt min
- // share.
- clock.tickSec(6);
- Resource res = scheduler.resourceDeficit(schedC, clock.getTime());
- assertEquals(1024, res.getMemorySize());
- // Demand = 3
- assertEquals(3, res.getVirtualCores());
-
- res = scheduler.resourceDeficit(schedD, clock.getTime());
- assertEquals(1024, res.getMemorySize());
- // Demand = 6, but min share = 2
- assertEquals(2, res.getVirtualCores());
-
- // After fairSharePreemptionTime has passed, they should want to preempt
- // fair share.
- scheduler.update();
- clock.tickSec(6);
- res = scheduler.resourceDeficit(schedC, clock.getTime());
- assertEquals(1536, res.getMemorySize());
- assertEquals(3, res.getVirtualCores());
+ sendEnoughNodeUpdatesToAssignFully();
- res = scheduler.resourceDeficit(schedD, clock.getTime());
- assertEquals(1536, res.getMemorySize());
- // Demand = 6, but fair share = 3
- assertEquals(3, res.getVirtualCores());
- stopResourceManager();
+ // Verify the preempted containers are assigned to starvingApp
+ assertEquals(2, starvingApp.getLiveContainers().size());
}
- @Test
- /**
- * Tests the various timing of decision to preempt tasks.
- */
- public void testPreemptionDecisionWithVariousTimeout() throws Exception {
- startResourceManagerWithRealFairScheduler();
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- ControlledClock clock = new ControlledClock();
- scheduler.setClock(clock);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("<maxResources>0mb,0vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>1</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>2</weight>");
- out.println("<minSharePreemptionTimeout>10</minSharePreemptionTimeout>");
- out.println("<fairSharePreemptionTimeout>25</fairSharePreemptionTimeout>");
- out.println("<queue name=\"queueB1\">");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("<minSharePreemptionTimeout>5</minSharePreemptionTimeout>");
- out.println("</queue>");
- out.println("<queue name=\"queueB2\">");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("<fairSharePreemptionTimeout>20</fairSharePreemptionTimeout>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("<queue name=\"queueC\">");
- out.println("<weight>1</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.print("<defaultMinSharePreemptionTimeout>15</defaultMinSharePreemptionTimeout>");
- out.print("<defaultFairSharePreemptionTimeout>30</defaultFairSharePreemptionTimeout>");
- out.println("</allocations>");
- out.close();
-
- scheduler.init(conf);
- scheduler.start();
- scheduler.reinitialize(conf, resourceManager.getRMContext());
-
- // Check the min/fair share preemption timeout for each queue
- QueueManager queueMgr = scheduler.getQueueManager();
- assertEquals(30000, queueMgr.getQueue("root")
- .getFairSharePreemptionTimeout());
- assertEquals(30000, queueMgr.getQueue("default")
- .getFairSharePreemptionTimeout());
- assertEquals(30000, queueMgr.getQueue("queueA")
- .getFairSharePreemptionTimeout());
- assertEquals(25000, queueMgr.getQueue("queueB")
- .getFairSharePreemptionTimeout());
- assertEquals(25000, queueMgr.getQueue("queueB.queueB1")
- .getFairSharePreemptionTimeout());
- assertEquals(20000, queueMgr.getQueue("queueB.queueB2")
- .getFairSharePreemptionTimeout());
- assertEquals(30000, queueMgr.getQueue("queueC")
- .getFairSharePreemptionTimeout());
- assertEquals(15000, queueMgr.getQueue("root")
- .getMinSharePreemptionTimeout());
- assertEquals(15000, queueMgr.getQueue("default")
- .getMinSharePreemptionTimeout());
- assertEquals(15000, queueMgr.getQueue("queueA")
- .getMinSharePreemptionTimeout());
- assertEquals(10000, queueMgr.getQueue("queueB")
- .getMinSharePreemptionTimeout());
- assertEquals(5000, queueMgr.getQueue("queueB.queueB1")
- .getMinSharePreemptionTimeout());
- assertEquals(10000, queueMgr.getQueue("queueB.queueB2")
- .getMinSharePreemptionTimeout());
- assertEquals(15000, queueMgr.getQueue("queueC")
- .getMinSharePreemptionTimeout());
-
- // Create one big node
- RMNode node1 =
- MockNodes.newNodeInfo(1, Resources.createResource(6 * 1024, 6), 1,
- "127.0.0.1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
-
- // Queue A takes all resources
- for (int i = 0; i < 6; i ++) {
- createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 1);
- }
-
- scheduler.update();
-
- // Sufficient node check-ins to fully schedule containers
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
- for (int i = 0; i < 6; i++) {
- scheduler.handle(nodeUpdate1);
+ private void verifyNoPreemption() throws InterruptedException {
+ // Sleep long enough to ensure not even one container is preempted.
+ for (int i = 0; i < 600; i++) {
+ if (greedyApp.getLiveContainers().size() != 8) {
+ break;
+ }
+ Thread.sleep(10);
}
-
- // Now new requests arrive from queues B1, B2 and C
- createSchedulingRequest(1 * 1024, "queueB.queueB1", "user1", 1, 1);
- createSchedulingRequest(1 * 1024, "queueB.queueB1", "user1", 1, 2);
- createSchedulingRequest(1 * 1024, "queueB.queueB1", "user1", 1, 3);
- createSchedulingRequest(1 * 1024, "queueB.queueB2", "user1", 1, 1);
- createSchedulingRequest(1 * 1024, "queueB.queueB2", "user1", 1, 2);
- createSchedulingRequest(1 * 1024, "queueB.queueB2", "user1", 1, 3);
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 1);
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 2);
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 3);
-
- scheduler.update();
-
- FSLeafQueue queueB1 = queueMgr.getLeafQueue("queueB.queueB1", true);
- FSLeafQueue queueB2 = queueMgr.getLeafQueue("queueB.queueB2", true);
- FSLeafQueue queueC = queueMgr.getLeafQueue("queueC", true);
-
- assertTrue(Resources.equals(
- Resources.none(), scheduler.resourceDeficit(queueB1, clock.getTime())));
- assertTrue(Resources.equals(
- Resources.none(), scheduler.resourceDeficit(queueB2, clock.getTime())));
- assertTrue(Resources.equals(
- Resources.none(), scheduler.resourceDeficit(queueC, clock.getTime())));
-
- // After 5 seconds, queueB1 wants to preempt min share
- scheduler.update();
- clock.tickSec(6);
- assertEquals(
- 1024, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemorySize());
- assertEquals(
- 0, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemorySize());
- assertEquals(
- 0, scheduler.resourceDeficit(queueC, clock.getTime()).getMemorySize());
-
- // After 10 seconds, queueB2 wants to preempt min share
- scheduler.update();
- clock.tickSec(5);
- assertEquals(
- 1024, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemorySize());
- assertEquals(
- 1024, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemorySize());
- assertEquals(
- 0, scheduler.resourceDeficit(queueC, clock.getTime()).getMemorySize());
-
- // After 15 seconds, queueC wants to preempt min share
- scheduler.update();
- clock.tickSec(5);
- assertEquals(
- 1024, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemorySize());
- assertEquals(
- 1024, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemorySize());
- assertEquals(
- 1024, scheduler.resourceDeficit(queueC, clock.getTime()).getMemorySize());
-
- // After 20 seconds, queueB2 should want to preempt fair share
- scheduler.update();
- clock.tickSec(5);
- assertEquals(
- 1024, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemorySize());
- assertEquals(
- 1536, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemorySize());
- assertEquals(
- 1024, scheduler.resourceDeficit(queueC, clock.getTime()).getMemorySize());
-
- // After 25 seconds, queueB1 should want to preempt fair share
- scheduler.update();
- clock.tickSec(5);
- assertEquals(
- 1536, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemorySize());
- assertEquals(
- 1536, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemorySize());
- assertEquals(
- 1024, scheduler.resourceDeficit(queueC, clock.getTime()).getMemorySize());
-
- // After 30 seconds, queueC should want to preempt fair share
- scheduler.update();
- clock.tickSec(5);
- assertEquals(
- 1536, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemorySize());
- assertEquals(
- 1536, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemorySize());
- assertEquals(
- 1536, scheduler.resourceDeficit(queueC, clock.getTime()).getMemorySize());
- stopResourceManager();
+ assertEquals(8, greedyApp.getLiveContainers().size());
}
@Test
- /**
- * Tests the decision to preempt tasks respect to non-preemptable queues
- * 1, Queues as follow:
- * queueA(non-preemptable)
- * queueB(preemptable)
- * parentQueue(non-preemptable)
- * --queueC(preemptable)
- * queueD(preemptable)
- * 2, Submit request to queueA, queueB, queueC, and all of them are over MinShare
- * 3, Now all resource are occupied
- * 4, Submit request to queueD, and need to preempt resource from other queues
- * 5, Only preemptable queue(queueB) would be preempted.
- */
- public void testPreemptionDecisionWithNonPreemptableQueue() throws Exception {
- startResourceManagerWithRealFairScheduler();
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- ControlledClock clock = new ControlledClock();
- scheduler.setClock(clock);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("<maxResources>0mb,0vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("<allowPreemptionFrom>false</allowPreemptionFrom>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"parentQueue\">");
- out.println("<allowPreemptionFrom>false</allowPreemptionFrom>");
- out.println("<queue name=\"queueC\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("<queue name=\"queueD\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>2048mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<defaultMinSharePreemptionTimeout>5</defaultMinSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionTimeout>10</defaultFairSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>");
- out.println("</allocations>");
- out.close();
-
- scheduler.init(conf);
- scheduler.start();
- scheduler.reinitialize(conf, resourceManager.getRMContext());
-
- // Create four nodes(3G each)
- RMNode node1 =
- MockNodes.newNodeInfo(1, Resources.createResource(3 * 1024, 3), 1,
- "127.0.0.1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
-
- RMNode node2 =
- MockNodes.newNodeInfo(1, Resources.createResource(3 * 1024, 3), 2,
- "127.0.0.2");
- NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
- scheduler.handle(nodeEvent2);
-
- RMNode node3 =
- MockNodes.newNodeInfo(1, Resources.createResource(3 * 1024, 3), 3,
- "127.0.0.3");
- NodeAddedSchedulerEvent nodeEvent3 = new NodeAddedSchedulerEvent(node3);
- scheduler.handle(nodeEvent3);
-
- RMNode node4 =
- MockNodes.newNodeInfo(1, Resources.createResource(3 * 1024, 3), 4,
- "127.0.0.4");
- NodeAddedSchedulerEvent nodeEvent4 = new NodeAddedSchedulerEvent(node4);
- scheduler.handle(nodeEvent4);
-
- // Submit apps to queueA, queueB, queueC,
- // now all resource of the cluster is occupied
- ApplicationAttemptId app1 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 4, 1);
- ApplicationAttemptId app2 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 4, 2);
- ApplicationAttemptId app3 =
- createSchedulingRequest(1 * 1024, "parentQueue.queueC", "user1", 4, 3);
-
- scheduler.update();
-
- // Sufficient node check-ins to fully schedule containers
- for (int i = 0; i < 3; i++) {
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
- scheduler.handle(nodeUpdate1);
-
- NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2);
- scheduler.handle(nodeUpdate2);
-
- NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3);
- scheduler.handle(nodeUpdate3);
-
- NodeUpdateSchedulerEvent nodeUpdate4 = new NodeUpdateSchedulerEvent(node4);
- scheduler.handle(nodeUpdate4);
- }
-
- assertEquals(4, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(4, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(4, scheduler.getSchedulerApp(app3).getLiveContainers().size());
-
- // Now new requests arrive from queues D
- ApplicationAttemptId app4 =
- createSchedulingRequest(1 * 1024, "queueD", "user1", 4, 1);
- scheduler.update();
- FSLeafQueue schedD =
- scheduler.getQueueManager().getLeafQueue("queueD", true);
-
- // After minSharePreemptionTime has passed, 2G resource should preempted from
- // queueB to queueD
- clock.tickSec(6);
- assertEquals(2048,
- scheduler.resourceDeficit(schedD, clock.getTime()).getMemorySize());
-
- scheduler.preemptResources(Resources.createResource(2 * 1024));
- // now only app2 is selected to be preempted
- assertTrue("App2 should have container to be preempted",
- !Collections.disjoint(
- scheduler.getSchedulerApp(app2).getLiveContainers(),
- scheduler.getSchedulerApp(app2).getPreemptionContainers()));
- assertTrue("App1 should not have container to be preempted",
- Collections.disjoint(
- scheduler.getSchedulerApp(app1).getLiveContainers(),
- scheduler.getSchedulerApp(app1).getPreemptionContainers()));
- assertTrue("App3 should not have container to be preempted",
- Collections.disjoint(
- scheduler.getSchedulerApp(app3).getLiveContainers(),
- scheduler.getSchedulerApp(app3).getPreemptionContainers()));
- // Pretend 20 seconds have passed
- clock.tickSec(20);
- scheduler.preemptResources(Resources.createResource(2 * 1024));
- for (int i = 0; i < 3; i++) {
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
- scheduler.handle(nodeUpdate1);
-
- NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2);
- scheduler.handle(nodeUpdate2);
-
- NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3);
- scheduler.handle(nodeUpdate3);
-
- NodeUpdateSchedulerEvent nodeUpdate4 = new NodeUpdateSchedulerEvent(node4);
- scheduler.handle(nodeUpdate4);
+ public void testPreemptionWithinSameLeafQueue() throws Exception {
+ setupCluster();
+ String queue = "root.preemptable.child-1";
+ submitApps(queue, queue);
+ if (fairsharePreemption) {
+ verifyPreemption();
+ } else {
+ verifyNoPreemption();
}
- // after preemption
- assertEquals(4, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(2, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(4, scheduler.getSchedulerApp(app3).getLiveContainers().size());
- assertEquals(2, scheduler.getSchedulerApp(app4).getLiveContainers().size());
- stopResourceManager();
}
@Test
- /**
- * Tests the decision to preempt tasks when allowPreemptionFrom is set false on
- * all queues.
- * Then none of them would be preempted actually.
- * 1, Queues as follow:
- * queueA(non-preemptable)
- * queueB(non-preemptable)
- * parentQueue(non-preemptable)
- * --queueC(preemptable)
- * parentQueue(preemptable)
- * --queueD(non-preemptable)
- * 2, Submit request to queueB, queueC, queueD, and all of them are over MinShare
- * 3, Now all resource are occupied
- * 4, Submit request to queueA, and need to preempt resource from other queues
- * 5, None of queues would be preempted.
- */
- public void testPreemptionDecisionWhenPreemptionDisabledOnAllQueues()
- throws Exception {
- startResourceManagerWithRealFairScheduler();
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- ControlledClock clock = new ControlledClock();
- scheduler.setClock(clock);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("<maxResources>0mb,0vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>2048mb,0vcores</minResources>");
- out.println("<allowPreemptionFrom>false</allowPreemptionFrom>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("<allowPreemptionFrom>false</allowPreemptionFrom>");
- out.println("</queue>");
- out.println("<queue name=\"parentQueue1\">");
- out.println("<allowPreemptionFrom>false</allowPreemptionFrom>");
- out.println("<queue name=\"queueC\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("<queue name=\"parentQueue2\">");
- out.println("<queue name=\"queueD\">");
- out.println("<weight>.25</weight>");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("<allowPreemptionFrom>false</allowPreemptionFrom>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("<defaultMinSharePreemptionTimeout>5</defaultMinSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionTimeout>10</defaultFairSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>");
- out.println("</allocations>");
- out.close();
-
- scheduler.init(conf);
- scheduler.start();
- scheduler.reinitialize(conf, resourceManager.getRMContext());
-
- // Create four nodes(3G each)
- RMNode node1 =
- MockNodes.newNodeInfo(1, Resources.createResource(3 * 1024, 3), 1,
- "127.0.0.1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
-
- RMNode node2 =
- MockNodes.newNodeInfo(1, Resources.createResource(3 * 1024, 3), 2,
- "127.0.0.2");
- NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
- scheduler.handle(nodeEvent2);
-
- RMNode node3 =
- MockNodes.newNodeInfo(1, Resources.createResource(3 * 1024, 3), 3,
- "127.0.0.3");
- NodeAddedSchedulerEvent nodeEvent3 = new NodeAddedSchedulerEvent(node3);
- scheduler.handle(nodeEvent3);
-
- RMNode node4 =
- MockNodes.newNodeInfo(1, Resources.createResource(3 * 1024, 3), 4,
- "127.0.0.4");
- NodeAddedSchedulerEvent nodeEvent4 = new NodeAddedSchedulerEvent(node4);
- scheduler.handle(nodeEvent4);
-
- // Submit apps to queueB, queueC, queueD
- // now all resource of the cluster is occupied
-
- ApplicationAttemptId app1 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 4, 1);
- ApplicationAttemptId app2 =
- createSchedulingRequest(1 * 1024, "parentQueue1.queueC", "user1", 4, 2);
- ApplicationAttemptId app3 =
- createSchedulingRequest(1 * 1024, "parentQueue2.queueD", "user1", 4, 3);
- scheduler.update();
-
- // Sufficient node check-ins to fully schedule containers
- for (int i = 0; i < 3; i++) {
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
- scheduler.handle(nodeUpdate1);
-
- NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2);
- scheduler.handle(nodeUpdate2);
-
- NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3);
- scheduler.handle(nodeUpdate3);
-
- NodeUpdateSchedulerEvent nodeUpdate4 = new NodeUpdateSchedulerEvent(node4);
- scheduler.handle(nodeUpdate4);
- }
-
- assertEquals(4, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(4, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(4, scheduler.getSchedulerApp(app3).getLiveContainers().size());
-
- // Now new requests arrive from queues A
- ApplicationAttemptId app4 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 4, 1);
- scheduler.update();
- FSLeafQueue schedA =
- scheduler.getQueueManager().getLeafQueue("queueA", true);
-
- // After minSharePreemptionTime has passed, resource deficit is 2G
- clock.tickSec(6);
- assertEquals(2048,
- scheduler.resourceDeficit(schedA, clock.getTime()).getMemorySize());
-
- scheduler.preemptResources(Resources.createResource(2 * 1024));
- // now none app is selected to be preempted
- assertTrue("App1 should have container to be preempted",
- Collections.disjoint(
- scheduler.getSchedulerApp(app1).getLiveContainers(),
- scheduler.getSchedulerApp(app1).getPreemptionContainers()));
- assertTrue("App2 should not have container to be preempted",
- Collections.disjoint(
- scheduler.getSchedulerApp(app2).getLiveContainers(),
- scheduler.getSchedulerApp(app2).getPreemptionContainers()));
- assertTrue("App3 should not have container to be preempted",
- Collections.disjoint(
- scheduler.getSchedulerApp(app3).getLiveContainers(),
- scheduler.getSchedulerApp(app3).getPreemptionContainers()));
- // Pretend 20 seconds have passed
- clock.tickSec(20);
- scheduler.preemptResources(Resources.createResource(2 * 1024));
- for (int i = 0; i < 3; i++) {
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
- scheduler.handle(nodeUpdate1);
-
- NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2);
- scheduler.handle(nodeUpdate2);
-
- NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3);
- scheduler.handle(nodeUpdate3);
-
- NodeUpdateSchedulerEvent nodeUpdate4 = new NodeUpdateSchedulerEvent(node4);
- scheduler.handle(nodeUpdate4);
- }
- // after preemption
- assertEquals(4, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(4, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(4, scheduler.getSchedulerApp(app3).getLiveContainers().size());
- assertEquals(0, scheduler.getSchedulerApp(app4).getLiveContainers().size());
- stopResourceManager();
+ public void testPreemptionBetweenTwoSiblingLeafQueues() throws Exception {
+ setupCluster();
+ submitApps("root.preemptable.child-1", "root.preemptable.child-2");
+ verifyPreemption();
}
@Test
- public void testBackwardsCompatiblePreemptionConfiguration() throws Exception {
- startResourceManagerWithRealFairScheduler();
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("</queue>");
- out.println("<queue name=\"queueA\">");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<queue name=\"queueB1\">");
- out.println("<minSharePreemptionTimeout>5</minSharePreemptionTimeout>");
- out.println("</queue>");
- out.println("<queue name=\"queueB2\">");
- out.println("</queue>");
- out.println("</queue>");
- out.println("<queue name=\"queueC\">");
- out.println("</queue>");
- out.print("<defaultMinSharePreemptionTimeout>15</defaultMinSharePreemptionTimeout>");
- out.print("<defaultFairSharePreemptionTimeout>30</defaultFairSharePreemptionTimeout>");
- out.print("<fairSharePreemptionTimeout>40</fairSharePreemptionTimeout>");
- out.println("</allocations>");
- out.close();
-
- scheduler.init(conf);
- scheduler.start();
- scheduler.reinitialize(conf, resourceManager.getRMContext());
-
- // Check the min/fair share preemption timeout for each queue
- QueueManager queueMgr = scheduler.getQueueManager();
- assertEquals(30000, queueMgr.getQueue("root")
- .getFairSharePreemptionTimeout());
- assertEquals(30000, queueMgr.getQueue("default")
- .getFairSharePreemptionTimeout());
- assertEquals(30000, queueMgr.getQueue("queueA")
- .getFairSharePreemptionTimeout());
- assertEquals(30000, queueMgr.getQueue("queueB")
- .getFairSharePreemptionTimeout());
- assertEquals(30000, queueMgr.getQueue("queueB.queueB1")
- .getFairSharePreemptionTimeout());
- assertEquals(30000, queueMgr.getQueue("queueB.queueB2")
- .getFairSharePreemptionTimeout());
- assertEquals(30000, queueMgr.getQueue("queueC")
- .getFairSharePreemptionTimeout());
- assertEquals(15000, queueMgr.getQueue("root")
- .getMinSharePreemptionTimeout());
- assertEquals(15000, queueMgr.getQueue("default")
- .getMinSharePreemptionTimeout());
- assertEquals(15000, queueMgr.getQueue("queueA")
- .getMinSharePreemptionTimeout());
- assertEquals(15000, queueMgr.getQueue("queueB")
- .getMinSharePreemptionTimeout());
- assertEquals(5000, queueMgr.getQueue("queueB.queueB1")
- .getMinSharePreemptionTimeout());
- assertEquals(15000, queueMgr.getQueue("queueB.queueB2")
- .getMinSharePreemptionTimeout());
- assertEquals(15000, queueMgr.getQueue("queueC")
- .getMinSharePreemptionTimeout());
-
- // If both exist, we take the default one
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("</queue>");
- out.println("<queue name=\"queueA\">");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<queue name=\"queueB1\">");
- out.println("<minSharePreemptionTimeout>5</minSharePreemptionTimeout>");
- out.println("</queue>");
- out.println("<queue name=\"queueB2\">");
- out.println("</queue>");
- out.println("</queue>");
- out.println("<queue name=\"queueC\">");
- out.println("</queue>");
- out.print("<defaultMinSharePreemptionTimeout>15</defaultMinSharePreemptionTimeout>");
- out.print("<defaultFairSharePreemptionTimeout>25</defaultFairSharePreemptionTimeout>");
- out.print("<fairSharePreemptionTimeout>30</fairSharePreemptionTimeout>");
- out.println("</allocations>");
- out.close();
-
- scheduler.reinitialize(conf, resourceManager.getRMContext());
-
- assertEquals(25000, queueMgr.getQueue("root")
- .getFairSharePreemptionTimeout());
- stopResourceManager();
+ public void testPreemptionBetweenNonSiblingQueues() throws Exception {
+ setupCluster();
+ submitApps("root.preemptable.child-1", "root.nonpreemptable.child-1");
+ verifyPreemption();
}
- @Test(timeout = 5000)
- public void testRecoverRequestAfterPreemption() throws Exception {
- startResourceManagerWithRealFairScheduler();
- conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10);
-
- ControlledClock clock = new ControlledClock();
- scheduler.setClock(clock);
- scheduler.init(conf);
- scheduler.start();
- scheduler.reinitialize(conf, resourceManager.getRMContext());
-
- SchedulerRequestKey schedulerKey = TestUtils.toSchedulerKey(20);
- String host = "127.0.0.1";
- int GB = 1024;
-
- // Create Node and raised Node Added event
- RMNode node = MockNodes.newNodeInfo(1,
- Resources.createResource(16 * 1024, 4), 0, host);
- NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
- scheduler.handle(nodeEvent);
-
- // Create 3 container requests and place it in ask
- List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
- ResourceRequest nodeLocalRequest = createResourceRequest(GB, 1, host,
- schedulerKey.getPriority().getPriority(), 1, true);
- ResourceRequest rackLocalRequest = createResourceRequest(GB, 1,
- node.getRackName(), schedulerKey.getPriority().getPriority(), 1,
- true);
- ResourceRequest offRackRequest = createResourceRequest(GB, 1,
- ResourceRequest.ANY, schedulerKey.getPriority().getPriority(), 1, true);
- ask.add(nodeLocalRequest);
- ask.add(rackLocalRequest);
- ask.add(offRackRequest);
-
- // Create Request and update
- ApplicationAttemptId appAttemptId = createSchedulingRequest("queueA",
- "user1", ask);
- scheduler.update();
-
- // Sufficient node check-ins to fully schedule containers
- NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
- scheduler.handle(nodeUpdate);
-
- assertEquals(1, scheduler.getSchedulerApp(appAttemptId).getLiveContainers()
- .size());
- SchedulerApplicationAttempt app = scheduler.getSchedulerApp(appAttemptId);
-
- // ResourceRequest will be empty once NodeUpdate is completed
- Assert.assertNull(app.getResourceRequest(schedulerKey, host));
-
- ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
- RMContainer rmContainer = app.getRMContainer(containerId1);
-
- // Create a preempt event and register for preemption
- scheduler.warnOrKillContainer(rmContainer);
-
- // Wait for few clock ticks
- clock.tickSec(5);
-
- // preempt now
- scheduler.warnOrKillContainer(rmContainer);
-
- // Trigger container rescheduled event
- scheduler.handle(new ContainerPreemptEvent(appAttemptId, rmContainer,
- SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE));
-
- List<ResourceRequest> requests = rmContainer.getResourceRequests();
- // Once recovered, resource request will be present again in app
- Assert.assertEquals(3, requests.size());
- for (ResourceRequest request : requests) {
- Assert.assertEquals(1,
- app.getResourceRequest(schedulerKey, request.getResourceName())
- .getNumContainers());
- }
-
- // Send node heartbeat
- scheduler.update();
- scheduler.handle(nodeUpdate);
-
- List<Container> containers = scheduler.allocate(appAttemptId,
- Collections.<ResourceRequest> emptyList(),
- Collections.<ContainerId> emptyList(), null, null, null, null).getContainers();
-
- // Now with updated ResourceRequest, a container is allocated for AM.
- Assert.assertTrue(containers.size() == 1);
- stopResourceManager();
+ @Test
+ public void testNoPreemptionFromDisallowedQueue() throws Exception {
+ setupCluster();
+ submitApps("root.nonpreemptable.child-1", "root.preemptable.child-1");
+ verifyNoPreemption();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java
new file mode 100644
index 0000000..5736f75
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * QueueManager tests that require a real scheduler
+ */
+public class TestQueueManagerRealScheduler extends FairSchedulerTestBase {
+ private final static File ALLOC_FILE = new File(TEST_DIR, "test-queue-mgr");
+
+ @Before
+ public void setup() throws IOException {
+ createConfiguration();
+ writeAllocFile(30, 40);
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ ALLOC_FILE.getAbsolutePath());
+
+ resourceManager = new MockRM(conf);
+ resourceManager.start();
+ scheduler = (FairScheduler) resourceManager.getResourceScheduler();
+ }
+
+ @After
+ public void teardown() {
+ ALLOC_FILE.deleteOnExit();
+ if (resourceManager != null) {
+ resourceManager.stop();
+ resourceManager = null;
+ }
+ }
+
+ private void writeAllocFile(int defaultFairShareTimeout,
+ int fairShareTimeout) throws IOException {
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"default\">");
+ out.println("</queue>");
+ out.println("<queue name=\"queueA\">");
+ out.println("</queue>");
+ out.println("<queue name=\"queueB\">");
+ out.println("<queue name=\"queueB1\">");
+ out.println("<minSharePreemptionTimeout>5</minSharePreemptionTimeout>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueB2\">");
+ out.println("</queue>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueC\">");
+ out.println("</queue>");
+ out.println("<defaultMinSharePreemptionTimeout>15"
+ + "</defaultMinSharePreemptionTimeout>");
+ out.println("<defaultFairSharePreemptionTimeout>" +
+ + defaultFairShareTimeout + "</defaultFairSharePreemptionTimeout>");
+ out.println("<fairSharePreemptionTimeout>"
+ + fairShareTimeout + "</fairSharePreemptionTimeout>");
+ out.println("</allocations>");
+ out.close();
+ }
+
+ @Test
+ public void testBackwardsCompatiblePreemptionConfiguration()
+ throws IOException {
+ // Check the min/fair share preemption timeout for each queue
+ QueueManager queueMgr = scheduler.getQueueManager();
+ assertEquals(30000, queueMgr.getQueue("root")
+ .getFairSharePreemptionTimeout());
+ assertEquals(30000, queueMgr.getQueue("default")
+ .getFairSharePreemptionTimeout());
+ assertEquals(30000, queueMgr.getQueue("queueA")
+ .getFairSharePreemptionTimeout());
+ assertEquals(30000, queueMgr.getQueue("queueB")
+ .getFairSharePreemptionTimeout());
+ assertEquals(30000, queueMgr.getQueue("queueB.queueB1")
+ .getFairSharePreemptionTimeout());
+ assertEquals(30000, queueMgr.getQueue("queueB.queueB2")
+ .getFairSharePreemptionTimeout());
+ assertEquals(30000, queueMgr.getQueue("queueC")
+ .getFairSharePreemptionTimeout());
+ assertEquals(15000, queueMgr.getQueue("root")
+ .getMinSharePreemptionTimeout());
+ assertEquals(15000, queueMgr.getQueue("default")
+ .getMinSharePreemptionTimeout());
+ assertEquals(15000, queueMgr.getQueue("queueA")
+ .getMinSharePreemptionTimeout());
+ assertEquals(15000, queueMgr.getQueue("queueB")
+ .getMinSharePreemptionTimeout());
+ assertEquals(5000, queueMgr.getQueue("queueB.queueB1")
+ .getMinSharePreemptionTimeout());
+ assertEquals(15000, queueMgr.getQueue("queueB.queueB2")
+ .getMinSharePreemptionTimeout());
+ assertEquals(15000, queueMgr.getQueue("queueC")
+ .getMinSharePreemptionTimeout());
+
+ // Lower the fairshare preemption timeouts and verify it is picked
+ // correctly.
+ writeAllocFile(25, 30);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+ assertEquals(25000, queueMgr.getQueue("root")
+ .getFairSharePreemptionTimeout());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
index dea2dd1..57c7301 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
@@ -330,11 +330,6 @@ public class TestSchedulingPolicy {
}
@Override
- public RMContainer preemptContainer() {
- throw new UnsupportedOperationException();
- }
-
- @Override
public Resource getFairShare() {
throw new UnsupportedOperationException();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/48] hadoop git commit: Revert "YARN-5921. Incorrect
synchronization in RMContextImpl#setHAServiceState/getHAServiceState.
Contributed by Varun Saxena."
Posted by li...@apache.org.
Revert "YARN-5921. Incorrect synchronization in RMContextImpl#setHAServiceState/getHAServiceState. Contributed by Varun Saxena."
This reverts commit 9f2960be19aa9c07c6b247bfec576c6b8e5f89bd.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2f77ca7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2f77ca7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2f77ca7
Branch: refs/heads/HADOOP-13345
Commit: e2f77ca77b80e3c5c7968301f6dc9b9273cded5d
Parents: 9f2960b
Author: Naganarasimha <na...@apache.org>
Authored: Sun Nov 27 23:32:59 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Sun Nov 27 23:32:59 2016 +0530
----------------------------------------------------------------------
.../TestTimelineReaderWebServicesHBaseStorage.java | 2 +-
.../timelineservice/storage/DataGeneratorForTest.java | 4 ++--
.../storage/TestHBaseTimelineStorageApps.java | 8 ++++----
.../storage/TestHBaseTimelineStorageEntities.java | 6 +++---
.../storage/flow/TestHBaseStorageFlowActivity.java | 6 +++---
.../storage/flow/TestHBaseStorageFlowRun.java | 14 +++++++-------
.../flow/TestHBaseStorageFlowRunCompaction.java | 2 +-
.../storage/HBaseTimelineWriterImpl.java | 5 +++++
8 files changed, 26 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f77ca7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index 17c01b5..2ed5d96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -334,7 +334,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.write(cluster, user, flow, flowVersion, runid, entity.getId(), te);
hbi.write(cluster, user, flow, flowVersion, runid, entity1.getId(), te1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f77ca7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
index 5cbb781..0938e9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
@@ -182,7 +182,7 @@ final class DataGeneratorForTest {
te2.addEntity(entity2);
HBaseTimelineWriterImpl hbi = null;
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(util.getConfiguration());
hbi.init(util.getConfiguration());
hbi.start();
String cluster = "cluster1";
@@ -360,7 +360,7 @@ final class DataGeneratorForTest {
te.addEntity(entity2);
HBaseTimelineWriterImpl hbi = null;
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(util.getConfiguration());
hbi.init(util.getConfiguration());
hbi.start();
String cluster = "cluster1";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f77ca7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
index b3e5197..e70198a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
@@ -155,7 +155,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.start();
String cluster = "cluster_check_null_application";
@@ -275,7 +275,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_write_app";
@@ -502,7 +502,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_events";
@@ -620,7 +620,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.start();
// Writing application entity.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f77ca7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
index 4b4c3e1..3076709 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
@@ -194,7 +194,7 @@ public class TestHBaseTimelineStorageEntities {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_write_entity";
@@ -391,7 +391,7 @@ public class TestHBaseTimelineStorageEntities {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_empty_eventkey";
@@ -496,7 +496,7 @@ public class TestHBaseTimelineStorageEntities {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.start();
String cluster = "clus!ter_\ttest_ev ents";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f77ca7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index 3ddb230..1906574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -119,7 +119,7 @@ public class TestHBaseStorageFlowActivity {
.getEntityMinStartTime(minStartTs);
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -228,7 +228,7 @@ public class TestHBaseStorageFlowActivity {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
String appName = "application_1111999999_1234";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -341,7 +341,7 @@ public class TestHBaseStorageFlowActivity {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
String appName = "application_11888888888_1111";
hbi.write(cluster, user, flow, flowVersion1, runid1, appName, te);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f77ca7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index 377611f..74b9e50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -184,7 +184,7 @@ public class TestHBaseStorageFlowRun {
.getEntityMinStartTime(minStartTs);
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -290,7 +290,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
String appName = "application_11111111111111_1111";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -559,7 +559,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
String appName = "application_11111111111111_1111";
hbi.write(cluster, user, flow, flowVersion, 1002345678919L, appName, te);
@@ -644,7 +644,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
String appName = "application_11111111111111_1111";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -737,7 +737,7 @@ public class TestHBaseStorageFlowRun {
TimelineEntity entityApp1 = null;
TimelineEntity entityApp2 = null;
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
for (int i = start; i < count; i++) {
@@ -826,7 +826,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L,
"application_11111111111111_1111", te);
@@ -908,7 +908,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L,
"application_11111111111111_1111", te);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f77ca7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
index 756d57b..3094088 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
@@ -287,7 +287,7 @@ public class TestHBaseStorageFlowRunCompaction {
TimelineEntities te1 = null;
TimelineEntity entityApp1 = null;
try {
- hbi = new HBaseTimelineWriterImpl();
+ hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
// now insert count * ( 100 + 100) metrics
// each call to getEntityMetricsApp1 brings back 100 values
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f77ca7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index b94b85f..3511a2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -101,6 +101,11 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
super(HBaseTimelineWriterImpl.class.getName());
}
+ public HBaseTimelineWriterImpl(Configuration conf) throws IOException {
+ super(conf.get("yarn.application.id",
+ HBaseTimelineWriterImpl.class.getName()));
+ }
+
/**
* initializes the hbase connection to write to the entity table.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/48] hadoop git commit: YARN-4206. Add Application timeouts in
Application report and CLI. Contributed by Rohith Sharma K S.
Posted by li...@apache.org.
YARN-4206. Add Application timeouts in Application report and CLI. Contributed by Rohith Sharma K S.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb0a483e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb0a483e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb0a483e
Branch: refs/heads/HADOOP-13345
Commit: eb0a483ed07399c7a06ed9db4bfbec382de470da
Parents: e15c20e
Author: Sunil <su...@apache.org>
Authored: Thu Nov 24 18:18:42 2016 +0530
Committer: Sunil <su...@apache.org>
Committed: Thu Nov 24 18:18:42 2016 +0530
----------------------------------------------------------------------
.../yarn/api/records/ApplicationReport.java | 10 ++
.../yarn/api/records/ApplicationTimeout.java | 99 ++++++++++++++
.../src/main/proto/yarn_protos.proto | 7 +
.../hadoop/yarn/client/api/YarnClient.java | 11 ++
.../yarn/client/api/impl/YarnClientImpl.java | 9 ++
.../hadoop/yarn/client/cli/ApplicationCLI.java | 45 ++++++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 40 ++++++
.../impl/pb/ApplicationReportPBImpl.java | 83 ++++++++++++
.../impl/pb/ApplicationTimeoutPBImpl.java | 130 +++++++++++++++++++
.../hadoop/yarn/api/TestPBImplRecords.java | 2 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 17 +++
.../rmapp/TestApplicationLifetimeMonitor.java | 30 ++++-
12 files changed, 479 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
index 33116a4..c039514 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.util.Records;
+import java.util.List;
import java.util.Set;
/**
@@ -447,4 +448,13 @@ public abstract class ApplicationReport {
@Unstable
public abstract void setAmNodeLabelExpression(String amNodeLabelExpression);
+
+ @Public
+ @Unstable
+ public abstract List<ApplicationTimeout> getApplicationTimeouts();
+
+ @Private
+ @Unstable
+ public abstract void setApplicationTimeouts(
+ List<ApplicationTimeout> timeouts);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
new file mode 100644
index 0000000..4beb11e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@code ApplicationTimeout} is a report for configured application timeouts.
+ * It includes details such as:
+ * <ul>
+ * <li>{@link ApplicationTimeoutType} of the timeout type.</li>
+ * <li>Expiry time in ISO8601 standard with format
+ * <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.</li>
+ * <li>Remaining time in seconds.</li>
+ * </ul>
+ */
+@Public
+@Unstable
+public abstract class ApplicationTimeout {
+
+ @Public
+ @Unstable
+ public static ApplicationTimeout newInstance(ApplicationTimeoutType type,
+ String expiryTime, long remainingTime) {
+ ApplicationTimeout timeouts = Records.newRecord(ApplicationTimeout.class);
+ timeouts.setTimeoutType(type);
+ timeouts.setExpiryTime(expiryTime);
+ timeouts.setRemainingTime(remainingTime);
+ return timeouts;
+ }
+
+ /**
+ * Get the application timeout type.
+ * @return timeoutType of an application timeout.
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationTimeoutType getTimeoutType();
+
+ /**
+ * Set the application timeout type.
+ * @param timeoutType of an application timeout.
+ */
+ @Public
+ @Unstable
+ public abstract void setTimeoutType(ApplicationTimeoutType timeoutType);
+
+ /**
+ * Get <code>expiryTime</code> for given timeout type.
+ * @return expiryTime in ISO8601 standard with format
+ * <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ */
+ @Public
+ @Unstable
+ public abstract String getExpiryTime();
+
+ /**
+ * Set <code>expiryTime</code> for given timeout type.
+ * @param expiryTime in ISO8601 standard with format
+ * <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ */
+ @Public
+ @Unstable
+ public abstract void setExpiryTime(String expiryTime);
+
+ /**
+ * Get <code>Remaining Time</code> of an application for given timeout type.
+ * @return Remaining Time in seconds.
+ */
+ @Public
+ @Unstable
+ public abstract long getRemainingTime();
+
+ /**
+ * Set <code>Remaining Time</code> of an application for given timeout type.
+ * @param remainingTime in seconds.
+ */
+ @Public
+ @Unstable
+ public abstract void setRemainingTime(long remainingTime);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index cb37126..43a661f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -214,6 +214,13 @@ message ApplicationReportProto {
optional PriorityProto priority = 23;
optional string appNodeLabelExpression = 24;
optional string amNodeLabelExpression = 25;
+ repeated ApplicationTimeoutProto application_timeouts = 26;
+}
+
+message ApplicationTimeoutProto {
+ required ApplicationTimeoutTypeProto application_timeout_type = 1;
+ optional string expire_time = 2;
+ optional int64 remaining_time = 3;
}
enum LogAggregationStatusProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 4e0ba2c..0bf1643 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -844,4 +846,13 @@ public abstract class YarnClient extends AbstractService {
*/
public abstract void signalToContainer(ContainerId containerId,
SignalContainerCommand command) throws YarnException, IOException;
+
+ @Public
+ @Unstable
+ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts(
+ UpdateApplicationTimeoutsRequest request)
+ throws YarnException, IOException {
+ throw new UnsupportedOperationException("The sub-class extending "
+ + YarnClient.class.getName() + " is expected to implement this !");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index f0fce22..a0f9678 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -84,6 +84,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -917,4 +919,11 @@ public class YarnClientImpl extends YarnClient {
SignalContainerRequest.newInstance(containerId, command);
rmClient.signalToContainer(request);
}
+
+ @Override
+ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts(
+ UpdateApplicationTimeoutsRequest request)
+ throws YarnException, IOException {
+ return rmClient.updateApplicationTimeouts(request);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index b841601..20a65bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -24,6 +24,7 @@ import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.text.DecimalFormat;
import java.util.Collection;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
@@ -39,11 +40,14 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -53,7 +57,6 @@ import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
@@ -84,6 +87,7 @@ public class ApplicationCLI extends YarnCLI {
public static final String CONTAINER = "container";
public static final String APP_ID = "appId";
public static final String UPDATE_PRIORITY = "updatePriority";
+ public static final String UPDATE_LIFETIME = "updateLifetime";
private boolean allAppStates;
@@ -139,6 +143,9 @@ public class ApplicationCLI extends YarnCLI {
opts.addOption(UPDATE_PRIORITY, true,
"update priority of an application. ApplicationId can be"
+ " passed using 'appId' option.");
+ opts.addOption(UPDATE_LIFETIME, true,
+ "update timeout of an application from NOW. ApplicationId can be"
+ + " passed using 'appId' option. Timeout value is in seconds.");
Option killOpt = new Option(KILL_CMD, true, "Kills the application. "
+ "Set of applications can be provided separated with space");
killOpt.setValueSeparator(' ');
@@ -150,6 +157,7 @@ public class ApplicationCLI extends YarnCLI {
opts.getOption(STATUS_CMD).setArgName("Application ID");
opts.getOption(APP_ID).setArgName("Application ID");
opts.getOption(UPDATE_PRIORITY).setArgName("Priority");
+ opts.getOption(UPDATE_LIFETIME).setArgName("Timeout");
} else if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
title = APPLICATION_ATTEMPT;
opts.addOption(STATUS_CMD, true,
@@ -296,6 +304,17 @@ public class ApplicationCLI extends YarnCLI {
}
updateApplicationPriority(cliParser.getOptionValue(APP_ID),
cliParser.getOptionValue(UPDATE_PRIORITY));
+ } else if (cliParser.hasOption(UPDATE_LIFETIME)) {
+ if (!cliParser.hasOption(APP_ID)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+
+ long timeoutInSec =
+ Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME));
+
+ updateApplicationTimeout(cliParser.getOptionValue(APP_ID),
+ ApplicationTimeoutType.LIFETIME, timeoutInSec);
} else if (cliParser.hasOption(SIGNAL_CMD)) {
if (args.length < 3 || args.length > 4) {
printUsage(title, opts);
@@ -316,6 +335,22 @@ public class ApplicationCLI extends YarnCLI {
return 0;
}
+ private void updateApplicationTimeout(String applicationId,
+ ApplicationTimeoutType timeoutType, long timeoutInSec)
+ throws YarnException, IOException {
+ ApplicationId appId = ApplicationId.fromString(applicationId);
+ String newTimeout =
+ Times.formatISO8601(System.currentTimeMillis() + timeoutInSec * 1000);
+ sysout.println("Updating timeout for given timeoutType: "
+ + timeoutType.toString() + " of an application " + applicationId);
+ UpdateApplicationTimeoutsRequest request = UpdateApplicationTimeoutsRequest
+ .newInstance(appId, Collections.singletonMap(timeoutType, newTimeout));
+ client.updateApplicationTimeouts(request);
+ sysout.println(
+ "Successfully updated " + timeoutType.toString() + " of an application "
+ + applicationId + ". New expiry time is " + newTimeout);
+ }
+
/**
* Signals the containerId
*
@@ -678,7 +713,13 @@ public class ApplicationCLI extends YarnCLI {
appReportStr.print("\tApplication Node Label Expression : ");
appReportStr.println(appReport.getAppNodeLabelExpression());
appReportStr.print("\tAM container Node Label Expression : ");
- appReportStr.print(appReport.getAmNodeLabelExpression());
+ appReportStr.println(appReport.getAmNodeLabelExpression());
+ for (ApplicationTimeout timeout : appReport.getApplicationTimeouts()) {
+ appReportStr.print("\tTimeoutType : " + timeout.getTimeoutType());
+ appReportStr.print("\tExpiryTime : " + timeout.getExpiryTime());
+ appReportStr.println(
+ "\tRemainingTime : " + timeout.getRemainingTime() + "seconds");
+ }
} else {
appReportStr.print("Application with id '" + applicationId
+ "' doesn't exist in RM.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 09dec89..f9ec5c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -36,6 +36,7 @@ import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashSet;
@@ -46,11 +47,14 @@ import java.util.regex.Pattern;
import org.apache.commons.cli.Options;
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -124,6 +128,11 @@ public class TestYarnCLI {
null, null, false, Priority.newInstance(0), "high-mem", "high-mem");
newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
newApplicationReport.setPriority(Priority.newInstance(0));
+ ApplicationTimeout timeout = ApplicationTimeout
+ .newInstance(ApplicationTimeoutType.LIFETIME, "UNLIMITED", -1);
+ newApplicationReport
+ .setApplicationTimeouts(Collections.singletonList(timeout));
+
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
@@ -155,6 +164,10 @@ public class TestYarnCLI {
pw.println("\tUnmanaged Application : false");
pw.println("\tApplication Node Label Expression : high-mem");
pw.println("\tAM container Node Label Expression : high-mem");
+ pw.print("\tTimeoutType : LIFETIME");
+ pw.print("\tExpiryTime : UNLIMITED");
+ pw.println("\tRemainingTime : -1seconds");
+ pw.println();
pw.close();
String appReportStr = baos.toString("UTF-8");
Assert.assertEquals(appReportStr, sysOutStream.toString());
@@ -1984,6 +1997,10 @@ public class TestYarnCLI {
pw.println(" specify which queue to move an");
pw.println(" application to.");
pw.println(" -status <Application ID> Prints the status of the application.");
+ pw.println(" -updateLifetime <Timeout> update timeout of an application from");
+ pw.println(" NOW. ApplicationId can be passed using");
+ pw.println(" 'appId' option. Timeout value is in");
+ pw.println(" seconds.");
pw.println(" -updatePriority <Priority> update priority of an application.");
pw.println(" ApplicationId can be passed using 'appId'");
pw.println(" option.");
@@ -2074,4 +2091,27 @@ public class TestYarnCLI {
applicationId.toString() });
assertEquals(0, result);
}
+
+ @Test(timeout = 60000)
+ public void testUpdateApplicationTimeout() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 6);
+
+ ApplicationReport appReport = ApplicationReport.newInstance(applicationId,
+ ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue",
+ "appname", "host", 124, null, YarnApplicationState.RUNNING,
+ "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null,
+ "N/A", 0.53789f, "YARN", null);
+ ApplicationTimeout timeout = ApplicationTimeout
+ .newInstance(ApplicationTimeoutType.LIFETIME, "N/A", -1);
+ appReport.setApplicationTimeouts(Collections.singletonList(timeout));
+ when(client.getApplicationReport(any(ApplicationId.class)))
+ .thenReturn(appReport);
+
+ int result = cli.run(new String[] { "application", "-appId",
+ applicationId.toString(), "-updateLifetime", "10" });
+ Assert.assertEquals(result, 0);
+ verify(client)
+ .updateApplicationTimeouts(any(UpdateApplicationTimeoutsRequest.class));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
index 1072815..f4987d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -35,6 +36,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationTimeoutProto;
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
@@ -42,7 +44,10 @@ import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import com.google.protobuf.TextFormat;
+import java.util.ArrayList;
import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
import java.util.Set;
@Private
@@ -58,6 +63,7 @@ public class ApplicationReportPBImpl extends ApplicationReport {
private Token amRmToken = null;
private Set<String> applicationTags = null;
private Priority priority = null;
+ private List<ApplicationTimeout> applicationTimeoutList = null;
public ApplicationReportPBImpl() {
builder = ApplicationReportProto.newBuilder();
@@ -492,6 +498,9 @@ public class ApplicationReportPBImpl extends ApplicationReport {
builder.getPriority())) {
builder.setPriority(convertToProtoFormat(this.priority));
}
+ if (this.applicationTimeoutList != null) {
+ addLocalApplicationTimeoutToProto();
+ }
}
private void mergeLocalToProto() {
@@ -668,4 +677,78 @@ public class ApplicationReportPBImpl extends ApplicationReport {
}
builder.setAmNodeLabelExpression((amNodeLabelExpression));
}
+
+ @Override
+ public List<ApplicationTimeout> getApplicationTimeouts() {
+ initLocalApplicationsList();
+ return this.applicationTimeoutList;
+ }
+
+ private void initLocalApplicationsList() {
+ if (this.applicationTimeoutList != null) {
+ return;
+ }
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ List<ApplicationTimeoutProto> list = p.getApplicationTimeoutsList();
+ this.applicationTimeoutList = new ArrayList<ApplicationTimeout>();
+
+ for (ApplicationTimeoutProto a : list) {
+ this.applicationTimeoutList.add(convertFromProtoFormat(a));
+ }
+ }
+
+ private void addLocalApplicationTimeoutToProto() {
+ maybeInitBuilder();
+ builder.clearApplicationTimeouts();
+ if (applicationTimeoutList == null) {
+ return;
+ }
+ Iterable<ApplicationTimeoutProto> iterable =
+ new Iterable<ApplicationTimeoutProto>() {
+ @Override
+ public Iterator<ApplicationTimeoutProto> iterator() {
+ return new Iterator<ApplicationTimeoutProto>() {
+
+ private Iterator<ApplicationTimeout> iter =
+ applicationTimeoutList.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public ApplicationTimeoutProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
+ builder.addAllApplicationTimeouts(iterable);
+ }
+
+ private ApplicationTimeoutPBImpl convertFromProtoFormat(
+ ApplicationTimeoutProto p) {
+ return new ApplicationTimeoutPBImpl(p);
+ }
+
+ private ApplicationTimeoutProto convertToProtoFormat(ApplicationTimeout t) {
+ return ((ApplicationTimeoutPBImpl) t).getProto();
+ }
+
+ @Override
+ public void setApplicationTimeouts(List<ApplicationTimeout> timeouts) {
+ maybeInitBuilder();
+ if (timeouts == null) {
+ builder.clearApplicationTimeouts();
+ }
+ this.applicationTimeoutList = timeouts;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationTimeoutPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationTimeoutPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationTimeoutPBImpl.java
new file mode 100644
index 0000000..086a089
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationTimeoutPBImpl.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationTimeoutProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationTimeoutProtoOrBuilder;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * PB implementation for ApplicationTimeout class.
+ */
+public class ApplicationTimeoutPBImpl extends ApplicationTimeout {
+
+ private ApplicationTimeoutProto proto =
+ ApplicationTimeoutProto.getDefaultInstance();
+ private ApplicationTimeoutProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public ApplicationTimeoutPBImpl() {
+ builder = ApplicationTimeoutProto.newBuilder();
+ }
+
+ public ApplicationTimeoutPBImpl(ApplicationTimeoutProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ApplicationTimeoutProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ApplicationTimeoutProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public ApplicationTimeoutType getTimeoutType() {
+ ApplicationTimeoutProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationTimeoutType()) {
+ return null;
+ }
+ return ProtoUtils.convertFromProtoFormat(p.getApplicationTimeoutType());
+ }
+
+ @Override
+ public void setTimeoutType(ApplicationTimeoutType type) {
+ maybeInitBuilder();
+ if (type == null) {
+ builder.clearApplicationTimeoutType();
+ return;
+ }
+ builder.setApplicationTimeoutType(ProtoUtils.convertToProtoFormat(type));
+ }
+
+ @Override
+ public String getExpiryTime() {
+ ApplicationTimeoutProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasExpireTime()) {
+ return null;
+ }
+ return p.getExpireTime();
+ }
+
+ @Override
+ public void setExpiryTime(String expiryTime) {
+ maybeInitBuilder();
+ if (expiryTime == null) {
+ builder.clearExpireTime();
+ return;
+ }
+ builder.setExpireTime(expiryTime);
+ }
+
+ @Override
+ public long getRemainingTime() {
+ ApplicationTimeoutProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getRemainingTime();
+ }
+
+ @Override
+ public void setRemainingTime(long remainingTime) {
+ maybeInitBuilder();
+ builder.setRemainingTime(remainingTime);
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 4b71282..10323d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -397,6 +398,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
generateByNewInstance(RestartContainerResponse.class);
generateByNewInstance(RollbackResponse.class);
generateByNewInstance(CommitResponse.class);
+ generateByNewInstance(ApplicationTimeout.class);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index a5962f7..0bf5f51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
@@ -112,6 +113,7 @@ import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -122,6 +124,8 @@ public class RMAppImpl implements RMApp, Recoverable {
private static final Log LOG = LogFactory.getLog(RMAppImpl.class);
private static final String UNAVAILABLE = "N/A";
+ private static final String UNLIMITED = "UNLIMITED";
+ private static final long UNKNOWN = -1L;
private static final EnumSet<RMAppState> COMPLETED_APP_STATES =
EnumSet.of(RMAppState.FINISHED, RMAppState.FINISHING, RMAppState.FAILED,
RMAppState.KILLED, RMAppState.FINAL_SAVING, RMAppState.KILLING);
@@ -787,6 +791,19 @@ public class RMAppImpl implements RMApp, Recoverable {
report.setUnmanagedApp(submissionContext.getUnmanagedAM());
report.setAppNodeLabelExpression(getAppNodeLabelExpression());
report.setAmNodeLabelExpression(getAmNodeLabelExpression());
+
+ ApplicationTimeout timeout = ApplicationTimeout
+ .newInstance(ApplicationTimeoutType.LIFETIME, UNLIMITED, UNKNOWN);
+ // Currently timeout type supported is LIFETIME. When more timeout types
+ // are supported in YARN-5692, the below logic need to be changed.
+ if (!this.applicationTimeouts.isEmpty()) {
+ long timeoutInMillis = applicationTimeouts
+ .get(ApplicationTimeoutType.LIFETIME).longValue();
+ timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
+ timeout.setRemainingTime(
+ Math.max((timeoutInMillis - systemClock.getTime()) / 1000, 0));
+ }
+ report.setApplicationTimeouts(Collections.singletonList(timeout));
return report;
} finally {
this.readLock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb0a483e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
index e803a88..aa4acca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
@@ -24,18 +24,23 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -101,8 +106,9 @@ public class TestApplicationLifetimeMonitor {
new HashMap<ApplicationTimeoutType, String>();
long newLifetime = 10L;
// update 10L seconds more to timeout
- updateTimeout.put(ApplicationTimeoutType.LIFETIME,
- Times.formatISO8601(System.currentTimeMillis() + newLifetime * 1000));
+ String formatISO8601 =
+ Times.formatISO8601(System.currentTimeMillis() + newLifetime * 1000);
+ updateTimeout.put(ApplicationTimeoutType.LIFETIME, formatISO8601);
UpdateApplicationTimeoutsRequest request =
UpdateApplicationTimeoutsRequest.newInstance(app2.getApplicationId(),
updateTimeout);
@@ -124,6 +130,26 @@ public class TestApplicationLifetimeMonitor {
Assert.assertTrue("Application lifetime value not updated",
afterUpdate > beforeUpdate);
+ // verify for application report.
+ RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
+ GetApplicationReportRequest appRequest =
+ recordFactory.newRecordInstance(GetApplicationReportRequest.class);
+ appRequest.setApplicationId(app2.getApplicationId());
+ List<ApplicationTimeout> applicationTimeoutList = rm.getRMContext()
+ .getClientRMService().getApplicationReport(appRequest)
+ .getApplicationReport().getApplicationTimeouts();
+ Assert.assertTrue("Application Timeout list are empty.",
+ !applicationTimeoutList.isEmpty());
+ ApplicationTimeout timeout = applicationTimeoutList.iterator().next();
+ Assert.assertEquals("Application timeout Type is incorrect.",
+ ApplicationTimeoutType.LIFETIME.toString(),
+ timeout.getTimeoutType().toString());
+ Assert.assertEquals("Application timeout string is incorrect.",
+ formatISO8601, timeout.getExpiryTime());
+ Assert.assertTrue("Application remaining time is incorrect",
+ timeout.getRemainingTime() > 0);
+
rm.waitForState(app2.getApplicationId(), RMAppState.KILLED);
// verify for app killed with updated lifetime
Assert.assertTrue("Application killed before lifetime value",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/48] hadoop git commit: YARN-5918. Handle Opportunistic scheduling
allocate request failure when NM is lost. (Bibin A Chundatt via asuresh)
Posted by li...@apache.org.
YARN-5918. Handle Opportunistic scheduling allocate request failure when NM is lost. (Bibin A Chundatt via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/005850b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/005850b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/005850b2
Branch: refs/heads/HADOOP-13345
Commit: 005850b28feb2f7bb8c2844d11e3f9d21b45d754
Parents: 3541ed8
Author: Arun Suresh <as...@apache.org>
Authored: Wed Nov 23 09:53:31 2016 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Nov 23 09:53:31 2016 -0800
----------------------------------------------------------------------
...pportunisticContainerAllocatorAMService.java | 13 ++-
.../distributed/NodeQueueLoadMonitor.java | 8 +-
...pportunisticContainerAllocatorAMService.java | 97 +++++++++++++++++++-
3 files changed, 110 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/005850b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index d3db96f..7814b84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.NodeQueueLoadMonitor;
@@ -409,15 +410,19 @@ public class OpportunisticContainerAllocatorAMService
private List<RemoteNode> convertToRemoteNodes(List<NodeId> nodeIds) {
ArrayList<RemoteNode> retNodes = new ArrayList<>();
for (NodeId nId : nodeIds) {
- retNodes.add(convertToRemoteNode(nId));
+ RemoteNode remoteNode = convertToRemoteNode(nId);
+ if (null != remoteNode) {
+ retNodes.add(remoteNode);
+ }
}
return retNodes;
}
private RemoteNode convertToRemoteNode(NodeId nodeId) {
- return RemoteNode.newInstance(nodeId,
- ((AbstractYarnScheduler)rmContext.getScheduler()).getNode(nodeId)
- .getHttpAddress());
+ SchedulerNode node =
+ ((AbstractYarnScheduler) rmContext.getScheduler()).getNode(nodeId);
+ return node != null ? RemoteNode.newInstance(nodeId, node.getHttpAddress())
+ : null;
}
private Resource createMaxContainerResource() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/005850b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
index 232b4ad..dec55ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
@@ -165,9 +165,11 @@ public class NodeQueueLoadMonitor implements ClusterMonitor {
}
@Override
- public void addNode(List<NMContainerStatus> containerStatuses, RMNode
- rmNode) {
- LOG.debug("Node added event from: " + rmNode.getNode().getName());
+ public void addNode(List<NMContainerStatus> containerStatuses,
+ RMNode rmNode) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Node added event from: " + rmNode.getNode().getName());
+ }
// Ignoring this currently : at least one NODE_UPDATE heartbeat is
// required to ensure node eligibility.
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/005850b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
index 3154d26..73d9e5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
@@ -61,11 +61,23 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RemoteNode;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSchedulingAllocateRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSchedulingAllocateResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterDistributedSchedulingAMResponsePBImpl;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
-
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mockito;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -77,6 +89,89 @@ import java.util.List;
*/
public class TestOpportunisticContainerAllocatorAMService {
+ private static final int GB = 1024;
+
+ @Test(timeout = 60000)
+ public void testNodeRemovalDuringAllocate() throws Exception {
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ YarnConfiguration conf = new YarnConfiguration(csConf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(
+ YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
+ conf.setInt(
+ YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS, 100);
+ MockRM rm = new MockRM(conf);
+ rm.start();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nm1.registerNode();
+ nm2.registerNode();
+ OpportunisticContainerAllocatorAMService amservice =
+ (OpportunisticContainerAllocatorAMService) rm
+ .getApplicationMasterService();
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ ApplicationAttemptId attemptId =
+ app1.getCurrentAppAttempt().getAppAttemptId();
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2);
+ ResourceScheduler scheduler = rm.getResourceScheduler();
+ RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+ RMNode rmNode2 = rm.getRMContext().getRMNodes().get(nm2.getNodeId());
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+ ((RMNodeImpl) rmNode1)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+ ((RMNodeImpl) rmNode2)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+ OpportunisticContainerContext ctxt = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(attemptId).getOpportunisticContainerContext();
+ // Send add and update node events to AM Service.
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode1));
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode2));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode2));
+ // Both node 1 and node 2 will be applicable for scheduling.
+ for (int i = 0; i < 10; i++) {
+ am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1),
+ "*", Resources.createResource(1 * GB), 2)),
+ null);
+ if (ctxt.getNodeMap().size() == 2) {
+ break;
+ }
+ Thread.sleep(50);
+ }
+ Assert.assertEquals(2, ctxt.getNodeMap().size());
+ // Remove node from scheduler but not from AM Service.
+ scheduler.handle(new NodeRemovedSchedulerEvent(rmNode1));
+ // After removal of node 1, only 1 node will be applicable for scheduling.
+ for (int i = 0; i < 10; i++) {
+ try {
+ am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1),
+ "*", Resources.createResource(1 * GB), 2)),
+ null);
+ } catch (Exception e) {
+ Assert.fail("Allocate request should be handled on node removal");
+ }
+ if (ctxt.getNodeMap().size() == 1) {
+ break;
+ }
+ Thread.sleep(50);
+ }
+ Assert.assertEquals(1, ctxt.getNodeMap().size());
+ }
+
+ private OpportunisticContainersStatus getOppurtunisticStatus(int waitTime,
+ int queueLength) {
+ OpportunisticContainersStatus status1 =
+ Mockito.mock(OpportunisticContainersStatus.class);
+ Mockito.when(status1.getEstimatedQueueWaitTime()).thenReturn(waitTime);
+ Mockito.when(status1.getWaitQueueLength()).thenReturn(queueLength);
+ return status1;
+ }
+
// Test if the OpportunisticContainerAllocatorAMService can handle both
// DSProtocol as well as AMProtocol clients
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/48] hadoop git commit: HADOOP-13833
TestSymlinkHdfsFileSystem#testCreateLinkUsingPartQualPath2 fails after
HADOOP-13605. Contributed by Brahma Reddy Battula.
Posted by li...@apache.org.
HADOOP-13833 TestSymlinkHdfsFileSystem#testCreateLinkUsingPartQualPath2 fails after HADOOP-13605. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3df3fa39
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3df3fa39
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3df3fa39
Branch: refs/heads/HADOOP-13345
Commit: 3df3fa396c7b338928f77344edd8c4dcda957671
Parents: 1350105
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 24 17:44:14 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 24 17:44:36 2016 +0000
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3df3fa39/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
index a5808e6..90e8c90 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
@@ -571,7 +571,8 @@ public abstract class SymlinkBaseTest {
GenericTestUtils.assertExceptionContains(
AbstractFileSystem.NO_ABSTRACT_FS_ERROR, e);
} else if (wrapper instanceof FileSystemTestWrapper) {
- assertEquals("No FileSystem for scheme: null", e.getMessage());
+ assertEquals("No FileSystem for scheme " + "\"" + "null" + "\"",
+ e.getMessage());
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/48] hadoop git commit: YARN-5920. Fix deadlock in
TestRMHA.testTransitionedToStandbyShouldNotHang. Contributed by Varun Saxena.
Posted by li...@apache.org.
YARN-5920. Fix deadlock in TestRMHA.testTransitionedToStandbyShouldNotHang. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e15c20ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e15c20ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e15c20ed
Branch: refs/heads/HADOOP-13345
Commit: e15c20edba1e9a23475ee6a4dfbadbdb8c1f668a
Parents: 1046852
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Thu Nov 24 12:18:38 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Thu Nov 24 12:18:38 2016 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e15c20ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
index 47c053c..000f4a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
@@ -465,8 +465,7 @@ public class TestRMHA {
MemoryRMStateStore memStore = new MemoryRMStateStore() {
@Override
- public synchronized void updateApplicationState(
- ApplicationStateData appState) {
+ public void updateApplicationState(ApplicationStateData appState) {
notifyStoreOperationFailed(new StoreFencedException());
}
};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/48] hadoop git commit: HADOOP-1381. The distance between sync
blocks in SequenceFiles should be configurable rather than hard coded to 2000
bytes. Contributed by Harsh J.
Posted by li...@apache.org.
HADOOP-1381. The distance between sync blocks in SequenceFiles should be configurable rather than hard coded to 2000 bytes. Contributed by Harsh J.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07825f2b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07825f2b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07825f2b
Branch: refs/heads/HADOOP-13345
Commit: 07825f2b49384dbec92bfae87ea661cef9ffab49
Parents: ee3d437
Author: Harsh J <ha...@cloudera.com>
Authored: Wed Oct 26 20:04:33 2016 +0530
Committer: Harsh J <ha...@cloudera.com>
Committed: Fri Nov 25 22:22:23 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/io/SequenceFile.java | 74 ++++++++----
.../apache/hadoop/io/TestSequenceFileSync.java | 113 +++++++++++++++----
2 files changed, 146 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07825f2b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index 2ac1389..c510ff7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -24,6 +24,7 @@ import java.util.*;
import java.rmi.server.UID;
import java.security.MessageDigest;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.*;
import org.apache.hadoop.util.Options;
import org.apache.hadoop.fs.*;
@@ -146,7 +147,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* </ul>
* </li>
* <li>
- * A sync-marker every few <code>100</code> bytes or so.
+ * A sync-marker every few <code>100</code> kilobytes or so.
* </li>
* </ul>
*
@@ -165,7 +166,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* </ul>
* </li>
* <li>
- * A sync-marker every few <code>100</code> bytes or so.
+ * A sync-marker every few <code>100</code> kilobytes or so.
* </li>
* </ul>
*
@@ -217,8 +218,11 @@ public class SequenceFile {
private static final int SYNC_HASH_SIZE = 16; // number of bytes in hash
private static final int SYNC_SIZE = 4+SYNC_HASH_SIZE; // escape + hash
- /** The number of bytes between sync points.*/
- public static final int SYNC_INTERVAL = 100*SYNC_SIZE;
+ /**
+ * The number of bytes between sync points. 100 KB, default.
+ * Computed as 5 KB * 20 = 100 KB
+ */
+ public static final int SYNC_INTERVAL = 5 * 1024 * SYNC_SIZE; // 5KB*(16+4)
/**
* The compression type used to compress key/value pairs in the
@@ -856,6 +860,9 @@ public class SequenceFile {
// starts and ends by scanning for this value.
long lastSyncPos; // position of last sync
byte[] sync; // 16 random bytes
+ @VisibleForTesting
+ int syncInterval;
+
{
try {
MessageDigest digester = MessageDigest.getInstance("MD5");
@@ -987,7 +994,16 @@ public class SequenceFile {
private static Option filesystem(FileSystem fs) {
return new SequenceFile.Writer.FileSystemOption(fs);
}
-
+
+ private static class SyncIntervalOption extends Options.IntegerOption
+ implements Option {
+ SyncIntervalOption(int val) {
+ // If a negative sync interval is provided,
+ // fall back to the default sync interval.
+ super(val < 0 ? SYNC_INTERVAL : val);
+ }
+ }
+
public static Option bufferSize(int value) {
return new BufferSizeOption(value);
}
@@ -1032,11 +1048,15 @@ public class SequenceFile {
CompressionCodec codec) {
return new CompressionOption(value, codec);
}
-
+
+ public static Option syncInterval(int value) {
+ return new SyncIntervalOption(value);
+ }
+
/**
* Construct a uncompressed writer from a set of options.
* @param conf the configuration to use
- * @param options the options used when creating the writer
+ * @param opts the options used when creating the writer
* @throws IOException if it fails
*/
Writer(Configuration conf,
@@ -1062,6 +1082,8 @@ public class SequenceFile {
Options.getOption(MetadataOption.class, opts);
CompressionOption compressionTypeOption =
Options.getOption(CompressionOption.class, opts);
+ SyncIntervalOption syncIntervalOption =
+ Options.getOption(SyncIntervalOption.class, opts);
// check consistency of options
if ((fileOption == null) == (streamOption == null)) {
throw new IllegalArgumentException("file or stream must be specified");
@@ -1163,7 +1185,12 @@ public class SequenceFile {
"GzipCodec without native-hadoop " +
"code!");
}
- init(conf, out, ownStream, keyClass, valueClass, codec, metadata);
+ this.syncInterval = (syncIntervalOption == null) ?
+ SYNC_INTERVAL :
+ syncIntervalOption.getValue();
+ init(
+ conf, out, ownStream, keyClass, valueClass,
+ codec, metadata, syncInterval);
}
/** Create the named file.
@@ -1176,7 +1203,7 @@ public class SequenceFile {
Class keyClass, Class valClass) throws IOException {
this.compress = CompressionType.NONE;
init(conf, fs.create(name), true, keyClass, valClass, null,
- new Metadata());
+ new Metadata(), SYNC_INTERVAL);
}
/** Create the named file with write-progress reporter.
@@ -1190,7 +1217,7 @@ public class SequenceFile {
Progressable progress, Metadata metadata) throws IOException {
this.compress = CompressionType.NONE;
init(conf, fs.create(name, progress), true, keyClass, valClass,
- null, metadata);
+ null, metadata, SYNC_INTERVAL);
}
/** Create the named file with write-progress reporter.
@@ -1206,7 +1233,7 @@ public class SequenceFile {
this.compress = CompressionType.NONE;
init(conf,
fs.create(name, true, bufferSize, replication, blockSize, progress),
- true, keyClass, valClass, null, metadata);
+ true, keyClass, valClass, null, metadata, SYNC_INTERVAL);
}
boolean isCompressed() { return compress != CompressionType.NONE; }
@@ -1234,18 +1261,21 @@ public class SequenceFile {
/** Initialize. */
@SuppressWarnings("unchecked")
- void init(Configuration conf, FSDataOutputStream out, boolean ownStream,
- Class keyClass, Class valClass,
- CompressionCodec codec, Metadata metadata)
+ void init(Configuration config, FSDataOutputStream outStream,
+ boolean ownStream, Class key, Class val,
+ CompressionCodec compCodec, Metadata meta,
+ int syncIntervalVal)
throws IOException {
- this.conf = conf;
- this.out = out;
+ this.conf = config;
+ this.out = outStream;
this.ownOutputStream = ownStream;
- this.keyClass = keyClass;
- this.valClass = valClass;
- this.codec = codec;
- this.metadata = metadata;
- SerializationFactory serializationFactory = new SerializationFactory(conf);
+ this.keyClass = key;
+ this.valClass = val;
+ this.codec = compCodec;
+ this.metadata = meta;
+ this.syncInterval = syncIntervalVal;
+ SerializationFactory serializationFactory =
+ new SerializationFactory(config);
this.keySerializer = serializationFactory.getSerializer(keyClass);
if (this.keySerializer == null) {
throw new IOException(
@@ -1366,7 +1396,7 @@ public class SequenceFile {
synchronized void checkAndWriteSync() throws IOException {
if (sync != null &&
- out.getPos() >= lastSyncPos+SYNC_INTERVAL) { // time to emit sync
+ out.getPos() >= lastSyncPos+this.syncInterval) { // time to emit sync
sync();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07825f2b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSync.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSync.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSync.java
index bceb8af..363177b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSync.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSync.java
@@ -27,13 +27,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
+/** Tests sync based seek reads/write intervals inside SequenceFiles. */
public class TestSequenceFileSync {
private static final int NUMRECORDS = 2000;
private static final int RECORDSIZE = 80;
- private static final Random rand = new Random();
+ private static final Random RAND = new Random();
private final static String REC_FMT = "%d RECORDID %d : ";
@@ -46,37 +48,110 @@ public class TestSequenceFileSync {
reader.next(key, val);
assertEquals(key.get(), expectedRecord);
final String test = String.format(REC_FMT, expectedRecord, expectedRecord);
- assertEquals("Invalid value " + val, 0, val.find(test, 0));
+ assertEquals(
+ "Invalid value in iter " + iter + ": " + val,
+ 0,
+ val.find(test, 0));
+ }
+
+ @Test
+ public void testDefaultSyncInterval() throws IOException {
+ // Uses the default sync interval of 100 KB
+ final Configuration conf = new Configuration();
+ final FileSystem fs = FileSystem.getLocal(conf);
+ final Path path = new Path(GenericTestUtils.getTempPath(
+ "sequencefile.sync.test"));
+ final IntWritable input = new IntWritable();
+ final Text val = new Text();
+ SequenceFile.Writer writer = new SequenceFile.Writer(
+ conf,
+ SequenceFile.Writer.file(path),
+ SequenceFile.Writer.compression(CompressionType.NONE),
+ SequenceFile.Writer.keyClass(IntWritable.class),
+ SequenceFile.Writer.valueClass(Text.class)
+ );
+ try {
+ writeSequenceFile(writer, NUMRECORDS*4);
+ for (int i = 0; i < 5; i++) {
+ final SequenceFile.Reader reader;
+
+ //try different SequenceFile.Reader constructors
+ if (i % 2 == 0) {
+ final int buffersize = conf.getInt("io.file.buffer.size", 4096);
+ reader = new SequenceFile.Reader(conf,
+ SequenceFile.Reader.file(path),
+ SequenceFile.Reader.bufferSize(buffersize));
+ } else {
+ final FSDataInputStream in = fs.open(path);
+ final long length = fs.getFileStatus(path).getLen();
+ reader = new SequenceFile.Reader(conf,
+ SequenceFile.Reader.stream(in),
+ SequenceFile.Reader.start(0L),
+ SequenceFile.Reader.length(length));
+ }
+
+ try {
+ forOffset(reader, input, val, i, 0, 0);
+ forOffset(reader, input, val, i, 65, 0);
+ // There would be over 1000 records within
+ // this sync interval
+ forOffset(reader, input, val, i, 2000, 1101);
+ forOffset(reader, input, val, i, 0, 0);
+ } finally {
+ reader.close();
+ }
+ }
+ } finally {
+ fs.delete(path, false);
+ }
}
@Test
public void testLowSyncpoint() throws IOException {
+ // Uses a smaller sync interval of 2000 bytes
final Configuration conf = new Configuration();
final FileSystem fs = FileSystem.getLocal(conf);
final Path path = new Path(GenericTestUtils.getTempPath(
"sequencefile.sync.test"));
final IntWritable input = new IntWritable();
final Text val = new Text();
- SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path,
- IntWritable.class, Text.class);
+ SequenceFile.Writer writer = new SequenceFile.Writer(
+ conf,
+ SequenceFile.Writer.file(path),
+ SequenceFile.Writer.compression(CompressionType.NONE),
+ SequenceFile.Writer.keyClass(IntWritable.class),
+ SequenceFile.Writer.valueClass(Text.class),
+ SequenceFile.Writer.syncInterval(20*100)
+ );
+ // Ensure the custom sync interval value is set
+ assertEquals(writer.syncInterval, 20*100);
try {
writeSequenceFile(writer, NUMRECORDS);
- for (int i = 0; i < 5 ; i++) {
- final SequenceFile.Reader reader;
+ for (int i = 0; i < 5; i++) {
+ final SequenceFile.Reader reader;
- //try different SequenceFile.Reader constructors
- if (i % 2 == 0) {
- reader = new SequenceFile.Reader(fs, path, conf);
- } else {
- final FSDataInputStream in = fs.open(path);
- final long length = fs.getFileStatus(path).getLen();
- final int buffersize = conf.getInt("io.file.buffer.size", 4096);
- reader = new SequenceFile.Reader(in, buffersize, 0L, length, conf);
- }
+ //try different SequenceFile.Reader constructors
+ if (i % 2 == 0) {
+ final int bufferSize = conf.getInt("io.file.buffer.size", 4096);
+ reader = new SequenceFile.Reader(
+ conf,
+ SequenceFile.Reader.file(path),
+ SequenceFile.Reader.bufferSize(bufferSize));
+ } else {
+ final FSDataInputStream in = fs.open(path);
+ final long length = fs.getFileStatus(path).getLen();
+ reader = new SequenceFile.Reader(
+ conf,
+ SequenceFile.Reader.stream(in),
+ SequenceFile.Reader.start(0L),
+ SequenceFile.Reader.length(length));
+ }
- try {
+ try {
forOffset(reader, input, val, i, 0, 0);
forOffset(reader, input, val, i, 65, 0);
+ // There would be only a few records within
+ // this sync interval
forOffset(reader, input, val, i, 2000, 21);
forOffset(reader, input, val, i, 0, 0);
} finally {
@@ -88,7 +163,7 @@ public class TestSequenceFileSync {
}
}
- public static void writeSequenceFile(SequenceFile.Writer writer,
+ private static void writeSequenceFile(SequenceFile.Writer writer,
int numRecords) throws IOException {
final IntWritable key = new IntWritable();
final Text val = new Text();
@@ -100,13 +175,13 @@ public class TestSequenceFileSync {
writer.close();
}
- static void randomText(Text val, int id, int recordSize) {
+ private static void randomText(Text val, int id, int recordSize) {
val.clear();
final StringBuilder ret = new StringBuilder(recordSize);
ret.append(String.format(REC_FMT, id, id));
recordSize -= ret.length();
for (int i = 0; i < recordSize; ++i) {
- ret.append(rand.nextInt(9));
+ ret.append(RAND.nextInt(9));
}
val.set(ret.toString());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/48] hadoop git commit: YARN-5890. FairScheduler should log
information about AM-resource-usage and max-AM-share for queues (Contributed
by Yufei Gu via Daniel Templeton)
Posted by li...@apache.org.
YARN-5890. FairScheduler should log information about AM-resource-usage and max-AM-share for queues
(Contributed by Yufei Gu via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b9d3acd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b9d3acd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b9d3acd
Branch: refs/heads/HADOOP-13345
Commit: 3b9d3acd203cef4d861c5182fc4dccc55128d347
Parents: b407d53
Author: Daniel Templeton <te...@apache.org>
Authored: Tue Nov 29 12:44:08 2016 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Tue Nov 29 12:46:05 2016 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSLeafQueue.java | 42 ++++--
.../resourcemanager/scheduler/fair/FSQueue.java | 7 +
.../scheduler/fair/FSQueueMetrics.java | 60 ++++++++
.../scheduler/fair/TestFairScheduler.java | 137 +++++++++++++++++++
4 files changed, 232 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b9d3acd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 343e9c3..2754616 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -81,6 +81,7 @@ public class FSLeafQueue extends FSQueue {
this.lastTimeAtMinShare = scheduler.getClock().getTime();
activeUsersManager = new ActiveUsersManager(getMetrics());
amResourceUsage = Resource.newInstance(0, 0);
+ getMetrics().setAMResourceUsage(amResourceUsage);
}
void addApp(FSAppAttempt app, boolean runnable) {
@@ -132,6 +133,7 @@ public class FSLeafQueue extends FSQueue {
// running an unmanaged AM.
if (runnable && app.isAmRunning()) {
Resources.subtractFrom(amResourceUsage, app.getAMResource());
+ getMetrics().setAMResourceUsage(amResourceUsage);
}
return runnable;
@@ -468,19 +470,14 @@ public class FSLeafQueue extends FSQueue {
}
/**
- * Check whether this queue can run this application master under the
- * maxAMShare limit.
- *
- * @param amResource resources required to run the AM
- * @return true if this queue can run
- */
- boolean canRunAppAM(Resource amResource) {
- if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
- return true;
- }
-
- // If FairShare is zero, use min(maxShare, available resource) to compute
- // maxAMResource
+ * Compute the maximum resource AM can use. The value is the result of
+ * multiplying FairShare and maxAMShare. If FairShare is zero, use
+ * min(maxShare, available resource) instead to prevent zero value for
+ * maximum AM resource since it forbids any job running in the queue.
+ *
+ * @return the maximum resource AM can use
+ */
+ private Resource computeMaxAMResource() {
Resource maxResource = Resources.clone(getFairShare());
if (maxResource.getMemorySize() == 0) {
maxResource.setMemorySize(
@@ -494,7 +491,23 @@ public class FSLeafQueue extends FSQueue {
getMaxShare().getVirtualCores()));
}
- Resource maxAMResource = Resources.multiply(maxResource, maxAMShare);
+ return Resources.multiply(maxResource, maxAMShare);
+ }
+
+ /**
+ * Check whether this queue can run the Application Master under the
+ * maxAMShare limit.
+ *
+ * @param amResource resources required to run the AM
+ * @return true if this queue can run
+ */
+ public boolean canRunAppAM(Resource amResource) {
+ if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
+ return true;
+ }
+
+ Resource maxAMResource = computeMaxAMResource();
+ getMetrics().setMaxAMShare(maxAMResource);
Resource ifRunAMResource = Resources.add(amResourceUsage, amResource);
return Resources.fitsIn(ifRunAMResource, maxAMResource);
}
@@ -502,6 +515,7 @@ public class FSLeafQueue extends FSQueue {
void addAMResourceUsage(Resource amResource) {
if (amResource != null) {
Resources.addTo(amResourceUsage, amResource);
+ getMetrics().setAMResourceUsage(amResourceUsage);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b9d3acd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 572b5f9..38c0340 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -41,6 +41,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
+
@Private
@Unstable
public abstract class FSQueue implements Queue, Schedulable {
@@ -160,6 +162,11 @@ public abstract class FSQueue implements Queue, Schedulable {
return maxRunningApps;
}
+ @VisibleForTesting
+ protected float getMaxAMShare() {
+ return maxAMShare;
+ }
+
public void setMaxAMShare(float maxAMShare){
this.maxAMShare = maxAMShare;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b9d3acd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
index 42c8825..a970815 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
@@ -41,6 +41,10 @@ public class FSQueueMetrics extends QueueMetrics {
@Metric("Maximum share of memory in MB") MutableGaugeLong maxShareMB;
@Metric("Maximum share of CPU in vcores") MutableGaugeLong maxShareVCores;
@Metric("Maximum number of applications") MutableGaugeInt maxApps;
+ @Metric("Maximum AM share of memory in MB") MutableGaugeLong maxAMShareMB;
+ @Metric("Maximum AM share of CPU in vcores") MutableGaugeInt maxAMShareVCores;
+ @Metric("AM resource usage of memory in MB") MutableGaugeLong amResourceUsageMB;
+ @Metric("AM resource usage of CPU in vcores") MutableGaugeInt amResourceUsageVCores;
private String schedulingPolicy;
@@ -109,6 +113,62 @@ public class FSQueueMetrics extends QueueMetrics {
maxApps.set(max);
}
+ /**
+ * Get the maximum memory size AM can use in MB.
+ *
+ * @return the maximum memory size AM can use
+ */
+ public long getMaxAMShareMB() {
+ return maxAMShareMB.value();
+ }
+
+ /**
+ * Get the maximum number of VCores AM can use.
+ *
+ * @return the maximum number of VCores AM can use
+ */
+ public int getMaxAMShareVCores() {
+ return maxAMShareVCores.value();
+ }
+
+ /**
+ * Set the maximum resource AM can use.
+ *
+ * @param resource the maximum resource AM can use
+ */
+ public void setMaxAMShare(Resource resource) {
+ maxAMShareMB.set(resource.getMemorySize());
+ maxAMShareVCores.set(resource.getVirtualCores());
+ }
+
+ /**
+ * Get the AM memory usage in MB.
+ *
+ * @return the AM memory usage
+ */
+ public long getAMResourceUsageMB() {
+ return amResourceUsageMB.value();
+ }
+
+ /**
+ * Get the AM VCore usage.
+ *
+ * @return the AM VCore usage
+ */
+ public int getAMResourceUsageVCores() {
+ return amResourceUsageVCores.value();
+ }
+
+ /**
+ * Set the AM resource usage.
+ *
+ * @param resource the AM resource usage
+ */
+ public void setAMResourceUsage(Resource resource) {
+ amResourceUsageMB.set(resource.getMemorySize());
+ amResourceUsageVCores.set(resource.getVirtualCores());
+ }
+
public String getSchedulingPolicy() {
return schedulingPolicy;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b9d3acd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index ffbfec8..c5ff5e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -594,6 +594,143 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals(0, queue.getFairShare().getMemorySize());
}
+ /**
+ * Test if we compute the maximum AM resource correctly.
+ *
+ * @throws IOException if scheduler reinitialization fails
+ */
+ @Test
+ public void testComputeMaxAMResource() throws IOException {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queueFSZeroWithMax\">");
+ out.println("<weight>0</weight>");
+ out.println("<maxAMShare>0.5</maxAMShare>");
+ out.println("<maxResources>4096 mb 4 vcores</maxResources>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueFSZeroWithAVL\">");
+ out.println("<weight>0.0</weight>");
+ out.println("<maxAMShare>0.5</maxAMShare>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueFSNonZero\">");
+ out.println("<weight>1</weight>");
+ out.println("<maxAMShare>0.5</maxAMShare>");
+ out.println("</queue>");
+ out.println("<defaultQueueSchedulingPolicy>drf" +
+ "</defaultQueueSchedulingPolicy>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ long memCapacity = 20 * GB;
+ int cpuCapacity = 20;
+ RMNode node =
+ MockNodes.newNodeInfo(1, Resources.createResource(memCapacity,
+ cpuCapacity), 0, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+ scheduler.update();
+
+ Resource amResource = Resource.newInstance(1 * GB, 1);
+ int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority();
+
+ // queueFSZeroWithMax
+ FSLeafQueue queueFSZeroWithMax = scheduler.getQueueManager().
+ getLeafQueue("queueFSZeroWithMax", true);
+ ApplicationAttemptId attId1 = createAppAttemptId(1, 1);
+ createApplicationWithAMResource(attId1, "queueFSZeroWithMax", "user1",
+ amResource);
+ createSchedulingRequestExistingApplication(1 * GB, 1, amPriority, attId1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // queueFSZeroWithMax's weight is 0.0, so its fair share should be 0, we use
+ // the min(maxShare, available resource) to compute maxAMShare, in this
+ // case, we use maxShare, since it is smaller than available resource.
+ assertEquals("QueueFSZeroWithMax's fair share should be zero",
+ 0, queueFSZeroWithMax.getFairShare().getMemorySize());
+ assertEquals("QueueFSZeroWithMax's maximum AM resource should be "
+ + "maxShare * maxAMShare",
+ (long)(queueFSZeroWithMax.getMaxShare().getMemorySize() *
+ queueFSZeroWithMax.getMaxAMShare()),
+ queueFSZeroWithMax.getMetrics().getMaxAMShareMB());
+ assertEquals("QueueFSZeroWithMax's maximum AM resource should be "
+ + "maxShare * maxAMShare",
+ (long)(queueFSZeroWithMax.getMaxShare().getVirtualCores() *
+ queueFSZeroWithMax.getMaxAMShare()),
+ queueFSZeroWithMax.getMetrics().getMaxAMShareVCores());
+ assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to "
+ + "AM resource request",
+ amResource.getMemorySize(),
+ queueFSZeroWithMax.getMetrics().getAMResourceUsageMB());
+
+ // queueFSZeroWithAVL
+ amResource = Resources.createResource(1 * GB, 1);
+ FSLeafQueue queueFSZeroWithAVL = scheduler.getQueueManager().
+ getLeafQueue("queueFSZeroWithAVL", true);
+ ApplicationAttemptId attId2 = createAppAttemptId(2, 1);
+ createApplicationWithAMResource(attId2, "queueFSZeroWithAVL", "user1",
+ amResource);
+ createSchedulingRequestExistingApplication(1 * GB, 1, amPriority, attId2);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // queueFSZeroWithAVL's weight is 0.0, so its fair share is 0, and we use
+ // the min(maxShare, available resource) to compute maxAMShare, in this
+ // case, we use available resource since it is smaller than the
+ // default maxShare.
+ assertEquals("QueueFSZeroWithAVL's fair share should be zero",
+ 0, queueFSZeroWithAVL.getFairShare().getMemorySize());
+ assertEquals("QueueFSZeroWithAVL's maximum AM resource should be "
+ + " available resource * maxAMShare",
+ (long) ((memCapacity - amResource.getMemorySize()) *
+ queueFSZeroWithAVL.getMaxAMShare()),
+ queueFSZeroWithAVL.getMetrics().getMaxAMShareMB());
+ assertEquals("QueueFSZeroWithAVL's maximum AM resource should be "
+ + " available resource * maxAMShare",
+ (long) ((cpuCapacity - amResource.getVirtualCores()) *
+ queueFSZeroWithAVL.getMaxAMShare()),
+ queueFSZeroWithAVL.getMetrics().getMaxAMShareVCores());
+ assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to "
+ + "AM resource request",
+ amResource.getMemorySize(),
+ queueFSZeroWithAVL.getMetrics().getAMResourceUsageMB());
+
+ // queueFSNonZero
+ amResource = Resources.createResource(1 * GB, 1);
+ FSLeafQueue queueFSNonZero = scheduler.getQueueManager().
+ getLeafQueue("queueFSNonZero", true);
+ ApplicationAttemptId attId3 = createAppAttemptId(3, 1);
+ createApplicationWithAMResource(attId3, "queueFSNonZero", "user1",
+ amResource);
+ createSchedulingRequestExistingApplication(1 * GB, 1, amPriority, attId3);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // queueFSNonZero's weight is 1, so its fair share is not 0, and we use the
+ // fair share to compute maxAMShare
+ assertNotEquals("QueueFSNonZero's fair share shouldn't be zero",
+ 0, queueFSNonZero.getFairShare().getMemorySize());
+ assertEquals("QueueFSNonZero's maximum AM resource should be "
+ + " fair share * maxAMShare",
+ (long)(memCapacity * queueFSNonZero.getMaxAMShare()),
+ queueFSNonZero.getMetrics().getMaxAMShareMB());
+ assertEquals("QueueFSNonZero's maximum AM resource should be "
+ + " fair share * maxAMShare",
+ (long)(cpuCapacity * queueFSNonZero.getMaxAMShare()),
+ queueFSNonZero.getMetrics().getMaxAMShareVCores());
+ assertEquals("QueueFSNonZero's AM resource usage should be the same to "
+ + "AM resource request",
+ amResource.getMemorySize(),
+ queueFSNonZero.getMetrics().getAMResourceUsageMB());
+ }
+
@Test
public void testFairShareWithZeroWeightNoneZeroMinRes() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/48] hadoop git commit: HADOOP-13812. Upgrade Tomcat to 6.0.48.
Contributed by John Zhuge.
Posted by li...@apache.org.
HADOOP-13812. Upgrade Tomcat to 6.0.48. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00096dcc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00096dcc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00096dcc
Branch: refs/heads/HADOOP-13345
Commit: 00096dcc0c23af38e697cd7c6c29e3f00d744e54
Parents: 0945125
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 29 19:47:02 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 29 19:47:02 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm | 2 ++
.../hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm | 2 ++
hadoop-project/pom.xml | 2 +-
3 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00096dcc/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
index 39aab82..8fd0f58 100644
--- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
@@ -298,6 +298,8 @@ The answer to "What is your first and last name?" (i.e. "CN") must be the hostna
NOTE: You need to restart the KMS for the configuration changes to take effect.
+NOTE: Some old SSL clients may use weak ciphers that are not supported by the KMS server. It is recommended to upgrade the SSL client.
+
$H4 ACLs (Access Control Lists)
KMS supports ACLs (Access Control Lists) for fine-grained permission control.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00096dcc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
index a4f02e7..0cb89de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
@@ -119,3 +119,5 @@ The answer to "What is your first and last name?" (i.e. "CN") must be the hostna
Start HttpFS. It should work over HTTPS.
Using the Hadoop `FileSystem` API or the Hadoop FS shell, use the `swebhdfs://` scheme. Make sure the JVM is picking up the truststore containing the public key of the SSL certificate if using a self-signed certificate.
+
+NOTE: Some old SSL clients may use weak ciphers that are not supported by the HttpFS server. It is recommended to upgrade the SSL client.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00096dcc/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index fa43afd..1ae60ed 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -85,7 +85,7 @@
<curator.version>2.7.1</curator.version>
<findbugs.version>3.0.0</findbugs.version>
- <tomcat.version>6.0.44</tomcat.version>
+ <tomcat.version>6.0.48</tomcat.version>
<guice.version>4.0</guice.version>
<joda-time.version>2.9.4</joda-time.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/48] hadoop git commit: HADOOP-13605. Clean up FileSystem
javadocs, logging;
improve diagnostics on FS load. Contributed by Steve Loughran
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/860d49aa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 9e98455..f581f61 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -75,6 +75,8 @@ import org.apache.htrace.core.TraceScope;
import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
@@ -88,79 +90,118 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
* <p>
*
* All user code that may potentially use the Hadoop Distributed
- * File System should be written to use a FileSystem object. The
- * Hadoop DFS is a multi-machine system that appears as a single
- * disk. It's useful because of its fault tolerance and potentially
- * very large capacity.
- *
+ * File System should be written to use a FileSystem object or its
+ * successor, {@link FileContext}.
+ *
* <p>
* The local implementation is {@link LocalFileSystem} and distributed
- * implementation is DistributedFileSystem.
+ * implementation is DistributedFileSystem. There are other implementations
+ * for object stores and (outside the Apache Hadoop codebase),
+ * third party filesystems.
+ * <p>
+ * Notes
+ * <ol>
+ * <li>The behaviour of the filesystem is
+ * <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/filesystem/filesystem.html">
+ * specified in the Hadoop documentation. </a>
+ * However, the normative specification of the behavior of this class is
+ * actually HDFS: if HDFS does not behave the way these Javadocs or
+ * the specification in the Hadoop documentations define, assume that
+ * the documentation is incorrect.
+ * </li>
+ * <li>The term {@code FileSystem} refers to an instance of this class.</li>
+ * <li>The acronym "FS" is used as an abbreviation of FileSystem.</li>
+ * <li>The term {@code filesystem} refers to the distributed/local filesystem
+ * itself, rather than the class used to interact with it.</li>
+ * <li>The term "file" refers to a file in the remote filesystem,
+ * rather than instances of {@code java.io.File}.</li>
+ * </ol>
*****************************************************************/
+@SuppressWarnings("DeprecatedIsStillUsed")
@InterfaceAudience.Public
@InterfaceStability.Stable
public abstract class FileSystem extends Configured implements Closeable {
- public static final String FS_DEFAULT_NAME_KEY =
+ public static final String FS_DEFAULT_NAME_KEY =
CommonConfigurationKeys.FS_DEFAULT_NAME_KEY;
- public static final String DEFAULT_FS =
+ public static final String DEFAULT_FS =
CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT;
+ /**
+ * This log is widely used in the org.apache.hadoop.fs code and tests,
+ * so must be considered something to only be changed with care.
+ */
+ @InterfaceAudience.Private
public static final Log LOG = LogFactory.getLog(FileSystem.class);
/**
- * Priority of the FileSystem shutdown hook.
+ * The SLF4J logger to use in logging within the FileSystem class itself.
+ */
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(FileSystem.class);
+
+ /**
+ * Priority of the FileSystem shutdown hook: {@value}.
*/
public static final int SHUTDOWN_HOOK_PRIORITY = 10;
+ /**
+ * Prefix for trash directory: {@value}.
+ */
public static final String TRASH_PREFIX = ".Trash";
public static final String USER_HOME_PREFIX = "/user";
- /** FileSystem cache */
+ /** FileSystem cache. */
static final Cache CACHE = new Cache();
/** The key this instance is stored under in the cache. */
private Cache.Key key;
- /** Recording statistics per a FileSystem class */
- private static final Map<Class<? extends FileSystem>, Statistics>
- statisticsTable =
- new IdentityHashMap<Class<? extends FileSystem>, Statistics>();
-
+ /** Recording statistics per a FileSystem class. */
+ private static final Map<Class<? extends FileSystem>, Statistics>
+ statisticsTable = new IdentityHashMap<>();
+
/**
* The statistics for this file system.
*/
protected Statistics statistics;
/**
- * A cache of files that should be deleted when filesystem is closed
+ * A cache of files that should be deleted when the FileSystem is closed
* or the JVM is exited.
*/
- private Set<Path> deleteOnExit = new TreeSet<Path>();
-
+ private final Set<Path> deleteOnExit = new TreeSet<>();
+
+ /**
+ * Should symbolic links be resolved by {@link FileSystemLinkResolver}.
+ * Set to the value of
+ * {@link CommonConfigurationKeysPublic#FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY}
+ */
boolean resolveSymlinks;
/**
- * This method adds a file system for testing so that we can find it later. It
- * is only for testing.
+ * This method adds a FileSystem instance to the cache so that it can
+ * be retrieved later. It is only for testing.
* @param uri the uri to store it under
* @param conf the configuration to store it under
- * @param fs the file system to store
- * @throws IOException
+ * @param fs the FileSystem to store
+ * @throws IOException if the current user cannot be determined.
*/
+ @VisibleForTesting
static void addFileSystemForTesting(URI uri, Configuration conf,
FileSystem fs) throws IOException {
CACHE.map.put(new Cache.Key(uri, conf), fs);
}
/**
- * Get a filesystem instance based on the uri, the passed
- * configuration and the user
+ * Get a FileSystem instance based on the uri, the passed in
+ * configuration and the user.
* @param uri of the filesystem
* @param conf the configuration to use
* @param user to perform the get as
* @return the filesystem instance
- * @throws IOException
- * @throws InterruptedException
+ * @throws IOException failure to load
+ * @throws InterruptedException If the {@code UGI.doAs()} call was
+ * somehow interrupted.
*/
public static FileSystem get(final URI uri, final Configuration conf,
final String user) throws IOException, InterruptedException {
@@ -177,14 +218,15 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Returns the configured filesystem implementation.
+ * Returns the configured FileSystem implementation.
* @param conf the configuration to use
*/
public static FileSystem get(Configuration conf) throws IOException {
return get(getDefaultUri(conf), conf);
}
-
- /** Get the default filesystem URI from a configuration.
+
+ /**
+ * Get the default FileSystem URI from a configuration.
* @param conf the configuration to use
* @return the uri of the default filesystem
*/
@@ -196,7 +238,8 @@ public abstract class FileSystem extends Configured implements Closeable {
return uri;
}
- /** Set the default filesystem URI in a configuration.
+ /**
+ * Set the default FileSystem URI in a configuration.
* @param conf the configuration to alter
* @param uri the new default filesystem uri
*/
@@ -204,7 +247,7 @@ public abstract class FileSystem extends Configured implements Closeable {
conf.set(FS_DEFAULT_NAME_KEY, uri.toString());
}
- /** Set the default filesystem URI in a configuration.
+ /** Set the default FileSystem URI in a configuration.
* @param conf the configuration to alter
* @param uri the new default filesystem uri
*/
@@ -212,10 +255,21 @@ public abstract class FileSystem extends Configured implements Closeable {
setDefaultUri(conf, URI.create(fixName(uri)));
}
- /** Called after a new FileSystem instance is constructed.
- * @param name a uri whose authority section names the host, port, etc.
+ /**
+ * Initialize a FileSystem.
+ *
+ * Called after the new FileSystem instance is constructed, and before it
+ * is ready for use.
+ *
+ * FileSystem implementations overriding this method MUST forward it to
+ * their superclass, though the order in which it is done, and whether
+ * to alter the configuration before the invocation are options of the
+ * subclass.
+ * @param name a URI whose authority section names the host, port, etc.
* for this FileSystem
* @param conf the configuration
+ * @throws IOException on any failure to initialize this instance.
+ * @throws IllegalArgumentException if the URI is considered invalid.
*/
public void initialize(URI name, Configuration conf) throws IOException {
final String scheme;
@@ -226,27 +280,34 @@ public abstract class FileSystem extends Configured implements Closeable {
}
statistics = getStatistics(scheme, getClass());
resolveSymlinks = conf.getBoolean(
- CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
- CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
+ CommonConfigurationKeysPublic.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
+ CommonConfigurationKeysPublic.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
}
/**
- * Return the protocol scheme for the FileSystem.
- * <p/>
+ * Return the protocol scheme for this FileSystem.
+ * <p>
* This implementation throws an <code>UnsupportedOperationException</code>.
*
- * @return the protocol scheme for the FileSystem.
+ * @return the protocol scheme for this FileSystem.
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public String getScheme() {
- throw new UnsupportedOperationException("Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation");
+ throw new UnsupportedOperationException("Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
}
- /** Returns a URI whose scheme and authority identify this FileSystem.*/
+ /**
+ * Returns a URI which identifies this FileSystem.
+ *
+ * @return the URI of this filesystem.
+ */
public abstract URI getUri();
-
+
/**
* Return a canonicalized form of this FileSystem's URI.
- *
+ *
* The default implementation simply calls {@link #canonicalizeUri(URI)}
* on the filesystem's own URI, so subclasses typically only need to
* implement that method.
@@ -256,16 +317,17 @@ public abstract class FileSystem extends Configured implements Closeable {
protected URI getCanonicalUri() {
return canonicalizeUri(getUri());
}
-
+
/**
* Canonicalize the given URI.
- *
- * This is filesystem-dependent, but may for example consist of
+ *
+ * This is implementation-dependent, and may for example consist of
* canonicalizing the hostname using DNS and adding the default
* port if not specified.
- *
+ *
* The default implementation simply fills in the default port if
- * not specified and if the filesystem has a default port.
+ * not specified and if {@link #getDefaultPort()} returns a
+ * default port.
*
* @return URI
* @see NetUtils#getCanonicalUri(URI, int)
@@ -283,12 +345,12 @@ public abstract class FileSystem extends Configured implements Closeable {
uri);
}
}
-
+
return uri;
}
-
+
/**
- * Get the default port for this file system.
+ * Get the default port for this FileSystem.
* @return the default port or 0 if there isn't one
*/
protected int getDefaultPort() {
@@ -301,25 +363,25 @@ public abstract class FileSystem extends Configured implements Closeable {
absOrFqPath.checkNotSchemeWithRelative();
absOrFqPath.checkNotRelative();
- // Uses the default file system if not fully qualified
+ // Uses the default FileSystem if not fully qualified
return get(absOrFqPath.toUri(), conf);
}
/**
- * Get a canonical service name for this file system. The token cache is
- * the only user of the canonical service name, and uses it to lookup this
- * filesystem's service tokens.
- * If file system provides a token of its own then it must have a canonical
- * name, otherwise canonical name can be null.
- *
- * Default Impl: If the file system has child file systems
- * (such as an embedded file system) then it is assumed that the fs has no
+ * Get a canonical service name for this FileSystem.
+ * The token cache is the only user of the canonical service name,
+ * and uses it to lookup this FileSystem's service tokens.
+ * If the file system provides a token of its own then it must have a
+ * canonical name, otherwise the canonical name can be null.
+ *
+ * Default implementation: If the FileSystem has child file systems
+ * (such as an embedded file system) then it is assumed that the FS has no
* tokens of its own and hence returns a null name; otherwise a service
* name is built using Uri and port.
- *
+ *
* @return a service string that uniquely identifies this file system, null
* if the filesystem does not implement tokens
- * @see SecurityUtil#buildDTServiceName(URI, int)
+ * @see SecurityUtil#buildDTServiceName(URI, int)
*/
@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
public String getCanonicalServiceName() {
@@ -328,28 +390,29 @@ public abstract class FileSystem extends Configured implements Closeable {
: null;
}
- /** @deprecated call #getUri() instead.*/
+ /** @deprecated call {@link #getUri()} instead.*/
@Deprecated
public String getName() { return getUri().toString(); }
- /** @deprecated call #get(URI,Configuration) instead. */
+ /** @deprecated call {@link #get(URI, Configuration)} instead. */
@Deprecated
public static FileSystem getNamed(String name, Configuration conf)
throws IOException {
return get(URI.create(fixName(name)), conf);
}
-
+
/** Update old-format filesystem names, for back-compatibility. This should
* eventually be replaced with a checkName() method that throws an exception
- * for old-format names. */
+ * for old-format names.
+ */
private static String fixName(String name) {
// convert old-format name to new-format name
if (name.equals("local")) { // "local" is now "file:///".
- LOG.warn("\"local\" is a deprecated filesystem name."
+ LOGGER.warn("\"local\" is a deprecated filesystem name."
+" Use \"file:///\" instead.");
name = "file:///";
} else if (name.indexOf('/')==-1) { // unqualified is "hdfs://"
- LOG.warn("\""+name+"\" is a deprecated filesystem name."
+ LOGGER.warn("\""+name+"\" is a deprecated filesystem name."
+" Use \"hdfs://"+name+"/\" instead.");
name = "hdfs://"+name;
}
@@ -357,19 +420,36 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Get the local file system.
- * @param conf the configuration to configure the file system with
+ * Get the local FileSystem.
+ * @param conf the configuration to configure the FileSystem with
+ * if it is newly instantiated.
* @return a LocalFileSystem
+ * @throws IOException if somehow the local FS cannot be instantiated.
*/
public static LocalFileSystem getLocal(Configuration conf)
throws IOException {
return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
}
- /** Returns the FileSystem for this URI's scheme and authority. The scheme
- * of the URI determines a configuration property name,
- * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
- * The entire URI is passed to the FileSystem instance's initialize method.
+ /**
+ * Get a FileSystem for this URI's scheme and authority.
+ * <ol>
+ * <li>
+ * If the configuration has the property
+ * {@code "fs.$SCHEME.impl.disable.cache"} set to true,
+ * a new instance will be created, initialized with the supplied URI and
+ * configuration, then returned without being cached.
+ * </li>
+ * <li>
+ * If the there is a cached FS instance matching the same URI, it will
+ * be returned.
+ * </li>
+ * <li>
+ * Otherwise: a new FS instance will be created, initialized with the
+ * configuration and URI, cached and returned to the caller.
+ * </li>
+ * </ol>
+ * @throws IOException if the FileSystem cannot be instantiated.
*/
public static FileSystem get(URI uri, Configuration conf) throws IOException {
String scheme = uri.getScheme();
@@ -386,9 +466,9 @@ public abstract class FileSystem extends Configured implements Closeable {
return get(defaultUri, conf); // return default
}
}
-
String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
if (conf.getBoolean(disableCacheName, false)) {
+ LOGGER.debug("Bypassing cache to create filesystem {}", uri);
return createFileSystem(uri, conf);
}
@@ -396,14 +476,15 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Returns the FileSystem for this URI's scheme and authority and the
- * passed user. Internally invokes {@link #newInstance(URI, Configuration)}
+ * Returns the FileSystem for this URI's scheme and authority and the
+ * given user. Internally invokes {@link #newInstance(URI, Configuration)}
* @param uri of the filesystem
* @param conf the configuration to use
* @param user to perform the get as
* @return filesystem instance
- * @throws IOException
- * @throws InterruptedException
+ * @throws IOException if the FileSystem cannot be instantiated.
+ * @throws InterruptedException If the {@code UGI.doAs()} call was
+ * somehow interrupted.
*/
public static FileSystem newInstance(final URI uri, final Configuration conf,
final String user) throws IOException, InterruptedException {
@@ -414,47 +495,56 @@ public abstract class FileSystem extends Configured implements Closeable {
return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
@Override
public FileSystem run() throws IOException {
- return newInstance(uri,conf);
+ return newInstance(uri, conf);
}
});
}
- /** Returns the FileSystem for this URI's scheme and authority. The scheme
- * of the URI determines a configuration property name,
- * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
+
+ /**
+ * Returns the FileSystem for this URI's scheme and authority.
* The entire URI is passed to the FileSystem instance's initialize method.
* This always returns a new FileSystem object.
+ * @param uri FS URI
+ * @param config configuration to use
+ * @return the new FS instance
+ * @throws IOException FS creation or initialization failure.
*/
- public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
+ public static FileSystem newInstance(URI uri, Configuration config)
+ throws IOException {
String scheme = uri.getScheme();
String authority = uri.getAuthority();
if (scheme == null) { // no scheme: use default FS
- return newInstance(conf);
+ return newInstance(config);
}
if (authority == null) { // no authority
- URI defaultUri = getDefaultUri(conf);
+ URI defaultUri = getDefaultUri(config);
if (scheme.equals(defaultUri.getScheme()) // if scheme matches default
&& defaultUri.getAuthority() != null) { // & default has authority
- return newInstance(defaultUri, conf); // return default
+ return newInstance(defaultUri, config); // return default
}
}
- return CACHE.getUnique(uri, conf);
+ return CACHE.getUnique(uri, config);
}
- /** Returns a unique configured filesystem implementation.
+ /**
+ * Returns a unique configured FileSystem implementation for the default
+ * filesystem of the supplied configuration.
* This always returns a new FileSystem object.
* @param conf the configuration to use
+ * @return the new FS instance
+ * @throws IOException FS creation or initialization failure.
*/
public static FileSystem newInstance(Configuration conf) throws IOException {
return newInstance(getDefaultUri(conf), conf);
}
/**
- * Get a unique local file system object
- * @param conf the configuration to configure the file system with
- * @return a LocalFileSystem
- * This always returns a new FileSystem object.
+ * Get a unique local FileSystem object.
+ * @param conf the configuration to configure the FileSystem with
+ * @return a new LocalFileSystem object.
+ * @throws IOException FS creation or initialization failure.
*/
public static LocalFileSystem newInstanceLocal(Configuration conf)
throws IOException {
@@ -462,65 +552,70 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Close all cached filesystems. Be sure those filesystems are not
- * used anymore.
- *
- * @throws IOException
+ * Close all cached FileSystem instances. After this operation, they
+ * may not be used in any operations.
+ *
+ * @throws IOException a problem arose closing one or more filesystem.
*/
public static void closeAll() throws IOException {
CACHE.closeAll();
}
/**
- * Close all cached filesystems for a given UGI. Be sure those filesystems
- * are not used anymore.
+ * Close all cached FileSystem instances for a given UGI.
+ * Be sure those filesystems are not used anymore.
* @param ugi user group info to close
- * @throws IOException
+ * @throws IOException a problem arose closing one or more filesystem.
*/
- public static void closeAllForUGI(UserGroupInformation ugi)
+ public static void closeAllForUGI(UserGroupInformation ugi)
throws IOException {
CACHE.closeAll(ugi);
}
- /**
- * Make sure that a path specifies a FileSystem.
- * @param path to use
+ /**
+ * Qualify a path to one which uses this FileSystem and, if relative,
+ * made absolute.
+ * @param path to qualify.
+ * @return this path if it contains a scheme and authority and is absolute, or
+ * a new path that includes a path and authority and is fully qualified
+ * @see Path#makeQualified(URI, Path)
+ * @throws IllegalArgumentException if the path has a schema/URI different
+ * from this FileSystem.
*/
public Path makeQualified(Path path) {
checkPath(path);
return path.makeQualified(this.getUri(), this.getWorkingDirectory());
}
-
+
/**
- * Get a new delegation token for this file system.
+ * Get a new delegation token for this FileSystem.
* This is an internal method that should have been declared protected
* but wasn't historically.
* Callers should use {@link #addDelegationTokens(String, Credentials)}
- *
+ *
* @param renewer the account name that is allowed to renew the token.
- * @return a new delegation token
- * @throws IOException
+ * @return a new delegation token or null if the FS does not support tokens.
+ * @throws IOException on any problem obtaining a token
*/
@InterfaceAudience.Private()
public Token<?> getDelegationToken(String renewer) throws IOException {
return null;
}
-
+
/**
* Obtain all delegation tokens used by this FileSystem that are not
- * already present in the given Credentials. Existing tokens will neither
+ * already present in the given Credentials. Existing tokens will neither
* be verified as valid nor having the given renewer. Missing tokens will
* be acquired and added to the given Credentials.
- *
- * Default Impl: works for simple fs with its own token
- * and also for an embedded fs whose tokens are those of its
- * children file system (i.e. the embedded fs has not tokens of its
- * own).
- *
+ *
+ * Default Impl: works for simple FS with its own token
+ * and also for an embedded FS whose tokens are those of its
+ * child FileSystems (i.e. the embedded FS has no tokens of its own).
+ *
* @param renewer the user allowed to renew the delegation tokens
* @param credentials cache in which to add new delegation tokens
* @return list of new delegation tokens
- * @throws IOException
+ * @throws IOException problems obtaining a token
*/
@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
public Token<?>[] addDelegationTokens(
@@ -528,18 +623,18 @@ public abstract class FileSystem extends Configured implements Closeable {
if (credentials == null) {
credentials = new Credentials();
}
- final List<Token<?>> tokens = new ArrayList<Token<?>>();
+ final List<Token<?>> tokens = new ArrayList<>();
collectDelegationTokens(renewer, credentials, tokens);
return tokens.toArray(new Token<?>[tokens.size()]);
}
-
+
/**
- * Recursively obtain the tokens for this FileSystem and all descended
- * FileSystems as determined by getChildFileSystems().
+ * Recursively obtain the tokens for this FileSystem and all descendant
+ * FileSystems as determined by {@link #getChildFileSystems()}.
* @param renewer the user allowed to renew the delegation tokens
* @param credentials cache in which to add the new delegation tokens
* @param tokens list in which to add acquired tokens
- * @throws IOException
+ * @throws IOException problems obtaining a token
*/
private void collectDelegationTokens(final String renewer,
final Credentials credentials,
@@ -570,30 +665,34 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Get all the immediate child FileSystems embedded in this FileSystem.
* It does not recurse and get grand children. If a FileSystem
- * has multiple child FileSystems, then it should return a unique list
+ * has multiple child FileSystems, then it must return a unique list
* of those FileSystems. Default is to return null to signify no children.
- *
- * @return FileSystems used by this FileSystem
+ *
+ * @return FileSystems that are direct children of this FileSystem,
+ * or null for "no children"
*/
@InterfaceAudience.LimitedPrivate({ "HDFS" })
@VisibleForTesting
public FileSystem[] getChildFileSystems() {
return null;
}
-
- /** create a file with the provided permission
+
+ /**
+ * Create a file with the provided permission.
+ *
* The permission of the file is set to be the provided permission as in
* setPermission, not permission&~umask
- *
- * It is implemented using two RPCs. It is understood that it is inefficient,
+ *
+ * The HDFS implementation is implemented using two RPCs.
+ * It is understood that it is inefficient,
* but the implementation is thread-safe. The other option is to change the
* value of umask in configuration to be 0, but it is not thread-safe.
- *
- * @param fs file system handle
+ *
+ * @param fs FileSystem
* @param file the name of the file to be created
* @param permission the permission of the file
* @return an output stream
- * @throws IOException
+ * @throws IOException IO failure
*/
public static FSDataOutputStream create(FileSystem fs,
Path file, FsPermission permission) throws IOException {
@@ -604,20 +703,21 @@ public abstract class FileSystem extends Configured implements Closeable {
return out;
}
- /** create a directory with the provided permission
+ /**
+ * Create a directory with the provided permission.
* The permission of the directory is set to be the provided permission as in
* setPermission, not permission&~umask
- *
+ *
* @see #create(FileSystem, Path, FsPermission)
- *
- * @param fs file system handle
+ *
+ * @param fs FileSystem handle
* @param dir the name of the directory to be created
* @param permission the permission of the directory
* @return true if the directory creation succeeds; false otherwise
- * @throws IOException
+ * @throws IOException A problem creating the directories.
*/
public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission)
- throws IOException {
+ throws IOException {
// create the directory using the default permission
boolean result = fs.mkdirs(dir);
// set its permission to be the supplied one
@@ -633,9 +733,16 @@ public abstract class FileSystem extends Configured implements Closeable {
super(null);
}
- /**
+ /**
* Check that a Path belongs to this FileSystem.
+ *
+ * The base implementation performs case insensitive equality checks
+ * of the URIs' schemes and authorities. Subclasses may implement slightly
+ * different checks.
* @param path to check
+ * @throws IllegalArgumentException if the path is not considered to be
+ * part of this FileSystem.
+ *
*/
protected void checkPath(Path path) {
URI uri = path.toUri();
@@ -667,25 +774,37 @@ public abstract class FileSystem extends Configured implements Closeable {
return;
}
}
- throw new IllegalArgumentException("Wrong FS: "+path+
- ", expected: "+this.getUri());
+ throw new IllegalArgumentException("Wrong FS: " + path +
+ ", expected: " + this.getUri());
}
/**
- * Return an array containing hostnames, offset and size of
- * portions of the given file. For a nonexistent
- * file or regions, null will be returned.
+ * Return an array containing hostnames, offset and size of
+ * portions of the given file. For nonexistent
+ * file or regions, {@code null} is returned.
*
- * This call is most helpful with DFS, where it returns
- * hostnames of machines that contain the given file.
+ * <pre>
+ * if f == null :
+ * result = null
+ * elif f.getLen() <= start:
+ * result = []
+ * else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)]
+ * </pre>
+ * This call is most helpful with and distributed filesystem
+ * where the hostnames of machines that contain blocks of the given file
+ * can be determined.
*
- * The FileSystem will simply return an elt containing 'localhost'.
+ * The default implementation returns an array containing one element:
+ * <pre>
+ * BlockLocation( { "localhost:50010" }, { "localhost" }, 0, file.getLen())
+ * </pre>>
*
* @param file FilesStatus to get data from
* @param start offset into the given file
* @param len length for which to get locations for
+ * @throws IOException IO failure
*/
- public BlockLocation[] getFileBlockLocations(FileStatus file,
+ public BlockLocation[] getFileBlockLocations(FileStatus file,
long start, long len) throws IOException {
if (file == null) {
return null;
@@ -704,24 +823,28 @@ public abstract class FileSystem extends Configured implements Closeable {
return new BlockLocation[] {
new BlockLocation(name, host, 0, file.getLen()) };
}
-
/**
- * Return an array containing hostnames, offset and size of
- * portions of the given file. For a nonexistent
- * file or regions, null will be returned.
+ * Return an array containing hostnames, offset and size of
+ * portions of the given file. For a nonexistent
+ * file or regions, {@code null} is returned.
*
- * This call is most helpful with DFS, where it returns
- * hostnames of machines that contain the given file.
+ * This call is most helpful with location-aware distributed
+ * filesystems, where it returns hostnames of machines that
+ * contain the given file.
*
- * The FileSystem will simply return an elt containing 'localhost'.
+ * A FileSystem will normally return the equivalent result
+ * of passing the {@code FileStatus} of the path to
+ * {@link #getFileBlockLocations(FileStatus, long, long)}
*
* @param p path is used to identify an FS since an FS could have
* another FS that it could be delegating the call to
* @param start offset into the given file
* @param len length for which to get locations for
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException IO failure
*/
- public BlockLocation[] getFileBlockLocations(Path p,
+ public BlockLocation[] getFileBlockLocations(Path p,
long start, long len) throws IOException {
if (p == null) {
throw new NullPointerException();
@@ -729,46 +852,47 @@ public abstract class FileSystem extends Configured implements Closeable {
FileStatus file = getFileStatus(p);
return getFileBlockLocations(file, start, len);
}
-
+
/**
- * Return a set of server default configuration values
+ * Return a set of server default configuration values.
* @return server default configuration values
- * @throws IOException
+ * @throws IOException IO failure
* @deprecated use {@link #getServerDefaults(Path)} instead
*/
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
- Configuration conf = getConf();
- // CRC32 is chosen as default as it is available in all
+ Configuration config = getConf();
+ // CRC32 is chosen as default as it is available in all
// releases that support checksum.
// The client trash configuration is ignored.
- return new FsServerDefaults(getDefaultBlockSize(),
- conf.getInt("io.bytes.per.checksum", 512),
- 64 * 1024,
+ return new FsServerDefaults(getDefaultBlockSize(),
+ config.getInt("io.bytes.per.checksum", 512),
+ 64 * 1024,
getDefaultReplication(),
- conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT),
+ config.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT),
false,
FS_TRASH_INTERVAL_DEFAULT,
DataChecksum.Type.CRC32);
}
/**
- * Return a set of server default configuration values
+ * Return a set of server default configuration values.
* @param p path is used to identify an FS since an FS could have
* another FS that it could be delegating the call to
* @return server default configuration values
- * @throws IOException
+ * @throws IOException IO failure
*/
public FsServerDefaults getServerDefaults(Path p) throws IOException {
return getServerDefaults();
}
/**
- * Return the fully-qualified path of path f resolving the path
- * through any symlinks or mount point
+ * Return the fully-qualified path of path, resolving the path
+ * through any symlinks or mount point.
* @param p path to be resolved
- * @return fully qualified path
- * @throws FileNotFoundException
+ * @return fully qualified path
+ * @throws FileNotFoundException if the path is not present
+ * @throws IOException for any other error
*/
public Path resolvePath(final Path p) throws IOException {
checkPath(p);
@@ -779,13 +903,15 @@ public abstract class FileSystem extends Configured implements Closeable {
* Opens an FSDataInputStream at the indicated Path.
* @param f the file name to open
* @param bufferSize the size of the buffer to be used.
+ * @throws IOException IO failure
*/
public abstract FSDataInputStream open(Path f, int bufferSize)
throws IOException;
-
+
/**
* Opens an FSDataInputStream at the indicated Path.
* @param f the file to open
+ * @throws IOException IO failure
*/
public FSDataInputStream open(Path f) throws IOException {
return open(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
@@ -796,6 +922,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* Create an FSDataOutputStream at the indicated Path.
* Files are overwritten by default.
* @param f the file to create
+ * @throws IOException IO failure
*/
public FSDataOutputStream create(Path f) throws IOException {
return create(f, true);
@@ -806,10 +933,11 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param f the file to create
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an exception will be thrown.
+ * @throws IOException IO failure
*/
public FSDataOutputStream create(Path f, boolean overwrite)
throws IOException {
- return create(f, overwrite,
+ return create(f, overwrite,
getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT),
getDefaultReplication(f),
@@ -822,10 +950,11 @@ public abstract class FileSystem extends Configured implements Closeable {
* Files are overwritten by default.
* @param f the file to create
* @param progress to report progress
+ * @throws IOException IO failure
*/
- public FSDataOutputStream create(Path f, Progressable progress)
+ public FSDataOutputStream create(Path f, Progressable progress)
throws IOException {
- return create(f, true,
+ return create(f, true,
getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT),
getDefaultReplication(f),
@@ -837,10 +966,11 @@ public abstract class FileSystem extends Configured implements Closeable {
* Files are overwritten by default.
* @param f the file to create
* @param replication the replication factor
+ * @throws IOException IO failure
*/
public FSDataOutputStream create(Path f, short replication)
throws IOException {
- return create(f, true,
+ return create(f, true,
getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT),
replication,
@@ -854,65 +984,70 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param f the file to create
* @param replication the replication factor
* @param progress to report progress
+ * @throws IOException IO failure
*/
- public FSDataOutputStream create(Path f, short replication,
+ public FSDataOutputStream create(Path f, short replication,
Progressable progress) throws IOException {
- return create(f, true,
+ return create(f, true,
getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT),
replication, getDefaultBlockSize(f), progress);
}
-
+
/**
* Create an FSDataOutputStream at the indicated Path.
- * @param f the file name to create
- * @param overwrite if a file with this name already exists, then if true,
+ * @param f the file to create
+ * @param overwrite if a path with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
+ * @throws IOException IO failure
*/
- public FSDataOutputStream create(Path f,
+ public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize
) throws IOException {
- return create(f, overwrite, bufferSize,
+ return create(f, overwrite, bufferSize,
getDefaultReplication(f),
getDefaultBlockSize(f));
}
-
+
/**
- * Create an FSDataOutputStream at the indicated Path with write-progress
- * reporting.
+ * Create an {@link FSDataOutputStream} at the indicated Path
+ * with write-progress reporting.
+ *
+ * The frequency of callbacks is implementation-specific; it may be "none".
* @param f the path of the file to open
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
+ * @throws IOException IO failure
*/
- public FSDataOutputStream create(Path f,
+ public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize,
Progressable progress
) throws IOException {
- return create(f, overwrite, bufferSize,
+ return create(f, overwrite, bufferSize,
getDefaultReplication(f),
getDefaultBlockSize(f), progress);
}
-
-
+
+
/**
* Create an FSDataOutputStream at the indicated Path.
* @param f the file name to open
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
- * @param replication required block replication for the file.
+ * @param replication required block replication for the file.
+ * @throws IOException IO failure
*/
- public FSDataOutputStream create(Path f,
- boolean overwrite,
- int bufferSize,
- short replication,
- long blockSize
- ) throws IOException {
+ public FSDataOutputStream create(Path f,
+ boolean overwrite,
+ int bufferSize,
+ short replication,
+ long blockSize) throws IOException {
return create(f, overwrite, bufferSize, replication, blockSize, null);
}
@@ -923,7 +1058,8 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
- * @param replication required block replication for the file.
+ * @param replication required block replication for the file.
+ * @throws IOException IO failure
*/
public FSDataOutputStream create(Path f,
boolean overwrite,
@@ -948,7 +1084,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param replication required block replication for the file.
* @param blockSize block size
* @param progress the progress reporter
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public abstract FSDataOutputStream create(Path f,
@@ -958,7 +1094,7 @@ public abstract class FileSystem extends Configured implements Closeable {
short replication,
long blockSize,
Progressable progress) throws IOException;
-
+
/**
* Create an FSDataOutputStream at the indicated Path with write-progress
* reporting.
@@ -969,7 +1105,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param replication required block replication for the file.
* @param blockSize block size
* @param progress the progress reporter
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public FSDataOutputStream create(Path f,
@@ -982,10 +1118,10 @@ public abstract class FileSystem extends Configured implements Closeable {
return create(f, permission, flags, bufferSize, replication,
blockSize, progress, null);
}
-
+
/**
* Create an FSDataOutputStream at the indicated Path with a custom
- * checksum option
+ * checksum option.
* @param f the file name to open
* @param permission file permission
* @param flags {@link CreateFlag}s to use for this stream.
@@ -995,7 +1131,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param progress the progress reporter
* @param checksumOpt checksum parameter. If null, the values
* found in conf will be used.
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public FSDataOutputStream create(Path f,
@@ -1009,27 +1145,31 @@ public abstract class FileSystem extends Configured implements Closeable {
// Checksum options are ignored by default. The file systems that
// implement checksum need to override this method. The full
// support is currently only available in DFS.
- return create(f, permission, flags.contains(CreateFlag.OVERWRITE),
+ return create(f, permission, flags.contains(CreateFlag.OVERWRITE),
bufferSize, replication, blockSize, progress);
}
- /*.
+ /**
* This create has been added to support the FileContext that processes
- * the permission
- * with umask before calling this method.
+ * the permission with umask before calling this method.
* This a temporary method added to support the transition from FileSystem
* to FileContext for user applications.
+ * @throws IOException IO failure
*/
@Deprecated
protected FSDataOutputStream primitiveCreate(Path f,
- FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
- short replication, long blockSize, Progressable progress,
- ChecksumOpt checksumOpt) throws IOException {
+ FsPermission absolutePermission,
+ EnumSet<CreateFlag> flag,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress,
+ ChecksumOpt checksumOpt) throws IOException {
boolean pathExists = exists(f);
CreateFlag.validate(f, pathExists, flag);
-
- // Default impl assumes that permissions do not matter and
+
+ // Default impl assumes that permissions do not matter and
// nor does the bytesPerChecksum hence
// calling the regular create is good enough.
// FSs that implement permissions should override this.
@@ -1037,25 +1177,27 @@ public abstract class FileSystem extends Configured implements Closeable {
if (pathExists && flag.contains(CreateFlag.APPEND)) {
return append(f, bufferSize, progress);
}
-
+
return this.create(f, absolutePermission,
flag.contains(CreateFlag.OVERWRITE), bufferSize, replication,
blockSize, progress);
}
-
+
/**
* This version of the mkdirs method assumes that the permission is absolute.
* It has been added to support the FileContext that processes the permission
* with umask before calling this method.
* This a temporary method added to support the transition from FileSystem
* to FileContext for user applications.
+ * @param f path
+ * @param absolutePermission permissions
+ * @return true if the directory was actually created.
+ * @throws IOException IO failure
+ * @see #mkdirs(Path, FsPermission)
*/
@Deprecated
protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
throws IOException {
- // Default impl is to assume that permissions do not matter and hence
- // calling the regular mkdirs is good enough.
- // FSs that implement permissions should override this.
return this.mkdirs(f, absolutePermission);
}
@@ -1068,10 +1210,10 @@ public abstract class FileSystem extends Configured implements Closeable {
* to FileContext for user applications.
*/
@Deprecated
- protected void primitiveMkdir(Path f, FsPermission absolutePermission,
+ protected void primitiveMkdir(Path f, FsPermission absolutePermission,
boolean createParent)
throws IOException {
-
+
if (!createParent) { // parent must exist.
// since the this.mkdirs makes parent dirs automatically
// we must throw exception if parent does not exist.
@@ -1103,7 +1245,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param replication required block replication for the file.
* @param blockSize block size
* @param progress the progress reporter
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public FSDataOutputStream createNonRecursive(Path f,
@@ -1126,7 +1268,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param replication required block replication for the file.
* @param blockSize block size
* @param progress the progress reporter
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
@@ -1149,7 +1291,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param replication required block replication for the file.
* @param blockSize block size
* @param progress the progress reporter
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
@@ -1162,8 +1304,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Creates the given Path as a brand-new zero-length file. If
* create fails, or if it already existed, return false.
- *
+ * <i>Important: the default implementation is not atomic</i>
* @param f path to use for create
+ * @throws IOException IO failure
*/
public boolean createNewFile(Path f) throws IOException {
if (exists(f)) {
@@ -1177,21 +1320,27 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Append to an existing file (optional operation).
- * Same as append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
- * IO_FILE_BUFFER_SIZE_DEFAULT), null)
+ * Same as
+ * {@code append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
+ * IO_FILE_BUFFER_SIZE_DEFAULT), null)}
* @param f the existing file to be appended.
- * @throws IOException
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public FSDataOutputStream append(Path f) throws IOException {
return append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT), null);
}
+
/**
* Append to an existing file (optional operation).
* Same as append(f, bufferSize, null).
* @param f the existing file to be appended.
* @param bufferSize the size of the buffer to be used.
- * @throws IOException
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
return append(f, bufferSize, null);
@@ -1202,7 +1351,9 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param f the existing file to be appended.
* @param bufferSize the size of the buffer to be used.
* @param progress for reporting progress if it is not null.
- * @throws IOException
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public abstract FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException;
@@ -1211,34 +1362,40 @@ public abstract class FileSystem extends Configured implements Closeable {
* Concat existing files together.
* @param trg the path to the target destination.
* @param psrcs the paths to the sources to use for the concatenation.
- * @throws IOException
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public void concat(final Path trg, final Path [] psrcs) throws IOException {
- throw new UnsupportedOperationException("Not implemented by the " +
+ throw new UnsupportedOperationException("Not implemented by the " +
getClass().getSimpleName() + " FileSystem implementation");
}
/**
- * Get replication.
- *
- * @deprecated Use getFileStatus() instead
+ * Get the replication factor.
+ *
+ * @deprecated Use {@link #getFileStatus(Path)} instead
* @param src file name
* @return file replication
- * @throws IOException
- */
+ * @throws FileNotFoundException if the path does not resolve.
+ * @throws IOException an IO failure
+ */
@Deprecated
public short getReplication(Path src) throws IOException {
return getFileStatus(src).getReplication();
}
/**
- * Set replication for an existing file.
- *
+ * Set the replication for an existing file.
+ * If a filesystem does not support replication, it will always
+ * return true: the check for a file existing may be bypassed.
+ * This is the default behavior.
* @param src file name
* @param replication new replication
* @throws IOException
- * @return true if successful;
- * false if file does not exist or is a directory
+ * @return true if successful, or the feature in unsupported;
+ * false if replication is supported but the file does not exist,
+ * or is a directory
*/
public boolean setReplication(Path src, short replication)
throws IOException {
@@ -1246,8 +1403,7 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Renames Path src to Path dst. Can take place on local fs
- * or remote DFS.
+ * Renames Path src to Path dst.
* @param src path to be renamed
* @param dst new path after rename
* @throws IOException on failure
@@ -1258,9 +1414,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Renames Path src to Path dst
* <ul>
- * <li>Fails if src is a file and dst is a directory.
- * <li>Fails if src is a directory and dst is a file.
- * <li>Fails if the parent of dst does not exist or is a file.
+ * <li>Fails if src is a file and dst is a directory.</li>
+ * <li>Fails if src is a directory and dst is a file.</li>
+ * <li>Fails if the parent of dst does not exist or is a file.</li>
* </ul>
* <p>
* If OVERWRITE option is not passed as an argument, rename fails
@@ -1274,12 +1430,17 @@ public abstract class FileSystem extends Configured implements Closeable {
* implementation. Please refer to the file system documentation for
* details. This default implementation is non atomic.
* <p>
- * This method is deprecated since it is a temporary method added to
- * support the transition from FileSystem to FileContext for user
+ * This method is deprecated since it is a temporary method added to
+ * support the transition from FileSystem to FileContext for user
* applications.
- *
+ *
* @param src path to be renamed
* @param dst new path after rename
+ * @throws FileNotFoundException src path does not exist, or the parent
+ * path of dst does not exist.
+ * @throws FileAlreadyExistsException dest path exists and is a file
+ * @throws ParentNotDirectoryException if the parent path of dest is not
+ * a directory
* @throws IOException on failure
*/
@Deprecated
@@ -1344,10 +1505,10 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Truncate the file in the indicated path to the indicated size.
* <ul>
- * <li>Fails if path is a directory.
- * <li>Fails if path does not exist.
- * <li>Fails if path is not closed.
- * <li>Fails if new size is greater than current size.
+ * <li>Fails if path is a directory.</li>
+ * <li>Fails if path does not exist.</li>
+ * <li>Fails if path is not closed.</li>
+ * <li>Fails if new size is greater than current size.</li>
* </ul>
* @param f The path to the file to be truncated
* @param newLength The size the file is to be truncated to
@@ -1358,44 +1519,61 @@ public abstract class FileSystem extends Configured implements Closeable {
* <code>false</code> if a background process of adjusting the length of
* the last block has been started, and clients should wait for it to
* complete before proceeding with further file updates.
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public boolean truncate(Path f, long newLength) throws IOException {
throw new UnsupportedOperationException("Not implemented by the " +
getClass().getSimpleName() + " FileSystem implementation");
}
-
+
/**
- * Delete a file
+ * Delete a file/directory.
* @deprecated Use {@link #delete(Path, boolean)} instead.
*/
@Deprecated
public boolean delete(Path f) throws IOException {
return delete(f, true);
}
-
+
/** Delete a file.
*
* @param f the path to delete.
- * @param recursive if path is a directory and set to
+ * @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception. In
- * case of a file the recursive can be set to either true or false.
- * @return true if delete is successful else false.
- * @throws IOException
+ * case of a file the recursive can be set to either true or false.
+ * @return true if delete is successful else false.
+ * @throws IOException IO failure
*/
public abstract boolean delete(Path f, boolean recursive) throws IOException;
/**
- * Mark a path to be deleted when FileSystem is closed.
- * When the JVM shuts down,
- * all FileSystem objects will be closed automatically.
- * Then,
- * the marked path will be deleted as a result of closing the FileSystem.
+ * Mark a path to be deleted when its FileSystem is closed.
+ * When the JVM shuts down cleanly, all cached FileSystem objects will be
+ * closed automatically \u2014these the marked paths will be deleted as a result.
+ *
+ * If a FileSystem instance is not cached, i.e. has been created with
+ * {@link #createFileSystem(URI, Configuration)}, then the paths will
+ * be deleted in when {@link #close()} is called on that instance.
*
- * The path has to exist in the file system.
- *
+ * The path must exist in the filesystem at the time of the method call;
+ * it does not have to exist at the time of JVM shutdown.
+ *
+ * Notes
+ * <ol>
+ * <li>Clean shutdown of the JVM cannot be guaranteed.</li>
+ * <li>The time to shut down a FileSystem will depends on the number of
+ * files to delete. For filesystems where the cost of checking
+ * for the existence of a file/directory and the actual delete operation
+ * (for example: object stores) is high, the time to shutdown the JVM can be
+ * significantly extended by over-use of this feature.</li>
+ * <li>Connectivity problems with a remote filesystem may delay shutdown
+ * further, and may cause the files to not be deleted.</li>
+ * </ol>
* @param f the path to delete.
* @return true if deleteOnExit is successful, otherwise false.
- * @throws IOException
+ * @throws IOException IO failure
*/
public boolean deleteOnExit(Path f) throws IOException {
if (!exists(f)) {
@@ -1406,10 +1584,11 @@ public abstract class FileSystem extends Configured implements Closeable {
}
return true;
}
-
+
/**
- * Cancel the deletion of the path when the FileSystem is closed
+ * Cancel the scheduled deletion of the path when the FileSystem is closed.
* @param f the path to cancel deletion
+ * @return true if the path was found in the delete-on-exit list.
*/
public boolean cancelDeleteOnExit(Path f) {
synchronized (deleteOnExit) {
@@ -1418,8 +1597,12 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Delete all files that were marked as delete-on-exit. This recursively
- * deletes all files in the specified paths.
+ * Delete all paths that were marked as delete-on-exit. This recursively
+ * deletes all files and directories in the specified paths.
+ *
+ * The time to process this operation is {@code O(paths)}, with the actual
+ * time dependent on the time for existence and deletion operations to
+ * complete, successfully or not.
*/
protected void processDeleteOnExit() {
synchronized (deleteOnExit) {
@@ -1431,15 +1614,17 @@ public abstract class FileSystem extends Configured implements Closeable {
}
}
catch (IOException e) {
- LOG.info("Ignoring failure to deleteOnExit for path " + path);
+ LOGGER.info("Ignoring failure to deleteOnExit for path {}", path);
}
iter.remove();
}
}
}
-
- /** Check if exists.
- * @param f source file
+
+ /** Check if a path exists.
+ * @param f source path
+ * @return true if the path exists
+ * @throws IOException IO failure
*/
public boolean exists(Path f) throws IOException {
try {
@@ -1450,9 +1635,10 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/** True iff the named path is a directory.
- * Note: Avoid using this method. Instead reuse the FileStatus
+ * Note: Avoid using this method. Instead reuse the FileStatus
* returned by getFileStatus() or listStatus() methods.
* @param f path to check
+ * @throws IOException IO failure
*/
public boolean isDirectory(Path f) throws IOException {
try {
@@ -1463,9 +1649,10 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/** True iff the named path is a regular file.
- * Note: Avoid using this method. Instead reuse the FileStatus
- * returned by getFileStatus() or listStatus() methods.
+ * Note: Avoid using this method. Instead reuse the FileStatus
+ * returned by {@link #getFileStatus(Path)} or listStatus() methods.
* @param f path to check
+ * @throws IOException IO failure
*/
public boolean isFile(Path f) throws IOException {
try {
@@ -1474,17 +1661,24 @@ public abstract class FileSystem extends Configured implements Closeable {
return false; // f does not exist
}
}
-
- /** The number of bytes in a file. */
- /** @deprecated Use getFileStatus() instead */
+
+ /**
+ * The number of bytes in a file.
+ * @return the number of bytes; 0 for a directory
+ * @deprecated Use {@link #getFileStatus(Path)} instead.
+ * @throws FileNotFoundException if the path does not resolve
+ * @throws IOException IO failure
+ */
@Deprecated
public long getLength(Path f) throws IOException {
return getFileStatus(f).getLen();
}
-
+
/** Return the {@link ContentSummary} of a given {@link Path}.
- * @param f path to use
- */
+ * @param f path to use
+ * @throws FileNotFoundException if the path does not resolve
+ * @throws IOException IO failure
+ */
public ContentSummary getContentSummary(Path f) throws IOException {
FileStatus status = getFileStatus(f);
if (status.isFile()) {
@@ -1511,18 +1705,23 @@ public abstract class FileSystem extends Configured implements Closeable {
/** Return the {@link QuotaUsage} of a given {@link Path}.
* @param f path to use
+ * @return the quota usage
+ * @throws IOException IO failure
*/
public QuotaUsage getQuotaUsage(Path f) throws IOException {
return getContentSummary(f);
}
- final private static PathFilter DEFAULT_FILTER = new PathFilter() {
- @Override
- public boolean accept(Path file) {
- return true;
- }
- };
-
+ /**
+ * The default filter accepts all paths.
+ */
+ private static final PathFilter DEFAULT_FILTER = new PathFilter() {
+ @Override
+ public boolean accept(Path file) {
+ return true;
+ }
+ };
+
/**
* List the statuses of the files/directories in the given path if the path is
* a directory.
@@ -1536,7 +1735,8 @@ public abstract class FileSystem extends Configured implements Closeable {
* @throws FileNotFoundException when the path does not exist
* @throws IOException see specific implementation
*/
- public abstract FileStatus[] listStatus(Path f) throws IOException;
+ public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException,
+ IOException;
/**
* Represents a batch of directory entries when iteratively listing a
@@ -1599,9 +1799,11 @@ public abstract class FileSystem extends Configured implements Closeable {
return new DirectoryEntries(listing, null, false);
}
- /*
+ /**
* Filter files/directories in the given path using the user-supplied path
* filter. Results are added to the given array <code>results</code>.
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException see specific implementation
*/
private void listStatus(ArrayList<FileStatus> results, Path f,
PathFilter filter) throws FileNotFoundException, IOException {
@@ -1615,15 +1817,17 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
+ * List corrupted file blocks.
* @return an iterator over the corrupt files under the given path
* (may contain duplicates if a file has more than one corrupt block)
- * @throws IOException
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
+ * @throws IOException IO failure
*/
public RemoteIterator<Path> listCorruptFileBlocks(Path path)
throws IOException {
throw new UnsupportedOperationException(getClass().getCanonicalName() +
- " does not support" +
- " listCorruptFileBlocks");
+ " does not support listCorruptFileBlocks");
}
/**
@@ -1632,19 +1836,19 @@ public abstract class FileSystem extends Configured implements Closeable {
* <p>
* Does not guarantee to return the List of files/directories status in a
* sorted order.
- *
+ *
* @param f
* a path name
* @param filter
* the user-supplied path filter
* @return an array of FileStatus objects for the files under the given path
* after applying the filter
- * @throws FileNotFoundException when the path does not exist;
- * IOException see specific implementation
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException see specific implementation
*/
- public FileStatus[] listStatus(Path f, PathFilter filter)
+ public FileStatus[] listStatus(Path f, PathFilter filter)
throws FileNotFoundException, IOException {
- ArrayList<FileStatus> results = new ArrayList<FileStatus>();
+ ArrayList<FileStatus> results = new ArrayList<>();
listStatus(results, f, filter);
return results.toArray(new FileStatus[results.size()]);
}
@@ -1655,13 +1859,13 @@ public abstract class FileSystem extends Configured implements Closeable {
* <p>
* Does not guarantee to return the List of files/directories status in a
* sorted order.
- *
+ *
* @param files
* a list of paths
* @return a list of statuses for the files under the given paths after
* applying the filter default Path filter
- * @throws FileNotFoundException when the path does not exist;
- * IOException see specific implementation
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException see specific implementation
*/
public FileStatus[] listStatus(Path[] files)
throws FileNotFoundException, IOException {
@@ -1674,15 +1878,15 @@ public abstract class FileSystem extends Configured implements Closeable {
* <p>
* Does not guarantee to return the List of files/directories status in a
* sorted order.
- *
+ *
* @param files
* a list of paths
* @param filter
* the user-supplied path filter
* @return a list of statuses for the files under the given paths after
* applying the filter
- * @throws FileNotFoundException when the path does not exist;
- * IOException see specific implementation
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException see specific implementation
*/
public FileStatus[] listStatus(Path[] files, PathFilter filter)
throws FileNotFoundException, IOException {
@@ -1696,7 +1900,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* <p>Return all the files that match filePattern and are not checksum
* files. Results are sorted by their names.
- *
+ *
* <p>
* A filename pattern is composed of <i>regular</i> characters and
* <i>special pattern matching</i> characters, which are:
@@ -1736,7 +1940,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* <p>
* <dt> <tt> {ab,cd} </tt>
* <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
- *
+ *
* <p>
* <dt> <tt> {ab,c{de,fh}} </tt>
* <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
@@ -1748,17 +1952,17 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param pathPattern a regular expression specifying a pth pattern
* @return an array of paths that match the path pattern
- * @throws IOException
+ * @throws IOException IO failure
*/
public FileStatus[] globStatus(Path pathPattern) throws IOException {
return new Globber(this, pathPattern, DEFAULT_FILTER).glob();
}
-
+
/**
- * Return an array of FileStatus objects whose path names match
+ * Return an array of {@link FileStatus} objects whose path names match
* {@code pathPattern} and is accepted by the user-supplied path filter.
* Results are sorted by their path names.
- *
+ *
* @param pathPattern a regular expression specifying the path pattern
* @param filter a user-supplied path filter
* @return null if {@code pathPattern} has no glob and the path does not exist
@@ -1771,17 +1975,17 @@ public abstract class FileSystem extends Configured implements Closeable {
throws IOException {
return new Globber(this, pathPattern, filter).glob();
}
-
+
/**
* List the statuses of the files/directories in the given path if the path is
- * a directory.
+ * a directory.
* Return the file's status and block locations If the path is a file.
- *
+ *
* If a returned status is a file, it contains the file's block locations.
- *
+ *
* @param f is the path
*
- * @return an iterator that traverses statuses of the files/directories
+ * @return an iterator that traverses statuses of the files/directories
* in the given path
*
* @throws FileNotFoundException If <code>f</code> does not exist
@@ -1793,12 +1997,12 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Listing a directory
+ * List a directory.
* The returned results include its block location if it is a file
* The results are filtered by the given path filter
* @param f a path
* @param filter a path filter
- * @return an iterator that traverses statuses of the files/directories
+ * @return an iterator that traverses statuses of the files/directories
* in the given path
* @throws FileNotFoundException if <code>f</code> does not exist
* @throws IOException if any I/O error occurred
@@ -1876,14 +2080,17 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Returns a remote iterator so that followup calls are made on demand
- * while consuming the entries. Each file system implementation should
+ * while consuming the entries. Each FileSystem implementation should
* override this method and provide a more efficient implementation, if
- * possible.
+ * possible.
+ *
* Does not guarantee to return the iterator that traverses statuses
* of the files in a sorted order.
*
* @param p target path
* @return remote iterator
+ * @throws FileNotFoundException if <code>p</code> does not exist
+ * @throws IOException if any I/O error occurred
*/
public RemoteIterator<FileStatus> listStatusIterator(final Path p)
throws FileNotFoundException, IOException {
@@ -1894,30 +2101,29 @@ public abstract class FileSystem extends Configured implements Closeable {
* List the statuses and block locations of the files in the given path.
* Does not guarantee to return the iterator that traverses statuses
* of the files in a sorted order.
- *
- * If the path is a directory,
+ * <pre>
+ * If the path is a directory,
* if recursive is false, returns files in the directory;
* if recursive is true, return files in the subtree rooted at the path.
* If the path is a file, return the file's status and block locations.
- *
+ * </pre>
* @param f is the path
* @param recursive if the subdirectories need to be traversed recursively
*
* @return an iterator that traverses statuses of the files
*
* @throws FileNotFoundException when the path does not exist;
- * IOException see specific implementation
+ * @throws IOException see specific implementation
*/
public RemoteIterator<LocatedFileStatus> listFiles(
final Path f, final boolean recursive)
throws FileNotFoundException, IOException {
return new RemoteIterator<LocatedFileStatus>() {
- private Stack<RemoteIterator<LocatedFileStatus>> itors =
- new Stack<RemoteIterator<LocatedFileStatus>>();
+ private Stack<RemoteIterator<LocatedFileStatus>> itors = new Stack<>();
private RemoteIterator<LocatedFileStatus> curItor =
listLocatedStatus(f);
private LocatedFileStatus curFile;
-
+
@Override
public boolean hasNext() throws IOException {
while (curFile == null) {
@@ -1955,14 +2161,14 @@ public abstract class FileSystem extends Configured implements Closeable {
LocatedFileStatus result = curFile;
curFile = null;
return result;
- }
+ }
throw new java.util.NoSuchElementException("No more entry in " + f);
}
};
}
-
- /** Return the current user's home directory in this filesystem.
- * The default implementation returns "/user/$USER/".
+
+ /** Return the current user's home directory in this FileSystem.
+ * The default implementation returns {@code "/user/$USER/"}.
*/
public Path getHomeDirectory() {
return this.makeQualified(
@@ -1971,29 +2177,28 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
- * Set the current working directory for the given file system. All relative
+ * Set the current working directory for the given FileSystem. All relative
* paths will be resolved relative to it.
- *
+ *
* @param new_dir Path of new working directory
*/
public abstract void setWorkingDirectory(Path new_dir);
-
+
/**
- * Get the current working directory for the given file system
+ * Get the current working directory for the given FileSystem
* @return the directory pathname
*/
public abstract Path getWorkingDirectory();
-
-
- /**
- * Note: with the new FilesContext class, getWorkingDirectory()
- * will be removed.
- * The working directory is implemented in FilesContext.
- *
- * Some file systems like LocalFileSystem have an initial workingDir
+
+ /**
+ * Note: with the new FileContext class, getWorkingDirectory()
+ * will be removed.
+ * The working directory is implemented in FileContext.
+ *
+ * Some FileSystems like LocalFileSystem have an initial workingDir
* that we use as the starting workingDir. For other file systems
* like HDFS there is no built in notion of an initial workingDir.
- *
+ *
* @return if there is built in notion of workingDir then it
* is returned; else a null is returned.
*/
@@ -2003,6 +2208,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Call {@link #mkdirs(Path, FsPermission)} with default permission.
+ * @param f path
+ * @return true if the directory was created
+ * @throws IOException IO failure
*/
public boolean mkdirs(Path f) throws IOException {
return mkdirs(f, FsPermission.getDirDefault());
@@ -2010,19 +2218,21 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Make the given file and all non-existent parents into
- * directories. Has the semantics of Unix 'mkdir -p'.
+ * directories. Has roughly the semantics of Unix @{code mkdir -p}.
* Existence of the directory hierarchy is not an error.
* @param f path to create
* @param permission to apply to f
+ * @throws IOException IO failure
*/
public abstract boolean mkdirs(Path f, FsPermission permission
) throws IOException;
/**
- * The src file is on the local disk. Add it to FS at
+ * The src file is on the local disk. Add it to filesystem at
* the given dst name and the source is kept intact afterwards
* @param src path
* @param dst path
+ * @throws IOException IO failure
*/
public void copyFromLocalFile(Path src, Path dst)
throws IOException {
@@ -2030,10 +2240,11 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * The src files is on the local disk. Add it to FS at
+ * The src files is on the local disk. Add it to filesystem at
* the given dst name, removing the source afterwards.
- * @param srcs path
+ * @param srcs source paths
* @param dst path
+ * @throws IOException IO failure
*/
public void moveFromLocalFile(Path[] srcs, Path dst)
throws IOException {
@@ -2041,10 +2252,11 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * The src file is on the local disk. Add it to FS at
+ * The src file is on the local disk. Add it to the filesystem at
* the given dst name, removing the source afterwards.
- * @param src path
+ * @param src local path
* @param dst path
+ * @throws IOException IO failure
*/
public void moveFromLocalFile(Path src, Path dst)
throws IOException {
@@ -2052,7 +2264,7 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * The src file is on the local disk. Add it to FS at
+ * The src file is on the local disk. Add it to the filesystem at
* the given dst name.
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
@@ -2063,80 +2275,83 @@ public abstract class FileSystem extends Configured implements Closeable {
throws IOException {
copyFromLocalFile(delSrc, true, src, dst);
}
-
+
/**
- * The src files are on the local disk. Add it to FS at
+ * The src files are on the local disk. Add it to the filesystem at
* the given dst name.
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file
* @param srcs array of paths which are source
* @param dst path
+ * @throws IOException IO failure
*/
- public void copyFromLocalFile(boolean delSrc, boolean overwrite,
+ public void copyFromLocalFile(boolean delSrc, boolean overwrite,
Path[] srcs, Path dst)
throws IOException {
Configuration conf = getConf();
FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf);
}
-
+
/**
- * The src file is on the local disk. Add it to FS at
+ * The src file is on the local disk. Add it to the filesystem at
* the given dst name.
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file
* @param src path
* @param dst path
+ * @throws IOException IO failure
*/
- public void copyFromLocalFile(boolean delSrc, boolean overwrite,
+ public void copyFromLocalFile(boolean delSrc, boolean overwrite,
Path src, Path dst)
throws IOException {
Configuration conf = getConf();
FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf);
}
-
+
/**
- * The src file is under FS, and the dst is on the local disk.
- * Copy it from FS control to the local dst name.
- * @param src path
- * @param dst path
+ * Copy it a file from the remote filesystem to the local one.
+ * @param src path src file in the remote filesystem
+ * @param dst path local destination
+ * @throws IOException IO failure
*/
public void copyToLocalFile(Path src, Path dst) throws IOException {
copyToLocalFile(false, src, dst);
}
-
+
/**
- * The src file is under FS, and the dst is on the local disk.
- * Copy it from FS control to the local dst name.
- * Remove the source afterwards
- * @param src path
- * @param dst path
+ * Copy a file to the local filesystem, then delete it from the
+ * remote filesystem (if successfully copied).
+ * @param src path src file in the remote filesystem
+ * @param dst path local destination
+ * @throws IOException IO failure
*/
public void moveToLocalFile(Path src, Path dst) throws IOException {
copyToLocalFile(true, src, dst);
}
/**
- * The src file is under FS, and the dst is on the local disk.
- * Copy it from FS control to the local dst name.
+ * Copy it a file from a remote filesystem to the local one.
* delSrc indicates if the src will be removed or not.
* @param delSrc whether to delete the src
- * @param src path
- * @param dst path
- */
+ * @param src path src file in the remote filesystem
+ * @param dst path local destination
+ * @throws IOException IO failure
+ */
public void copyToLocalFile(boolean delSrc, Path src, Path dst)
throws IOException {
copyToLocalFile(delSrc, src, dst, false);
}
-
- /**
- * The src file is under FS, and the dst is on the local disk. Copy it from FS
- * control to the local dst name. delSrc indicates if the src will be removed
+
+ /**
+ * The src file is under this filesystem, and the dst is on the local disk.
+ * Copy it from the remote filesystem to the local dst name.
+ * delSrc indicates if the src will be removed
* or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
- * as local file system or not. RawLocalFileSystem is non crc file system.So,
- * It will not create any crc files at local.
- *
+ * as the local file system or not. RawLocalFileSystem is non checksumming,
+ * So, It will not create any crc files at local.
+ *
* @param delSrc
* whether to delete the src
* @param src
@@ -2145,9 +2360,8 @@ public abstract class FileSystem extends Configured implements Closeable {
* path
* @param useRawLocalFileSystem
* whether to use RawLocalFileSystem as local file system or not.
- *
- * @throws IOException
- * - if any IO error
+ *
+ * @throws IOException for any IO error
*/
public void copyToLocalFile(boolean delSrc, Path src, Path dst,
boolean useRawLocalFileSystem) throws IOException {
@@ -2162,12 +2376,14 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Returns a local File that the user can write output to. The caller
- * provides both the eventual FS target name and the local working
- * file. If the FS is local, we write directly into the target. If
- * the FS is remote, we write into the tmp local area.
+ * Returns a local file that the user can write output to. The caller
+ * provides both the eventual target name in this FileSystem
+ * and the local working file path.
+ * If this FileSystem is local, we write directly into the target. If
+ * the FileSystem is not local, we write into the tmp local area.
* @param fsOutputFile path of output file
* @param tmpLocalFile path of local tmp file
+ * @throws IOException IO failure
*/
public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
throws IOException {
@@ -2175,12 +2391,14 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Called when we're all done writing to the target. A local FS will
- * do nothing, because we've written to exactly the right place. A remote
- * FS will copy the contents of tmpLocalFile to the correct target at
+ * Called when we're all done writing to the target.
+ * A local FS will do nothing, because we've written to exactly the
+ * right place.
+ * A remote FS will copy the contents of tmpLocalFile to the correct target at
* fsOutputFile.
* @param fsOutputFile path of output file
* @param tmpLocalFile path to local tmp file
+ * @throws IOException IO failure
*/
public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
throws IOException {
@@ -2188,8 +2406,14 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * No more filesystem operations are needed. Will
- * release any held locks.
+ * Close this FileSystem instance.
+ * Will release any held locks, delete all files queued for deletion
+ * through calls to {@link #deleteOnExit(Path)}, and remove this FS instance
+ * from the cache, if cached.
+ *
+ * After this operation, the outcome of any method call on this FileSystem
+ * instance, or any input/output stream created by it is <i>undefined</i>.
+ * @throws IOException IO failure
*/
@Override
public void close() throws IOException {
@@ -2198,13 +2422,19 @@ public abstract class FileSystem extends Configured implements Closeable {
CACHE.remove(this.key, this);
}
- /** Return the total size of all files in the filesystem. */
+ /**
+ * Return the total size of all files in the filesystem.
+ * @throws IOException IO failure
+ */
public long getUsed() throws IOException {
Path path = new Path("/");
return getUsed(path);
}
- /** Return the total size of all files from a specified path. */
+ /**
+ * Return the total size of all files from a specified path.
+ * @throws IOException IO failure
+ */
public long getUsed(Path path) throws IOException {
return getContentSummary(path).getLength();
}
@@ -2213,8 +2443,10 @@ public abstract class FileSystem extends Configured implements Closeable {
* Get the block size for a particular file.
* @param f the filename
* @return the number of bytes in a block
+ * @deprecated Use {@link #getFileStatus(Path)} instead
+ * @throws FileNotFoundException if the path is not present
+ * @throws IOException IO failure
*/
- /** @deprecated Use getFileStatus() instead */
@Deprecated
public long getBlockSize(Path f) throws IOException {
return getFileStatus(f).getBlockSize();
@@ -2222,7 +2454,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Return the number of bytes that large input files should be optimally
- * be split into to minimize i/o time.
+ * be split into to minimize I/O time.
* @deprecated use {@link #getDefaultBlockSize(Path)} instead
*/
@Deprecated
@@ -2230,9 +2462,10 @@ public abstract class FileSystem extends Configured implements Closeable {
// default to 32MB: large enough to minimize the impact of seeks
return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024);
}
-
- /** Return the number of bytes that large input files should be optimally
- * be split into to minimize i/o time. The given path will be used to
+
+ /**
+ * Return the number of bytes that large input files should be optimally
+ * be split into to minimize I/O time. The given path will be used to
* locate the actual filesystem. The full path does not have to exist.
* @param f path of
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/48] hadoop git commit: Merge branch 'trunk' into HADOOP-13345
Posted by li...@apache.org.
Merge branch 'trunk' into HADOOP-13345
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e93093e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e93093e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e93093e
Branch: refs/heads/HADOOP-13345
Commit: 5e93093e6fbd2493f020bba770385764ef593114
Parents: 86a67ff 8f6e143
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Nov 29 16:45:38 2016 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Nov 29 16:45:38 2016 -0800
----------------------------------------------------------------------
hadoop-assemblies/pom.xml | 1 -
hadoop-build-tools/pom.xml | 2 +
hadoop-client/pom.xml | 1 -
.../hadoop-cloud-storage/pom.xml | 1 -
hadoop-cloud-storage-project/pom.xml | 1 -
.../hadoop-annotations/pom.xml | 1 -
.../hadoop-auth-examples/pom.xml | 1 -
hadoop-common-project/hadoop-auth/pom.xml | 1 -
hadoop-common-project/hadoop-common/pom.xml | 15 +-
.../org/apache/hadoop/conf/Configuration.java | 8 +-
.../crypto/key/kms/KMSClientProvider.java | 43 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 1455 +++++++----
.../org/apache/hadoop/fs/shell/Display.java | 11 +-
.../org/apache/hadoop/fs/shell/FsUsage.java | 124 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 6 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 73 +-
.../hadoop/fs/viewfs/ViewFileSystemUtil.java | 164 ++
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 74 +-
.../io/erasurecode/ErasureCodeConstants.java | 3 +
.../apache/hadoop/ipc/DecayRpcScheduler.java | 2 +-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 75 +-
.../hadoop/ipc/ProtobufRpcEngineCallback.java | 29 +
.../main/java/org/apache/hadoop/ipc/RPC.java | 5 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 199 +-
.../apache/hadoop/ipc/WritableRpcEngine.java | 3 +-
.../hadoop/ipc/metrics/RpcDetailedMetrics.java | 6 +
.../apache/hadoop/ipc/metrics/RpcMetrics.java | 33 +
.../org/apache/hadoop/jmx/JMXJsonServlet.java | 6 +-
.../java/org/apache/hadoop/log/Log4Json.java | 14 +-
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 4 +-
.../hadoop/security/AccessControlException.java | 2 +-
.../org/apache/hadoop/security/Credentials.java | 2 +-
.../java/org/apache/hadoop/security/KDiag.java | 67 +
.../apache/hadoop/security/SecurityUtil.java | 10 +-
.../hadoop/security/UserGroupInformation.java | 6 +-
.../authorize/AuthorizationException.java | 2 +-
.../org/apache/hadoop/security/token/Token.java | 2 +-
.../AbstractDelegationTokenIdentifier.java | 2 +-
.../AbstractDelegationTokenSecretManager.java | 2 +-
.../DelegationTokenAuthenticationHandler.java | 6 +-
.../web/DelegationTokenAuthenticator.java | 2 +-
.../apache/hadoop/util/HttpExceptionUtils.java | 2 +-
.../src/site/markdown/DeprecatedProperties.md | 7 +-
.../src/site/markdown/filesystem/filesystem.md | 2 +-
.../apache/hadoop/conf/TestConfiguration.java | 2 +-
.../org/apache/hadoop/fs/SymlinkBaseTest.java | 3 +-
.../org/apache/hadoop/fs/TestDefaultUri.java | 40 +-
.../apache/hadoop/fs/TestFileSystemCaching.java | 8 +-
.../hadoop/fs/viewfs/TestViewfsFileStatus.java | 9 +-
.../fs/viewfs/ViewFileSystemBaseTest.java | 94 +-
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 22 +
.../apache/hadoop/io/TestSequenceFileSync.java | 113 +-
.../ipc/TestProtoBufRpcServerHandoff.java | 167 ++
.../apache/hadoop/ipc/TestRpcServerHandoff.java | 218 ++
.../org/apache/hadoop/log/TestLog4Json.java | 13 +-
.../apache/hadoop/security/TestKDiagNoKDC.java | 8 +
...tionTokenAuthenticationHandlerWithMocks.java | 3 +-
.../delegation/web/TestWebDelegationToken.java | 4 +-
.../hadoop/util/TestHttpExceptionUtils.java | 2 +-
.../hadoop-common/src/test/proto/test.proto | 11 +-
.../src/test/proto/test_rpc_service.proto | 6 +-
hadoop-common-project/hadoop-kms/pom.xml | 5 +-
.../crypto/key/kms/server/KMSJSONReader.java | 3 +-
.../crypto/key/kms/server/KMSJSONWriter.java | 3 +-
.../hadoop-kms/src/site/markdown/index.md.vm | 2 +
.../hadoop/crypto/key/kms/server/TestKMS.java | 97 +-
hadoop-common-project/hadoop-minikdc/pom.xml | 1 -
hadoop-common-project/hadoop-nfs/pom.xml | 1 -
hadoop-common-project/pom.xml | 1 -
hadoop-dist/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 -
.../apache/hadoop/hdfs/HdfsConfiguration.java | 10 +-
.../hdfs/client/HdfsClientConfigKeys.java | 10 +-
.../hadoop/hdfs/protocol/HdfsConstants.java | 1 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 1 -
.../src/site/markdown/ServerSetup.md.vm | 10 +-
.../hadoop-hdfs-native-client/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 -
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 19 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 4 +-
.../server/blockmanagement/BlockManager.java | 62 +-
.../BlockPlacementPolicyDefault.java | 8 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 4 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 4 +-
.../namenode/ErasureCodingPolicyManager.java | 23 +-
.../hdfs/server/namenode/FSNamesystem.java | 7 +
.../hadoop/hdfs/server/namenode/INodeFile.java | 8 +-
.../src/main/resources/hdfs-default.xml | 8 +-
.../src/site/markdown/TransparentEncryption.md | 2 +-
.../fs/viewfs/TestViewFileSystemHdfs.java | 53 +
.../apache/hadoop/hdfs/AdminStatesBaseTest.java | 4 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 28 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 2 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 50 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 29 +-
.../TestDFSStripedOutputStreamWithFailure.java | 39 +-
.../hdfs/TestDFSXORStripedInputStream.java | 33 +
.../hdfs/TestDFSXORStripedOutputStream.java | 35 +
...estDFSXORStripedOutputStreamWithFailure.java | 36 +
.../hdfs/TestDecommissionWithStriped.java | 10 +-
.../apache/hadoop/hdfs/TestDeprecatedKeys.java | 22 +-
.../hadoop/hdfs/TestEncryptedTransfer.java | 2 +-
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 2 +-
.../apache/hadoop/hdfs/TestFileChecksum.java | 2 +-
.../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +-
.../hadoop/hdfs/TestMissingBlocksAlert.java | 3 +-
.../hdfs/TestReadStripedFileWithDecoding.java | 3 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 4 +-
.../hdfs/TestReplaceDatanodeOnFailure.java | 3 +-
.../org/apache/hadoop/hdfs/TestReplication.java | 3 +-
.../hadoop/hdfs/TestWriteReadStripedFile.java | 2 +-
.../hdfs/server/balancer/TestBalancer.java | 26 +-
.../blockmanagement/BlockManagerTestUtil.java | 21 +-
.../TestBlocksWithNotEnoughRacks.java | 4 +-
.../server/blockmanagement/TestNodeCount.java | 2 +-
.../TestPendingInvalidateBlock.java | 2 +-
.../TestPendingReconstruction.java | 2 +-
...constructStripedBlocksWithRackAwareness.java | 8 +-
.../blockmanagement/TestReplicationPolicy.java | 2 +-
.../TestReplicationPolicyConsiderLoad.java | 2 +-
.../TestDataNodeErasureCodingMetrics.java | 2 +-
.../fsdataset/impl/TestLazyPersistFiles.java | 12 +-
.../hadoop/hdfs/server/mover/TestMover.java | 9 +-
.../hdfs/server/mover/TestStorageMover.java | 4 +-
.../server/namenode/NNThroughputBenchmark.java | 10 +-
.../TestAddOverReplicatedStripedBlocks.java | 2 +-
.../namenode/TestDecommissioningStatus.java | 5 +-
.../server/namenode/TestFSEditLogLoader.java | 2 +-
.../hdfs/server/namenode/TestHostsFiles.java | 4 +-
.../hdfs/server/namenode/TestMetaSave.java | 5 +-
.../namenode/TestReconstructStripedBlocks.java | 9 +-
.../server/namenode/TestStripedINodeFile.java | 5 +-
.../hdfs/server/namenode/ha/TestDNFencing.java | 11 +-
.../ha/TestDNFencingWithReplication.java | 2 +-
.../namenode/ha/TestPipelinesFailover.java | 5 +-
.../TestNNMetricFilesInGetListingOps.java | 2 +-
.../namenode/metrics/TestNameNodeMetrics.java | 12 +-
hadoop-hdfs-project/pom.xml | 1 -
.../hadoop-mapreduce-client-app/pom.xml | 1 -
.../org/apache/hadoop/mapred/YarnChild.java | 2 +
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 3 +
.../hadoop-mapreduce-client-common/pom.xml | 1 -
.../hadoop-mapreduce-client-core/pom.xml | 1 -
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 1 -
.../hadoop-mapreduce-client-hs/pom.xml | 1 -
.../hadoop-mapreduce-client-jobclient/pom.xml | 1 -
.../hadoop/mapred/ClientServiceDelegate.java | 4 +-
.../hadoop-mapreduce-client-nativetask/pom.xml | 11 +-
.../src/CMakeLists.txt | 2 +
.../src/main/native/gtest/gtest-all.cc | 1483 ++++++++---
.../src/main/native/gtest/include/gtest/gtest.h | 2341 +++++++++++++-----
.../hadoop-mapreduce-client-shuffle/pom.xml | 1 -
.../hadoop-mapreduce-client/pom.xml | 1 -
.../hadoop-mapreduce-examples/pom.xml | 1 -
hadoop-mapreduce-project/pom.xml | 1 -
hadoop-maven-plugins/pom.xml | 1 -
hadoop-minicluster/pom.xml | 1 -
hadoop-project-dist/pom.xml | 1 -
hadoop-project/pom.xml | 3 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 1 -
hadoop-tools/hadoop-archives/pom.xml | 1 -
.../hadoop/fs/s3a/RenameFailedException.java | 70 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 89 +-
.../fs/s3a/ITestS3AFileSystemContract.java | 15 -
.../fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 11 +-
.../src/test/resources/contract/s3a.xml | 2 +-
hadoop-tools/hadoop-azure-datalake/pom.xml | 1 -
hadoop-tools/hadoop-datajoin/pom.xml | 1 -
hadoop-tools/hadoop-distcp/pom.xml | 1 -
hadoop-tools/hadoop-extras/pom.xml | 1 -
hadoop-tools/hadoop-gridmix/pom.xml | 1 -
hadoop-tools/hadoop-pipes/pom.xml | 1 -
hadoop-tools/hadoop-rumen/pom.xml | 6 -
hadoop-tools/hadoop-sls/pom.xml | 1 -
hadoop-tools/hadoop-streaming/pom.xml | 1 -
hadoop-tools/hadoop-tools-dist/pom.xml | 1 -
hadoop-tools/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 1 -
.../api/records/AbstractResourceRequest.java | 55 +
.../yarn/api/records/ApplicationReport.java | 10 +
.../yarn/api/records/ApplicationTimeout.java | 99 +
.../yarn/api/records/ResourceRequest.java | 19 +-
.../api/records/UpdateContainerRequest.java | 18 +-
.../src/main/proto/yarn_protos.proto | 7 +
.../pom.xml | 1 -
.../pom.xml | 1 -
.../hadoop-yarn-applications/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 1 -
.../hadoop/yarn/client/api/YarnClient.java | 11 +
.../yarn/client/api/impl/YarnClientImpl.java | 9 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 45 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 40 +
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 1 -
.../impl/pb/ApplicationReportPBImpl.java | 83 +
.../impl/pb/ApplicationTimeoutPBImpl.java | 130 +
.../hadoop/yarn/event/AsyncDispatcher.java | 6 +-
.../yarn/util/ResourceCalculatorPlugin.java | 3 +
.../resource/DefaultResourceCalculator.java | 19 +-
.../resource/DominantResourceCalculator.java | 25 +-
.../yarn/util/resource/ResourceCalculator.java | 17 +-
.../hadoop/yarn/util/resource/Resources.java | 4 +
.../src/main/resources/yarn-default.xml | 6 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 2 +
.../hadoop/yarn/event/DrainDispatcher.java | 9 +-
.../hadoop/yarn/event/TestAsyncDispatcher.java | 42 +
.../util/resource/TestResourceCalculator.java | 79 +
.../pom.xml | 1 -
.../hadoop-yarn-server-common/pom.xml | 1 -
.../hadoop-yarn-server-nodemanager/pom.xml | 1 -
.../nodemanager/NodeResourceMonitorImpl.java | 5 +
.../monitor/ContainersMonitorImpl.java | 39 +-
.../util/NodeManagerHardwareUtils.java | 160 +-
.../TestContainersMonitorResourceChange.java | 16 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 1 -
.../server/resourcemanager/ClientRMService.java | 2 +-
...pportunisticContainerAllocatorAMService.java | 13 +-
.../server/resourcemanager/RMAppManager.java | 6 +-
.../server/resourcemanager/RMAuditLogger.java | 1 +
.../server/resourcemanager/RMServerUtils.java | 11 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 17 +
.../scheduler/AbstractYarnScheduler.java | 20 +
.../scheduler/AppSchedulingInfo.java | 17 +
.../scheduler/SchedulerApplicationAttempt.java | 16 +
.../scheduler/SchedulerUtils.java | 59 +-
.../scheduler/YarnScheduler.java | 8 +
.../scheduler/capacity/CapacityScheduler.java | 4 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 16 +
.../distributed/NodeQueueLoadMonitor.java | 8 +-
.../scheduler/fair/FSAppAttempt.java | 242 +-
.../scheduler/fair/FSContext.java | 54 +
.../scheduler/fair/FSLeafQueue.java | 319 +--
.../scheduler/fair/FSParentQueue.java | 52 +-
.../scheduler/fair/FSPreemptionThread.java | 188 ++
.../resourcemanager/scheduler/fair/FSQueue.java | 37 +-
.../scheduler/fair/FSQueueMetrics.java | 60 +
.../scheduler/fair/FSSchedulerNode.java | 36 +-
.../scheduler/fair/FSStarvedApps.java | 85 +
.../scheduler/fair/FairScheduler.java | 309 +--
.../scheduler/fair/Schedulable.java | 29 +-
.../scheduler/fifo/FifoScheduler.java | 4 +-
.../webapp/JAXBContextResolver.java | 3 +-
.../resourcemanager/webapp/RMWebServices.java | 186 ++
.../resourcemanager/webapp/dao/AppInfo.java | 24 +
.../webapp/dao/AppTimeoutInfo.java | 71 +
.../webapp/dao/AppTimeoutsInfo.java | 47 +
...pportunisticContainerAllocatorAMService.java | 97 +-
.../yarn/server/resourcemanager/TestRMHA.java | 3 +-
...tyPreemptionPolicyForReservedContainers.java | 430 ----
.../server/resourcemanager/rmapp/MockRMApp.java | 3 +-
.../rmapp/TestApplicationLifetimeMonitor.java | 30 +-
.../scheduler/TestSchedulerUtils.java | 22 +-
.../scheduler/fair/FairSchedulerTestBase.java | 39 +-
.../fair/FairSchedulerWithMockPreemption.java | 58 +
.../scheduler/fair/FakeSchedulable.java | 5 -
.../scheduler/fair/TestFSAppStarvation.java | 256 ++
.../scheduler/fair/TestFSLeafQueue.java | 165 +-
.../scheduler/fair/TestFairScheduler.java | 137 +
.../fair/TestFairSchedulerPreemption.java | 1540 ++----------
.../fair/TestQueueManagerRealScheduler.java | 128 +
.../scheduler/fair/TestSchedulingPolicy.java | 5 -
.../webapp/TestRMWebServicesApps.java | 2 +-
.../TestRMWebServicesAppsModification.java | 137 +-
.../pom.xml | 1 -
.../hadoop-yarn-server-tests/pom.xml | 1 -
.../hadoop/yarn/server/MiniYARNCluster.java | 7 +-
.../server/TestContainerManagerSecurity.java | 2 +-
.../pom.xml | 1 -
.../pom.xml | 1 -
...stTimelineReaderWebServicesHBaseStorage.java | 2 +-
.../storage/DataGeneratorForTest.java | 4 +-
.../storage/TestHBaseTimelineStorageApps.java | 8 +-
.../TestHBaseTimelineStorageEntities.java | 6 +-
.../flow/TestHBaseStorageFlowActivity.java | 6 +-
.../storage/flow/TestHBaseStorageFlowRun.java | 14 +-
.../flow/TestHBaseStorageFlowRunCompaction.java | 2 +-
.../hadoop-yarn-server-timelineservice/pom.xml | 1 -
.../storage/HBaseTimelineWriterImpl.java | 5 -
.../hadoop-yarn-server-web-proxy/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-site/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 1 -
hadoop-yarn-project/hadoop-yarn/pom.xml | 1 -
hadoop-yarn-project/pom.xml | 1 -
pom.xml | 2 +
286 files changed, 9225 insertions(+), 4927 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e93093e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 9b8692a,f1cde4e..cb08e57
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@@ -660,17 -659,21 +668,24 @@@ public class S3AFileSystem extends File
/**
* The inner rename operation. See {@link #rename(Path, Path)} for
* the description of the operation.
+ * This operation throws an exception on any failure which needs to be
+ * reported and downgraded to a failure. That is: if a rename
- * @param src path to be renamed
- * @param dst new path after rename
+ * @param source path to be renamed
+ * @param dest new path after rename
- * @return true if rename is successful
+ * @throws RenameFailedException if some criteria for a state changing
+ * rename was not met. This means work didn't happen; it's not something
+ * which is reported upstream to the FileSystem APIs, for which the semantics
+ * of "false" are pretty vague.
+ * @throws FileNotFoundException there's no source file.
* @throws IOException on IO failure.
* @throws AmazonClientException on failures inside the AWS SDK
*/
- private boolean innerRename(Path source, Path dest) throws IOException,
- AmazonClientException {
- private boolean innerRename(Path src, Path dst)
++ private boolean innerRename(Path source, Path dest)
+ throws RenameFailedException, FileNotFoundException, IOException,
+ AmazonClientException {
+ Path src = qualify(source);
+ Path dst = qualify(dest);
+
LOG.debug("Rename path {} to {}", src, dst);
incrementStatistic(INVOCATION_RENAME);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/48] hadoop git commit: HADOOP-13801 regression:
ITestS3AMiniYarnCluster failing. Contributed by Steve Loughran
Posted by li...@apache.org.
HADOOP-13801 regression: ITestS3AMiniYarnCluster failing. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0de0c32d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0de0c32d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0de0c32d
Branch: refs/heads/HADOOP-13345
Commit: 0de0c32ddd46eaf42198dbf24ec4344f6810ca09
Parents: 005850b
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 23 21:37:04 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 23 21:37:39 2016 +0000
----------------------------------------------------------------------
.../hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0de0c32d/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
index d8ae313..6db4eba 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
@@ -24,13 +24,14 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.examples.WordCount;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
@@ -140,8 +141,12 @@ public class ITestS3AMiniYarnCluster extends AbstractS3ATestBase {
* helper method.
*/
private String readStringFromFile(Path path) throws IOException {
- return ContractTestUtils.readBytesToString(fs, path,
- (int) fs.getFileStatus(path).getLen());
+ try (FSDataInputStream in = fs.open(path)) {
+ long bytesLen = fs.getFileStatus(path).getLen();
+ byte[] buffer = new byte[(int) bytesLen];
+ IOUtils.readFully(in, buffer, 0, buffer.length);
+ return new String(buffer);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/48] hadoop git commit: HADOOP-11552. Allow handoff on the server
side for RPC requests. Contributed by Siddharth Seth
Posted by li...@apache.org.
HADOOP-11552. Allow handoff on the server side for RPC requests. Contributed by Siddharth Seth
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d94da1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d94da1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d94da1e
Branch: refs/heads/HADOOP-13345
Commit: 3d94da1e00fc6238fad458e415219f87920f1fc3
Parents: 0de0c32
Author: Jian He <ji...@apache.org>
Authored: Wed Nov 23 16:01:55 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Wed Nov 23 16:01:55 2016 -0800
----------------------------------------------------------------------
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 75 ++++++-
.../hadoop/ipc/ProtobufRpcEngineCallback.java | 29 +++
.../main/java/org/apache/hadoop/ipc/Server.java | 197 +++++++++++++----
.../apache/hadoop/ipc/WritableRpcEngine.java | 3 +-
.../hadoop/ipc/metrics/RpcDetailedMetrics.java | 6 +
.../apache/hadoop/ipc/metrics/RpcMetrics.java | 33 +++
.../ipc/TestProtoBufRpcServerHandoff.java | 167 ++++++++++++++
.../apache/hadoop/ipc/TestRpcServerHandoff.java | 218 +++++++++++++++++++
.../hadoop-common/src/test/proto/test.proto | 11 +-
.../src/test/proto/test_rpc_service.proto | 6 +-
10 files changed, 696 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d94da1e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index e68bfd4..b3f5458 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.RPC.RpcInvoker;
-import org.apache.hadoop.ipc.RpcWritable;
import org.apache.hadoop.ipc.protobuf.ProtobufRpcEngineProtos.RequestHeaderProto;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.SecretManager;
@@ -345,6 +344,60 @@ public class ProtobufRpcEngine implements RpcEngine {
}
public static class Server extends RPC.Server {
+
+ static final ThreadLocal<ProtobufRpcEngineCallback> currentCallback =
+ new ThreadLocal<>();
+
+ static final ThreadLocal<CallInfo> currentCallInfo = new ThreadLocal<>();
+
+ static class CallInfo {
+ private final RPC.Server server;
+ private final String methodName;
+
+ public CallInfo(RPC.Server server, String methodName) {
+ this.server = server;
+ this.methodName = methodName;
+ }
+ }
+
+ static class ProtobufRpcEngineCallbackImpl
+ implements ProtobufRpcEngineCallback {
+
+ private final RPC.Server server;
+ private final Call call;
+ private final String methodName;
+ private final long setupTime;
+
+ public ProtobufRpcEngineCallbackImpl() {
+ this.server = currentCallInfo.get().server;
+ this.call = Server.getCurCall().get();
+ this.methodName = currentCallInfo.get().methodName;
+ this.setupTime = Time.now();
+ }
+
+ @Override
+ public void setResponse(Message message) {
+ long processingTime = Time.now() - setupTime;
+ call.setDeferredResponse(RpcWritable.wrap(message));
+ server.updateDeferredMetrics(methodName, processingTime);
+ }
+
+ @Override
+ public void error(Throwable t) {
+ long processingTime = Time.now() - setupTime;
+ String detailedMetricsName = t.getClass().getSimpleName();
+ server.updateDeferredMetrics(detailedMetricsName, processingTime);
+ call.setDeferredError(t);
+ }
+ }
+
+ @InterfaceStability.Unstable
+ public static ProtobufRpcEngineCallback registerForDeferredResponse() {
+ ProtobufRpcEngineCallback callback = new ProtobufRpcEngineCallbackImpl();
+ currentCallback.set(callback);
+ return callback;
+ }
+
/**
* Construct an RPC server.
*
@@ -462,9 +515,19 @@ public class ProtobufRpcEngine implements RpcEngine {
long startTime = Time.now();
int qTime = (int) (startTime - receiveTime);
Exception exception = null;
+ boolean isDeferred = false;
try {
server.rpcDetailedMetrics.init(protocolImpl.protocolClass);
+ currentCallInfo.set(new CallInfo(server, methodName));
result = service.callBlockingMethod(methodDescriptor, null, param);
+ // Check if this needs to be a deferred response,
+ // by checking the ThreadLocal callback being set
+ if (currentCallback.get() != null) {
+ Server.getCurCall().get().deferResponse();
+ isDeferred = true;
+ currentCallback.set(null);
+ return null;
+ }
} catch (ServiceException e) {
exception = (Exception) e.getCause();
throw (Exception) e.getCause();
@@ -472,10 +535,13 @@ public class ProtobufRpcEngine implements RpcEngine {
exception = e;
throw e;
} finally {
+ currentCallInfo.set(null);
int processingTime = (int) (Time.now() - startTime);
if (LOG.isDebugEnabled()) {
- String msg = "Served: " + methodName + " queueTime= " + qTime +
- " procesingTime= " + processingTime;
+ String msg =
+ "Served: " + methodName + (isDeferred ? ", deferred" : "") +
+ ", queueTime= " + qTime +
+ " procesingTime= " + processingTime;
if (exception != null) {
msg += " exception= " + exception.getClass().getSimpleName();
}
@@ -484,7 +550,8 @@ public class ProtobufRpcEngine implements RpcEngine {
String detailedMetricsName = (exception == null) ?
methodName :
exception.getClass().getSimpleName();
- server.updateMetrics(detailedMetricsName, qTime, processingTime);
+ server.updateMetrics(detailedMetricsName, qTime, processingTime,
+ isDeferred);
}
return RpcWritable.wrap(result);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d94da1e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngineCallback.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngineCallback.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngineCallback.java
new file mode 100644
index 0000000..463593e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngineCallback.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc;
+
+import com.google.protobuf.Message;
+
+public interface ProtobufRpcEngineCallback {
+
+ public void setResponse(Message message);
+
+ public void error(Throwable t);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d94da1e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 8f1956e..5f9de8a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -498,18 +498,25 @@ public abstract class Server {
}
}
- void updateMetrics(String name, int queueTime, int processingTime) {
+ void updateMetrics(String name, int queueTime, int processingTime,
+ boolean deferredCall) {
rpcMetrics.addRpcQueueTime(queueTime);
- rpcMetrics.addRpcProcessingTime(processingTime);
- rpcDetailedMetrics.addProcessingTime(name, processingTime);
- callQueue.addResponseTime(name, getPriorityLevel(), queueTime,
- processingTime);
-
- if (isLogSlowRPC()) {
- logSlowRpcCalls(name, processingTime);
+ if (!deferredCall) {
+ rpcMetrics.addRpcProcessingTime(processingTime);
+ rpcDetailedMetrics.addProcessingTime(name, processingTime);
+ callQueue.addResponseTime(name, getPriorityLevel(), queueTime,
+ processingTime);
+ if (isLogSlowRPC()) {
+ logSlowRpcCalls(name, processingTime);
+ }
}
}
+ void updateDeferredMetrics(String name, long processingTime) {
+ rpcMetrics.addDeferredRpcProcessingTime(processingTime);
+ rpcDetailedMetrics.addDeferredProcessingTime(name, processingTime);
+ }
+
/**
* A convenience method to bind to a given address and report
* better exceptions if the address is not a valid host.
@@ -675,6 +682,7 @@ public abstract class Server {
final byte[] clientId;
private final TraceScope traceScope; // the HTrace scope on the server side
private final CallerContext callerContext; // the call context
+ private boolean deferredResponse = false;
private int priorityLevel;
// the priority level assigned by scheduler, 0 by default
@@ -783,6 +791,22 @@ public abstract class Server {
public void setPriorityLevel(int priorityLevel) {
this.priorityLevel = priorityLevel;
}
+
+ @InterfaceStability.Unstable
+ public void deferResponse() {
+ this.deferredResponse = true;
+ }
+
+ @InterfaceStability.Unstable
+ public boolean isResponseDeferred() {
+ return this.deferredResponse;
+ }
+
+ public void setDeferredResponse(Writable response) {
+ }
+
+ public void setDeferredError(Throwable t) {
+ }
}
/** A RPC extended call queued for handling. */
@@ -836,43 +860,58 @@ public abstract class Server {
Server.LOG.info(Thread.currentThread().getName() + ": skipped " + this);
return null;
}
- String errorClass = null;
- String error = null;
- RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
- RpcErrorCodeProto detailedErr = null;
Writable value = null;
+ ResponseParams responseParams = new ResponseParams();
try {
value = call(
rpcKind, connection.protocolName, rpcRequest, timestamp);
} catch (Throwable e) {
- if (e instanceof UndeclaredThrowableException) {
- e = e.getCause();
- }
- logException(Server.LOG, e, this);
- if (e instanceof RpcServerException) {
- RpcServerException rse = ((RpcServerException)e);
- returnStatus = rse.getRpcStatusProto();
- detailedErr = rse.getRpcErrorCodeProto();
- } else {
- returnStatus = RpcStatusProto.ERROR;
- detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;
- }
- errorClass = e.getClass().getName();
- error = StringUtils.stringifyException(e);
- // Remove redundant error class name from the beginning of the
- // stack trace
- String exceptionHdr = errorClass + ": ";
- if (error.startsWith(exceptionHdr)) {
- error = error.substring(exceptionHdr.length());
+ populateResponseParamsOnError(e, responseParams);
+ }
+ if (!isResponseDeferred()) {
+ setupResponse(this, responseParams.returnStatus,
+ responseParams.detailedErr,
+ value, responseParams.errorClass, responseParams.error);
+ sendResponse();
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deferring response for callId: " + this.callId);
}
}
- setupResponse(this, returnStatus, detailedErr,
- value, errorClass, error);
- sendResponse();
return null;
}
+ /**
+ * @param t the {@link java.lang.Throwable} to use to set
+ * errorInfo
+ * @param responseParams the {@link ResponseParams} instance to populate
+ */
+ private void populateResponseParamsOnError(Throwable t,
+ ResponseParams responseParams) {
+ if (t instanceof UndeclaredThrowableException) {
+ t = t.getCause();
+ }
+ logException(Server.LOG, t, this);
+ if (t instanceof RpcServerException) {
+ RpcServerException rse = ((RpcServerException) t);
+ responseParams.returnStatus = rse.getRpcStatusProto();
+ responseParams.detailedErr = rse.getRpcErrorCodeProto();
+ } else {
+ responseParams.returnStatus = RpcStatusProto.ERROR;
+ responseParams.detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;
+ }
+ responseParams.errorClass = t.getClass().getName();
+ responseParams.error = StringUtils.stringifyException(t);
+ // Remove redundant error class name from the beginning of the
+ // stack trace
+ String exceptionHdr = responseParams.errorClass + ": ";
+ if (responseParams.error.startsWith(exceptionHdr)) {
+ responseParams.error =
+ responseParams.error.substring(exceptionHdr.length());
+ }
+ }
+
void setResponse(ByteBuffer response) throws IOException {
this.rpcResponse = response;
}
@@ -892,6 +931,80 @@ public abstract class Server {
connection.sendResponse(call);
}
+ /**
+ * Send a deferred response, ignoring errors.
+ */
+ private void sendDeferedResponse() {
+ try {
+ connection.sendResponse(this);
+ } catch (Exception e) {
+ // For synchronous calls, application code is done once it's returned
+ // from a method. It does not expect to receive an error.
+ // This is equivalent to what happens in synchronous calls when the
+ // Responder is not able to send out the response.
+ LOG.error("Failed to send deferred response. ThreadName=" + Thread
+ .currentThread().getName() + ", CallId="
+ + callId + ", hostname=" + getHostAddress());
+ }
+ }
+
+ @Override
+ public void setDeferredResponse(Writable response) {
+ if (this.connection.getServer().running) {
+ try {
+ setupResponse(this, RpcStatusProto.SUCCESS, null, response,
+ null, null);
+ } catch (IOException e) {
+ // For synchronous calls, application code is done once it has
+ // returned from a method. It does not expect to receive an error.
+ // This is equivalent to what happens in synchronous calls when the
+ // response cannot be sent.
+ LOG.error(
+ "Failed to setup deferred successful response. ThreadName=" +
+ Thread.currentThread().getName() + ", Call=" + this);
+ return;
+ }
+ sendDeferedResponse();
+ }
+ }
+
+ @Override
+ public void setDeferredError(Throwable t) {
+ if (this.connection.getServer().running) {
+ if (t == null) {
+ t = new IOException(
+ "User code indicated an error without an exception");
+ }
+ try {
+ ResponseParams responseParams = new ResponseParams();
+ populateResponseParamsOnError(t, responseParams);
+ setupResponse(this, responseParams.returnStatus,
+ responseParams.detailedErr,
+ null, responseParams.errorClass, responseParams.error);
+ } catch (IOException e) {
+ // For synchronous calls, application code is done once it has
+ // returned from a method. It does not expect to receive an error.
+ // This is equivalent to what happens in synchronous calls when the
+ // response cannot be sent.
+ LOG.error(
+ "Failed to setup deferred error response. ThreadName=" +
+ Thread.currentThread().getName() + ", Call=" + this);
+ }
+ sendDeferedResponse();
+ }
+ }
+
+ /**
+ * Holds response parameters. Defaults set to work for successful
+ * invocations
+ */
+ private class ResponseParams {
+ String errorClass = null;
+ String error = null;
+ RpcErrorCodeProto detailedErr = null;
+ RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
+ }
+
@Override
public String toString() {
return super.toString() + " " + rpcRequest + " from " + connection;
@@ -1593,6 +1706,10 @@ public abstract class Server {
return lastContact;
}
+ public Server getServer() {
+ return Server.this;
+ }
+
/* Return true if the connection has no outstanding rpc */
private boolean isIdle() {
return rpcCount.get() == 0;
@@ -2105,7 +2222,6 @@ public abstract class Server {
}
/** Reads the connection context following the connection header
- * @param dis - DataInputStream from which to read the header
* @throws WrappedRpcServerException - if the header cannot be
* deserialized, or the user is not authorized
*/
@@ -2215,7 +2331,7 @@ public abstract class Server {
* if SASL then SASL has been established and the buf we are passed
* has been unwrapped from SASL.
*
- * @param buf - contains the RPC request header and the rpc request
+ * @param bb - contains the RPC request header and the rpc request
* @throws IOException - internal error that should not be returned to
* client, typically failure to respond to client
* @throws WrappedRpcServerException - an exception that is sent back to the
@@ -2294,7 +2410,7 @@ public abstract class Server {
* - A successfully decoded RpcCall will be deposited in RPC-Q and
* its response will be sent later when the request is processed.
* @param header - RPC request header
- * @param dis - stream to request payload
+ * @param buffer - stream to request payload
* @throws WrappedRpcServerException - due to fatal rpc layer issues such
* as invalid header or deserialization error. In this case a RPC fatal
* status response will later be sent back to client.
@@ -2370,7 +2486,7 @@ public abstract class Server {
* Establish RPC connection setup by negotiating SASL if required, then
* reading and authorizing the connection header
* @param header - RPC header
- * @param dis - stream to request payload
+ * @param buffer - stream to request payload
* @throws WrappedRpcServerException - setup failed due to SASL
* negotiation failure, premature or invalid connection context,
* or other state errors. This exception needs to be sent to the
@@ -2439,8 +2555,6 @@ public abstract class Server {
/**
* Decode the a protobuf from the given input stream
- * @param builder - Builder of the protobuf to decode
- * @param dis - DataInputStream to read the protobuf
* @return Message - decoded protobuf
* @throws WrappedRpcServerException - deserialization failed
*/
@@ -2744,11 +2858,10 @@ public abstract class Server {
private void closeConnection(Connection connection) {
connectionManager.close(connection);
}
-
+
/**
* Setup response for the IPC Call.
*
- * @param responseBuf buffer to serialize the response into
* @param call {@link Call} to which we are setting up the response
* @param status of the IPC call
* @param rv return value for the IPC Call, if the call was successful
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d94da1e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
index 3d6d461..1763c7f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
@@ -552,7 +552,8 @@ public class WritableRpcEngine implements RpcEngine {
String detailedMetricsName = (exception == null) ?
call.getMethodName() :
exception.getClass().getSimpleName();
- server.updateMetrics(detailedMetricsName, qTime, processingTime);
+ server
+ .updateMetrics(detailedMetricsName, qTime, processingTime, false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d94da1e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcDetailedMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcDetailedMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcDetailedMetrics.java
index ad36742..8b7e995 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcDetailedMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcDetailedMetrics.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation;
public class RpcDetailedMetrics {
@Metric MutableRatesWithAggregation rates;
+ @Metric MutableRatesWithAggregation deferredRpcRates;
static final Log LOG = LogFactory.getLog(RpcDetailedMetrics.class);
final MetricsRegistry registry;
@@ -60,6 +61,7 @@ public class RpcDetailedMetrics {
*/
public void init(Class<?> protocol) {
rates.init(protocol);
+ deferredRpcRates.init(protocol);
}
/**
@@ -72,6 +74,10 @@ public class RpcDetailedMetrics {
rates.add(name, processingTime);
}
+ public void addDeferredProcessingTime(String name, long processingTime) {
+ deferredRpcRates.add(name, processingTime);
+ }
+
/**
* Shutdown the instrumentation for the process
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d94da1e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java
index ef43618..e5dde10 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java
@@ -61,6 +61,8 @@ public class RpcMetrics {
new MutableQuantiles[intervals.length];
rpcProcessingTimeMillisQuantiles =
new MutableQuantiles[intervals.length];
+ deferredRpcProcessingTimeMillisQuantiles =
+ new MutableQuantiles[intervals.length];
for (int i = 0; i < intervals.length; i++) {
int interval = intervals[i];
rpcQueueTimeMillisQuantiles[i] = registry.newQuantiles("rpcQueueTime"
@@ -69,6 +71,10 @@ public class RpcMetrics {
rpcProcessingTimeMillisQuantiles[i] = registry.newQuantiles(
"rpcProcessingTime" + interval + "s",
"rpc processing time in milli second", "ops", "latency", interval);
+ deferredRpcProcessingTimeMillisQuantiles[i] = registry
+ .newQuantiles("deferredRpcProcessingTime" + interval + "s",
+ "deferred rpc processing time in milli seconds", "ops",
+ "latency", interval);
}
}
LOG.debug("Initialized " + registry);
@@ -87,6 +93,8 @@ public class RpcMetrics {
MutableQuantiles[] rpcQueueTimeMillisQuantiles;
@Metric("Processing time") MutableRate rpcProcessingTime;
MutableQuantiles[] rpcProcessingTimeMillisQuantiles;
+ @Metric("Deferred Processing time") MutableRate deferredRpcProcessingTime;
+ MutableQuantiles[] deferredRpcProcessingTimeMillisQuantiles;
@Metric("Number of authentication failures")
MutableCounterLong rpcAuthenticationFailures;
@Metric("Number of authentication successes")
@@ -202,6 +210,15 @@ public class RpcMetrics {
}
}
+ public void addDeferredRpcProcessingTime(long processingTime) {
+ deferredRpcProcessingTime.add(processingTime);
+ if (rpcQuantileEnable) {
+ for (MutableQuantiles q : deferredRpcProcessingTimeMillisQuantiles) {
+ q.add(processingTime);
+ }
+ }
+ }
+
/**
* One client backoff event
*/
@@ -255,4 +272,20 @@ public class RpcMetrics {
public long getRpcSlowCalls() {
return rpcSlowCalls.value();
}
+
+ public MutableRate getDeferredRpcProcessingTime() {
+ return deferredRpcProcessingTime;
+ }
+
+ public long getDeferredRpcProcessingSampleCount() {
+ return deferredRpcProcessingTime.lastStat().numSamples();
+ }
+
+ public double getDeferredRpcProcessingMean() {
+ return deferredRpcProcessingTime.lastStat().mean();
+ }
+
+ public double getDeferredRpcProcessingStdDev() {
+ return deferredRpcProcessingTime.lastStat().stddev();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d94da1e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpcServerHandoff.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpcServerHandoff.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpcServerHandoff.java
new file mode 100644
index 0000000..f5fefe4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpcServerHandoff.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.protobuf.TestProtos;
+import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcHandoffProto;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestProtoBufRpcServerHandoff {
+
+ public static final Log LOG =
+ LogFactory.getLog(TestProtoBufRpcServerHandoff.class);
+
+ @Test(timeout = 20000)
+ public void test() throws Exception {
+ Configuration conf = new Configuration();
+
+ TestProtoBufRpcServerHandoffServer serverImpl =
+ new TestProtoBufRpcServerHandoffServer();
+ BlockingService blockingService =
+ TestProtobufRpcHandoffProto.newReflectiveBlockingService(serverImpl);
+
+ RPC.setProtocolEngine(conf, TestProtoBufRpcServerHandoffProtocol.class,
+ ProtobufRpcEngine.class);
+ RPC.Server server = new RPC.Builder(conf)
+ .setProtocol(TestProtoBufRpcServerHandoffProtocol.class)
+ .setInstance(blockingService)
+ .setVerbose(true)
+ .setNumHandlers(1) // Num Handlers explicitly set to 1 for test.
+ .build();
+ server.start();
+
+ InetSocketAddress address = server.getListenerAddress();
+ long serverStartTime = System.currentTimeMillis();
+ LOG.info("Server started at: " + address + " at time: " + serverStartTime);
+
+ final TestProtoBufRpcServerHandoffProtocol client = RPC.getProxy(
+ TestProtoBufRpcServerHandoffProtocol.class, 1, address, conf);
+
+ ExecutorService executorService = Executors.newFixedThreadPool(2);
+ CompletionService<ClientInvocationCallable> completionService =
+ new ExecutorCompletionService<ClientInvocationCallable>(
+ executorService);
+
+ completionService.submit(new ClientInvocationCallable(client, 5000l));
+ completionService.submit(new ClientInvocationCallable(client, 5000l));
+
+ long submitTime = System.currentTimeMillis();
+ Future<ClientInvocationCallable> future1 = completionService.take();
+ Future<ClientInvocationCallable> future2 = completionService.take();
+
+ ClientInvocationCallable callable1 = future1.get();
+ ClientInvocationCallable callable2 = future2.get();
+
+ LOG.info(callable1);
+ LOG.info(callable2);
+
+ // Ensure the 5 second sleep responses are within a reasonable time of each
+ // other.
+ Assert.assertTrue(Math.abs(callable1.endTime - callable2.endTime) < 2000l);
+ Assert.assertTrue(System.currentTimeMillis() - submitTime < 7000l);
+
+ }
+
+ private static class ClientInvocationCallable
+ implements Callable<ClientInvocationCallable> {
+ final TestProtoBufRpcServerHandoffProtocol client;
+ final long sleepTime;
+ TestProtos.SleepResponseProto2 result;
+ long startTime;
+ long endTime;
+
+
+ private ClientInvocationCallable(
+ TestProtoBufRpcServerHandoffProtocol client, long sleepTime) {
+ this.client = client;
+ this.sleepTime = sleepTime;
+ }
+
+ @Override
+ public ClientInvocationCallable call() throws Exception {
+ startTime = System.currentTimeMillis();
+ result = client.sleep(null,
+ TestProtos.SleepRequestProto2.newBuilder().setSleepTime(sleepTime)
+ .build());
+ endTime = System.currentTimeMillis();
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return "startTime=" + startTime + ", endTime=" + endTime +
+ (result != null ?
+ ", result.receiveTime=" + result.getReceiveTime() +
+ ", result.responseTime=" +
+ result.getResponseTime() : "");
+ }
+ }
+
+ @ProtocolInfo(
+ protocolName = "org.apache.hadoop.ipc.TestProtoBufRpcServerHandoff$TestProtoBufRpcServerHandoffProtocol",
+ protocolVersion = 1)
+ public interface TestProtoBufRpcServerHandoffProtocol
+ extends TestProtobufRpcHandoffProto.BlockingInterface {
+ }
+
+ public static class TestProtoBufRpcServerHandoffServer
+ implements TestProtoBufRpcServerHandoffProtocol {
+
+ @Override
+ public TestProtos.SleepResponseProto2 sleep
+ (RpcController controller,
+ TestProtos.SleepRequestProto2 request) throws
+ ServiceException {
+ final long startTime = System.currentTimeMillis();
+ final ProtobufRpcEngineCallback callback =
+ ProtobufRpcEngine.Server.registerForDeferredResponse();
+ final long sleepTime = request.getSleepTime();
+ new Thread() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ callback.setResponse(
+ TestProtos.SleepResponseProto2.newBuilder()
+ .setReceiveTime(startTime)
+ .setResponseTime(System.currentTimeMillis()).build());
+ }
+ }.start();
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d94da1e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcServerHandoff.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcServerHandoff.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcServerHandoff.java
new file mode 100644
index 0000000..3716bc3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcServerHandoff.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.net.NetUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestRpcServerHandoff {
+
+ public static final Log LOG =
+ LogFactory.getLog(TestRpcServerHandoff.class);
+
+ private static final String BIND_ADDRESS = "0.0.0.0";
+ private static final Configuration conf = new Configuration();
+
+
+ public static class ServerForHandoffTest extends Server {
+
+ private final AtomicBoolean invoked = new AtomicBoolean(false);
+ private final ReentrantLock lock = new ReentrantLock();
+ private final Condition invokedCondition = lock.newCondition();
+
+ private volatile Writable request;
+ private volatile Call deferredCall;
+
+ protected ServerForHandoffTest(int handlerCount) throws IOException {
+ super(BIND_ADDRESS, 0, BytesWritable.class, handlerCount, conf);
+ }
+
+ @Override
+ public Writable call(RPC.RpcKind rpcKind, String protocol, Writable param,
+ long receiveTime) throws Exception {
+ request = param;
+ deferredCall = Server.getCurCall().get();
+ Server.getCurCall().get().deferResponse();
+ lock.lock();
+ try {
+ invoked.set(true);
+ invokedCondition.signal();
+ } finally {
+ lock.unlock();
+ }
+ return null;
+ }
+
+ void awaitInvocation() throws InterruptedException {
+ lock.lock();
+ try {
+ while (!invoked.get()) {
+ invokedCondition.await();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ void sendResponse() {
+ deferredCall.setDeferredResponse(request);
+ }
+
+ void sendError() {
+ deferredCall.setDeferredError(new IOException("DeferredError"));
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testDeferredResponse() throws IOException, InterruptedException,
+ ExecutionException {
+
+
+ ServerForHandoffTest server = new ServerForHandoffTest(2);
+ server.start();
+ try {
+ InetSocketAddress serverAddress = NetUtils.getConnectAddress(server);
+ byte[] requestBytes = generateRandomBytes(1024);
+ ClientCallable clientCallable =
+ new ClientCallable(serverAddress, conf, requestBytes);
+
+ FutureTask<Writable> future = new FutureTask<Writable>(clientCallable);
+ Thread clientThread = new Thread(future);
+ clientThread.start();
+
+ server.awaitInvocation();
+ awaitResponseTimeout(future);
+
+ server.sendResponse();
+ BytesWritable response = (BytesWritable) future.get();
+
+ Assert.assertEquals(new BytesWritable(requestBytes), response);
+ } finally {
+ if (server != null) {
+ server.stop();
+ }
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testDeferredException() throws IOException, InterruptedException,
+ ExecutionException {
+ ServerForHandoffTest server = new ServerForHandoffTest(2);
+ server.start();
+ try {
+ InetSocketAddress serverAddress = NetUtils.getConnectAddress(server);
+ byte[] requestBytes = generateRandomBytes(1024);
+ ClientCallable clientCallable =
+ new ClientCallable(serverAddress, conf, requestBytes);
+
+ FutureTask<Writable> future = new FutureTask<Writable>(clientCallable);
+ Thread clientThread = new Thread(future);
+ clientThread.start();
+
+ server.awaitInvocation();
+ awaitResponseTimeout(future);
+
+ server.sendError();
+ try {
+ future.get();
+ Assert.fail("Call succeeded. Was expecting an exception");
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ Assert.assertTrue(cause instanceof RemoteException);
+ RemoteException re = (RemoteException) cause;
+ Assert.assertTrue(re.toString().contains("DeferredError"));
+ }
+ } finally {
+ if (server != null) {
+ server.stop();
+ }
+ }
+ }
+
+ private void awaitResponseTimeout(FutureTask<Writable> future) throws
+ ExecutionException,
+ InterruptedException {
+ long sleepTime = 3000L;
+ while (sleepTime > 0) {
+ try {
+ future.get(200L, TimeUnit.MILLISECONDS);
+ Assert.fail("Expected to timeout since" +
+ " the deferred response hasn't been registered");
+ } catch (TimeoutException e) {
+ // Ignoring. Expected to time out.
+ }
+ sleepTime -= 200L;
+ }
+ LOG.info("Done sleeping");
+ }
+
+ private static class ClientCallable implements Callable<Writable> {
+
+ private final InetSocketAddress address;
+ private final Configuration conf;
+ final byte[] requestBytes;
+
+
+ private ClientCallable(InetSocketAddress address, Configuration conf,
+ byte[] requestBytes) {
+ this.address = address;
+ this.conf = conf;
+ this.requestBytes = requestBytes;
+ }
+
+ @Override
+ public Writable call() throws Exception {
+ Client client = new Client(BytesWritable.class, conf);
+ Writable param = new BytesWritable(requestBytes);
+ final Client.ConnectionId remoteId =
+ Client.ConnectionId.getConnectionId(address, null,
+ null, 0, null, conf);
+ Writable result = client.call(RPC.RpcKind.RPC_BUILTIN, param, remoteId,
+ new AtomicBoolean(false));
+ return result;
+ }
+ }
+
+ private byte[] generateRandomBytes(int length) {
+ Random random = new Random();
+ byte[] bytes = new byte[length];
+ for (int i = 0; i < length; i++) {
+ bytes[i] = (byte) ('a' + random.nextInt(26));
+ }
+ return bytes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d94da1e/hadoop-common-project/hadoop-common/src/test/proto/test.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test.proto b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
index 6411f97..37e9a0b 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
@@ -90,4 +90,13 @@ message AuthMethodResponseProto {
message UserResponseProto {
required string user = 1;
-}
\ No newline at end of file
+}
+
+message SleepRequestProto2 {
+ optional int64 sleep_time = 1;
+}
+
+message SleepResponseProto2 {
+ optional int64 receive_time = 1;
+ optional int64 response_time = 2;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d94da1e/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
index 32eafeb..3746411 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
@@ -70,4 +70,8 @@ service NewerProtobufRpcProto {
service CustomProto {
rpc ping(EmptyRequestProto) returns (EmptyResponseProto);
-}
\ No newline at end of file
+}
+
+service TestProtobufRpcHandoffProto {
+ rpc sleep(SleepRequestProto2) returns (SleepResponseProto2);
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/48] hadoop git commit: HDFS-10994. Support an XOR policy
XOR-2-1-64k in HDFS. Contributed by Sammi Chen
Posted by li...@apache.org.
HDFS-10994. Support an XOR policy XOR-2-1-64k in HDFS. Contributed by Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d5614f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d5614f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d5614f8
Branch: refs/heads/HADOOP-13345
Commit: 5d5614f847b2ef2a5b70bd9a06edc4eba06174c6
Parents: 209e805
Author: Kai Zheng <ka...@intel.com>
Authored: Mon Nov 28 14:34:44 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Mon Nov 28 14:34:44 2016 +0800
----------------------------------------------------------------------
.../io/erasurecode/ErasureCodeConstants.java | 3 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 1 +
.../namenode/ErasureCodingPolicyManager.java | 23 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 8 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 28 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 50 ++-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 27 +-
.../TestDFSStripedOutputStreamWithFailure.java | 37 +-
.../hdfs/TestDFSXORStripedInputStream.java | 33 ++
.../hdfs/TestDFSXORStripedOutputStream.java | 35 ++
...estDFSXORStripedOutputStreamWithFailure.java | 36 ++
...tyPreemptionPolicyForReservedContainers.java | 430 -------------------
12 files changed, 240 insertions(+), 471 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index 8d6ff85..ffa0bce 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -38,4 +38,7 @@ public final class ErasureCodeConstants {
public static final ECSchema RS_6_3_LEGACY_SCHEMA = new ECSchema(
RS_LEGACY_CODEC_NAME, 6, 3);
+
+ public static final ECSchema XOR_2_1_SCHEMA = new ECSchema(
+ XOR_CODEC_NAME, 2, 1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index acbc8f6..b55b4df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -147,6 +147,7 @@ public final class HdfsConstants {
public static final byte RS_6_3_POLICY_ID = 0;
public static final byte RS_3_2_POLICY_ID = 1;
public static final byte RS_6_3_LEGACY_POLICY_ID = 2;
+ public static final byte XOR_2_1_POLICY_ID = 3;
/* Hidden constructor */
protected HdfsConstants() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index c4bc8de..8a85d23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -36,7 +36,7 @@ import java.util.TreeMap;
public final class ErasureCodingPolicyManager {
/**
- * TODO: HDFS-8095
+ * TODO: HDFS-8095.
*/
private static final int DEFAULT_CELLSIZE = 64 * 1024;
private static final ErasureCodingPolicy SYS_POLICY1 =
@@ -48,10 +48,14 @@ public final class ErasureCodingPolicyManager {
private static final ErasureCodingPolicy SYS_POLICY3 =
new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_LEGACY_SCHEMA,
DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_LEGACY_POLICY_ID);
+ private static final ErasureCodingPolicy SYS_POLICY4 =
+ new ErasureCodingPolicy(ErasureCodeConstants.XOR_2_1_SCHEMA,
+ DEFAULT_CELLSIZE, HdfsConstants.XOR_2_1_POLICY_ID);
//We may add more later.
private static final ErasureCodingPolicy[] SYS_POLICIES =
- new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2, SYS_POLICY3};
+ new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2, SYS_POLICY3,
+ SYS_POLICY4};
// Supported storage policies for striped EC files
private static final byte[] SUITABLE_STORAGE_POLICIES_FOR_EC_STRIPED_MODE = new byte[] {
@@ -97,6 +101,19 @@ public final class ErasureCodingPolicyManager {
}
/**
+ * Get system-wide policy by policy ID.
+ * @return ecPolicy
+ */
+ public static ErasureCodingPolicy getPolicyByPolicyID(byte id) {
+ for (ErasureCodingPolicy policy : SYS_POLICIES) {
+ if (policy.getId() == id) {
+ return policy;
+ }
+ }
+ return null;
+ }
+
+ /**
* Get all policies that's available to use.
* @return all policies
*/
@@ -141,7 +158,7 @@ public final class ErasureCodingPolicyManager {
}
/**
- * Clear and clean up
+ * Clear and clean up.
*/
public void clear() {
activePoliciesByName.clear();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 37f97db..a5dcee9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -455,9 +455,13 @@ public class INodeFile extends INodeWithAdditionalFields
if(!isStriped()){
return max;
}
- // TODO support more policies based on policyId
+
ErasureCodingPolicy ecPolicy =
- ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ ErasureCodingPolicyManager.getPolicyByPolicyID(
+ getErasureCodingPolicyID());
+ if (ecPolicy == null){
+ ecPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ }
return (short) (ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 13e2656..1fbc1d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1888,21 +1888,41 @@ public class DFSTestUtil {
* Creates the metadata of a file in striped layout. This method only
* manipulates the NameNode state without injecting data to DataNode.
* You should disable periodical heartbeat before use this.
- * @param file Path of the file to create
+ * @param file Path of the file to create
* @param dir Parent path of the file
* @param numBlocks Number of striped block groups to add to the file
* @param numStripesPerBlk Number of striped cells in each block
* @param toMkdir
*/
- public static void createStripedFile(MiniDFSCluster cluster, Path file, Path dir,
- int numBlocks, int numStripesPerBlk, boolean toMkdir) throws Exception {
+ public static void createStripedFile(MiniDFSCluster cluster, Path file,
+ Path dir, int numBlocks, int numStripesPerBlk, boolean toMkdir)
+ throws Exception {
+ createStripedFile(cluster, file, dir, numBlocks, numStripesPerBlk,
+ toMkdir, null);
+ }
+
+ /**
+ * Creates the metadata of a file in striped layout. This method only
+ * manipulates the NameNode state without injecting data to DataNode.
+ * You should disable periodical heartbeat before use this.
+ * @param file Path of the file to create
+ * @param dir Parent path of the file
+ * @param numBlocks Number of striped block groups to add to the file
+ * @param numStripesPerBlk Number of striped cells in each block
+ * @param toMkdir
+ * @param ecPolicy erasure coding policy apply to created file. A null value
+ * means using default erasure coding policy.
+ */
+ public static void createStripedFile(MiniDFSCluster cluster, Path file,
+ Path dir, int numBlocks, int numStripesPerBlk, boolean toMkdir,
+ ErasureCodingPolicy ecPolicy) throws Exception {
DistributedFileSystem dfs = cluster.getFileSystem();
// If outer test already set EC policy, dir should be left as null
if (toMkdir) {
assert dir != null;
dfs.mkdirs(dir);
try {
- dfs.getClient().setErasureCodingPolicy(dir.toString(), null);
+ dfs.getClient().setErasureCodingPolicy(dir.toString(), ecPolicy);
} catch (IOException e) {
if (!e.getMessage().contains("non-empty directory")) {
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 3b46c66..121b9a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -64,20 +64,34 @@ public class TestDFSStripedInputStream {
private DistributedFileSystem fs;
private final Path dirPath = new Path("/striped");
private Path filePath = new Path(dirPath, "file");
- private final ErasureCodingPolicy ecPolicy =
- ErasureCodingPolicyManager.getSystemDefaultPolicy();
- private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
- private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
- private final int cellSize = ecPolicy.getCellSize();
+ private ErasureCodingPolicy ecPolicy;
+ private short dataBlocks;
+ private short parityBlocks;
+ private int cellSize;
private final int stripesPerBlock = 2;
- private final int blockSize = stripesPerBlock * cellSize;
- private final int blockGroupSize = dataBlocks * blockSize;
+ private int blockSize;
+ private int blockGroupSize;
@Rule
public Timeout globalTimeout = new Timeout(300000);
+ public ErasureCodingPolicy getEcPolicy() {
+ return ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ }
+
@Before
public void setup() throws IOException {
+ /*
+ * Initialize erasure coding policy.
+ */
+ ecPolicy = getEcPolicy();
+ dataBlocks = (short) ecPolicy.getNumDataUnits();
+ parityBlocks = (short) ecPolicy.getNumParityUnits();
+ cellSize = ecPolicy.getCellSize();
+ blockSize = stripesPerBlock * cellSize;
+ blockGroupSize = dataBlocks * blockSize;
+ System.out.println("EC policy = " + ecPolicy);
+
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
@@ -94,7 +108,7 @@ public class TestDFSStripedInputStream {
}
fs = cluster.getFileSystem();
fs.mkdirs(dirPath);
- fs.getClient().setErasureCodingPolicy(dirPath.toString(), null);
+ fs.getClient().setErasureCodingPolicy(dirPath.toString(), ecPolicy);
}
@After
@@ -106,13 +120,13 @@ public class TestDFSStripedInputStream {
}
/**
- * Test {@link DFSStripedInputStream#getBlockAt(long)}
+ * Test {@link DFSStripedInputStream#getBlockAt(long)}.
*/
@Test
public void testRefreshBlock() throws Exception {
final int numBlocks = 4;
DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
- stripesPerBlock, false);
+ stripesPerBlock, false, ecPolicy);
LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(
filePath.toString(), 0, blockGroupSize * numBlocks);
final DFSStripedInputStream in = new DFSStripedInputStream(fs.getClient(),
@@ -136,7 +150,7 @@ public class TestDFSStripedInputStream {
public void testPread() throws Exception {
final int numBlocks = 2;
DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
- stripesPerBlock, false);
+ stripesPerBlock, false, ecPolicy);
LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(
filePath.toString(), 0, blockGroupSize * numBlocks);
int fileLen = blockGroupSize * numBlocks;
@@ -154,7 +168,9 @@ public class TestDFSStripedInputStream {
bg.getBlock().getBlockPoolId());
}
- /** A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks */
+ /**
+ * A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks
+ */
for (int i = 0; i < stripesPerBlock; i++) {
for (int j = 0; j < dataBlocks; j++) {
for (int k = 0; k < cellSize; k++) {
@@ -194,7 +210,7 @@ public class TestDFSStripedInputStream {
final int numBlocks = 4;
final int failedDNIdx = dataBlocks - 1;
DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
- stripesPerBlock, false);
+ stripesPerBlock, false, ecPolicy);
LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(
filePath.toString(), 0, blockGroupSize);
@@ -305,7 +321,7 @@ public class TestDFSStripedInputStream {
setup();
}
DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
- stripesPerBlock, false);
+ stripesPerBlock, false, ecPolicy);
LocatedBlocks lbs = fs.getClient().namenode.
getBlockLocations(filePath.toString(), 0, fileSize);
@@ -330,7 +346,9 @@ public class TestDFSStripedInputStream {
byte[] expected = new byte[fileSize];
for (LocatedBlock bg : lbs.getLocatedBlocks()) {
- /** A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks */
+ /**
+ * A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks
+ */
for (int i = 0; i < stripesPerBlock; i++) {
for (int j = 0; j < dataBlocks; j++) {
for (int k = 0; k < cellSize; k++) {
@@ -371,7 +389,7 @@ public class TestDFSStripedInputStream {
final int numBlocks = 4;
final int failedDNIdx = dataBlocks - 1;
DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
- stripesPerBlock, false);
+ stripesPerBlock, false, ecPolicy);
LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(
filePath.toString(), 0, blockGroupSize);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index b686f28..5bde16e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -47,23 +47,36 @@ public class TestDFSStripedOutputStream {
GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.ALL);
}
- private final ErasureCodingPolicy ecPolicy =
- ErasureCodingPolicyManager.getSystemDefaultPolicy();
- private final int dataBlocks = ecPolicy.getNumDataUnits();
- private final int parityBlocks = ecPolicy.getNumParityUnits();
+ private ErasureCodingPolicy ecPolicy;
+ private int dataBlocks;
+ private int parityBlocks;
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private Configuration conf;
- private final int cellSize = ecPolicy.getCellSize();
+ private int cellSize;
private final int stripesPerBlock = 4;
- private final int blockSize = cellSize * stripesPerBlock;
+ private int blockSize;
@Rule
public Timeout globalTimeout = new Timeout(300000);
+ public ErasureCodingPolicy getEcPolicy() {
+ return ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ }
+
@Before
public void setup() throws IOException {
+ /*
+ * Initialize erasure coding policy.
+ */
+ ecPolicy = getEcPolicy();
+ dataBlocks = (short) ecPolicy.getNumDataUnits();
+ parityBlocks = (short) ecPolicy.getNumParityUnits();
+ cellSize = ecPolicy.getCellSize();
+ blockSize = stripesPerBlock * cellSize;
+ System.out.println("EC policy = " + ecPolicy);
+
int numDNs = dataBlocks + parityBlocks + 2;
conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
@@ -76,7 +89,7 @@ public class TestDFSStripedOutputStream {
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy);
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index cde07a4..0baf9cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Assume;
+import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
@@ -76,18 +77,36 @@ public class TestDFSStripedOutputStreamWithFailure {
.getLogger().setLevel(Level.ALL);
}
- private final ErasureCodingPolicy ecPolicy =
- ErasureCodingPolicyManager.getSystemDefaultPolicy();
- private final int dataBlocks = ecPolicy.getNumDataUnits();
- private final int parityBlocks = ecPolicy.getNumParityUnits();
- private final int cellSize = ecPolicy.getCellSize();
+ private ErasureCodingPolicy ecPolicy;
+ private int dataBlocks;
+ private int parityBlocks;
+ private int cellSize;
private final int stripesPerBlock = 4;
- private final int blockSize = cellSize * stripesPerBlock;
- private final int blockGroupSize = blockSize * dataBlocks;
+ private int blockSize;
+ private int blockGroupSize;
private static final int FLUSH_POS =
9 * DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT + 1;
+ public ErasureCodingPolicy getEcPolicy() {
+ return ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ }
+
+ /*
+ * Initialize erasure coding policy.
+ */
+ @Before
+ public void init(){
+ ecPolicy = getEcPolicy();
+ dataBlocks = ecPolicy.getNumDataUnits();
+ parityBlocks = ecPolicy.getNumParityUnits();
+ cellSize = ecPolicy.getCellSize();
+ blockSize = cellSize * stripesPerBlock;
+ blockGroupSize = blockSize * dataBlocks;
+ dnIndexSuite = getDnIndexSuite();
+ lengths = newLengths();
+ }
+
List<Integer> newLengths() {
final List<Integer> lens = new ArrayList<>();
lens.add(FLUSH_POS + 2);
@@ -104,7 +123,7 @@ public class TestDFSStripedOutputStreamWithFailure {
return lens;
}
- private final int[][] dnIndexSuite = getDnIndexSuite();
+ private int[][] dnIndexSuite;
private int[][] getDnIndexSuite() {
final int maxNumLevel = 2;
@@ -167,7 +186,7 @@ public class TestDFSStripedOutputStreamWithFailure {
return positions;
}
- private final List<Integer> lengths = newLengths();
+ private List<Integer> lengths;
Integer getLength(int i) {
return i >= 0 && i < lengths.size()? lengths.get(i): null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java
new file mode 100644
index 0000000..75062e0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
+
+/**
+ * This tests read operation of DFS striped file with XOR-2-1-64k erasure code
+ * policy.
+ */
+public class TestDFSXORStripedInputStream extends TestDFSStripedInputStream{
+
+ public ErasureCodingPolicy getEcPolicy() {
+ return ErasureCodingPolicyManager.getPolicyByPolicyID(
+ HdfsConstants.XOR_2_1_POLICY_ID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java
new file mode 100644
index 0000000..64bddb8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
+
+/**
+ * This tests write operation of DFS striped file with XOR-2-1-64k erasure code
+ * policy.
+ */
+public class TestDFSXORStripedOutputStream extends TestDFSStripedOutputStream{
+
+ @Override
+ public ErasureCodingPolicy getEcPolicy() {
+ return ErasureCodingPolicyManager.getPolicyByPolicyID(
+ HdfsConstants.XOR_2_1_POLICY_ID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java
new file mode 100644
index 0000000..ed361a8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
+
+/**
+ * This tests write operation of DFS striped file with XOR-2-1-64k erasure code
+ * policy when there is data node failure.
+ */
+public class TestDFSXORStripedOutputStreamWithFailure
+ extends TestDFSStripedOutputStreamWithFailure{
+
+ @Override
+ public ErasureCodingPolicy getEcPolicy() {
+ return ErasureCodingPolicyManager.getPolicyByPolicyID(
+ HdfsConstants.XOR_2_1_POLICY_ID);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d5614f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForReservedContainers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForReservedContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForReservedContainers.java
deleted file mode 100644
index 38b2e78..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForReservedContainers.java
+++ /dev/null
@@ -1,430 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
-
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-public class TestProportionalCapacityPreemptionPolicyForReservedContainers
- extends ProportionalCapacityPreemptionPolicyMockFramework {
- @Before
- public void setup() {
- super.setup();
- conf.setBoolean(
- CapacitySchedulerConfiguration.PREEMPTION_SELECT_CANDIDATES_FOR_RESERVED_CONTAINERS,
- true);
- policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock);
- }
-
- @Test
- public void testPreemptionForSimpleReservedContainer() throws IOException {
- /**
- * The simplest test of reserved container, Queue structure is:
- *
- * <pre>
- * root
- * / \
- * a b
- * </pre>
- * Guaranteed resource of a/b are 50:50
- * Total cluster resource = 100
- * - A has 90 containers on two node, n1 has 45, n2 has 45, size of each
- * container is 1.
- * - B has am container at n1, and reserves 1 container with size = 9 at n1,
- * so B needs to preempt 9 containers from A at n1 instead of randomly
- * preempt from n1 and n2.
- */
- String labelsConfig =
- "=100,true;";
- String nodesConfig = // n1 / n2 has no label
- "n1= res=50;" +
- "n2= res=50";
- String queuesConfig =
- // guaranteed,max,used,pending,reserved
- "root(=[100 100 100 9 9]);" + //root
- "-a(=[50 100 90 0]);" + // a
- "-b(=[50 100 10 9 9])"; // b
- String appsConfig=
- //queueName\t(priority,resource,host,expression,#repeat,reserved)
- "a\t" // app1 in a
- + "(1,1,n1,,45,false)" // 45 in n1
- + "(1,1,n2,,45,false);" + // 45 in n2
- "b\t" // app2 in b
- + "(1,1,n1,,1,false)" // AM container in n1
- + "(1,9,n1,,1,true)"; // 1 container with size=9 reserved at n1
-
- buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
- policy.editSchedule();
-
- // Total 5 preempted from app1 at n1, don't preempt container from other
- // app/node
- verify(mDisp, times(5)).handle(argThat(
- new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
- getAppAttemptId(1))));
- verify(mDisp, times(5)).handle(
- argThat(new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n1", 1))));
- verify(mDisp, times(0)).handle(argThat(
- new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
- getAppAttemptId(2))));
- }
-
- @Test
- public void testUseReservedAndFifoSelectorTogether() throws IOException {
- /**
- * Queue structure is:
- *
- * <pre>
- * root
- * / \
- * a b
- * </pre>
- * Guaranteed resource of a/b are 30:70
- * Total cluster resource = 100
- * - A has 45 containers on two node, n1 has 10, n2 has 35, size of each
- * container is 1.
- * - B has 5 containers at n2, and reserves 1 container with size = 50 at n1,
- * B also has 20 pending resources.
- * so B needs to preempt:
- * - 10 containers from n1 (for reserved)
- * - 5 containers from n2 for pending resources
- */
- String labelsConfig =
- "=100,true;";
- String nodesConfig = // n1 / n2 has no label
- "n1= res=50;" +
- "n2= res=50";
- String queuesConfig =
- // guaranteed,max,used,pending,reserved
- "root(=[100 100 100 70 10]);" + //root
- "-a(=[30 100 45 0]);" + // a
- "-b(=[70 100 55 70 50])"; // b
- String appsConfig=
- //queueName\t(priority,resource,host,expression,#repeat,reserved)
- "a\t" // app1 in a
- + "(1,1,n2,,35,false)" // 35 in n2
- + "(1,1,n1,,10,false);" + // 10 in n1
- "b\t" // app2 in b
- + "(1,1,n2,,5,false)" // 5 in n2
- + "(1,50,n1,,1,true)"; // 1 container with size=50 reserved at n1
-
- buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
- policy.editSchedule();
-
- verify(mDisp, times(15)).handle(argThat(
- new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
- getAppAttemptId(1))));
- verify(mDisp, times(10)).handle(
- argThat(new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n1", 1))));
- verify(mDisp, times(5)).handle(
- argThat(new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n2", 1))));
- verify(mDisp, times(0)).handle(argThat(
- new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
- getAppAttemptId(2))));
- }
-
- @Test
- public void testReservedSelectorSkipsAMContainer() throws IOException {
- /**
- * Queue structure is:
- *
- * <pre>
- * root
- * / \
- * a b
- * </pre>
- * Guaranteed resource of a/b are 30:70
- * Total cluster resource = 100
- * - A has 45 containers on two node, n1 has 10, n2 has 35, size of each
- * container is 1.
- * - B has 5 containers at n2, and reserves 1 container with size = 50 at n1,
- * B also has 20 pending resources.
- *
- * Ideally B needs to preempt:
- * - 10 containers from n1 (for reserved)
- * - 5 containers from n2 for pending resources
- *
- * However, since one AM container is located at n1 (from queueA), we cannot
- * preempt 10 containers from n1 for reserved container. Instead, we will
- * preempt 15 containers from n2, since containers from queueA launched in n2
- * are later than containers from queueA launched in n1 (FIFO order of containers)
- */
- String labelsConfig =
- "=100,true;";
- String nodesConfig = // n1 / n2 has no label
- "n1= res=50;" +
- "n2= res=50";
- String queuesConfig =
- // guaranteed,max,used,pending,reserved
- "root(=[100 100 100 70 10]);" + //root
- "-a(=[30 100 45 0]);" + // a
- "-b(=[70 100 55 70 50])"; // b
- String appsConfig=
- //queueName\t(priority,resource,host,expression,#repeat,reserved)
- "a\t" // app1 in a
- + "(1,1,n1,,10,false)" // 10 in n1
- + "(1,1,n2,,35,false);" +// 35 in n2
- "b\t" // app2 in b
- + "(1,1,n2,,5,false)" // 5 in n2
- + "(1,50,n1,,1,true)"; // 1 container with size=50 reserved at n1
-
- buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
- policy.editSchedule();
-
- verify(mDisp, times(15)).handle(argThat(
- new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
- getAppAttemptId(1))));
- verify(mDisp, times(0)).handle(
- argThat(new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n1", 1))));
- verify(mDisp, times(15)).handle(
- argThat(new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n2", 1))));
- verify(mDisp, times(0)).handle(argThat(
- new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
- getAppAttemptId(2))));
- }
-
- @Test
- public void testPreemptionForReservedContainerRespectGuaranteedResource()
- throws IOException {
- /**
- * Queue structure is:
- *
- * <pre>
- * root
- * / \
- * a b
- * </pre>
- * Guaranteed resource of a/b are 85:15
- * Total cluster resource = 100
- * - A has 90 containers on two node, n1 has 45, n2 has 45, size of each
- * container is 1.
- * - B has am container at n1, and reserves 1 container with size = 9 at n1,
- *
- * If we preempt 9 containers from queue-A, queue-A will be below its
- * guaranteed resource = 90 - 9 = 81 < 85.
- *
- * So no preemption will take place
- */
- String labelsConfig =
- "=100,true;";
- String nodesConfig = // n1 / n2 has no label
- "n1= res=50;" +
- "n2= res=50";
- String queuesConfig =
- // guaranteed,max,used,pending,reserved
- "root(=[100 100 100 9 9]);" + //root
- "-a(=[85 100 90 0]);" + // a
- "-b(=[15 100 10 9 9])"; // b
- String appsConfig=
- //queueName\t(priority,resource,host,expression,#repeat,reserved)
- "a\t" // app1 in a
- + "(1,1,n1,,45,false)" // 45 in n1
- + "(1,1,n2,,45,false);" + // 45 in n2
- "b\t" // app2 in b
- + "(1,1,n1,,1,false)" // AM container in n1
- + "(1,9,n1,,1,true)"; // 1 container with size=9 reserved at n1
-
- buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
- policy.editSchedule();
-
- verify(mDisp, times(0)).handle(argThat(
- new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
- getAppAttemptId(1))));
- verify(mDisp, times(0)).handle(argThat(
- new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
- getAppAttemptId(2))));
- }
-
- @Test
- public void testPreemptionForReservedContainerWhichHasAvailableResource()
- throws IOException {
- /**
- * Queue structure is:
- *
- * <pre>
- * root
- * / \
- * a b
- * </pre>
- *
- * Guaranteed resource of a/b are 50:50
- * Total cluster resource = 100
- * - A has 90 containers on two node, n1 has 45, n2 has 45, size of each
- * container is 1.
- * - B has am container at n1, and reserves 1 container with size = 9 at n1,
- *
- * So we can get 4 containers preempted after preemption.
- * (reserved 5 + preempted 4) = 9
- */
- String labelsConfig =
- "=100,true;";
- String nodesConfig = // n1 / n2 has no label
- "n1= res=50;" +
- "n2= res=50";
- String queuesConfig =
- // guaranteed,max,used,pending,reserved
- "root(=[100 100 99 9 9]);" + //root
- "-a(=[50 100 90 0]);" + // a
- "-b(=[50 100 9 9 9])"; // b
- String appsConfig=
- //queueName\t(priority,resource,host,expression,#repeat,reserved)
- "a\t" // app1 in a
- + "(1,1,n1,,45,false)" // 45 in n1
- + "(1,1,n2,,45,false);" + // 45 in n2
- "b\t" // app2 in b
- + "(1,9,n1,,1,true)"; // 1 container with size=9 reserved at n1
-
- buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
- policy.editSchedule();
-
- // Total 4 preempted from app1 at n1, don't preempt container from other
- // app/node
- verify(mDisp, times(4)).handle(argThat(
- new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n1", 1))));
- verify(mDisp, times(0)).handle(argThat(
- new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n2", 1))));
- }
-
- @Test
- public void testPreemptionForReservedContainerWhichHasNondivisibleAvailableResource()
- throws IOException {
- /**
- * Queue structure is:
- *
- * <pre>
- * root
- * / \
- * a b
- * </pre>
- *
- * Guaranteed resource of a/b are 50:50
- * Total cluster resource = 100
- * - A has 45 containers on two node, size of each container is 2,
- * n1 has 23, n2 has 22
- * - B reserves 1 container with size = 9 at n1,
- *
- * So we can get 4 containers (total-resource = 8) preempted after
- * preemption. Actual required is 3.5, but we need to preempt integer
- * number of containers
- */
- String labelsConfig =
- "=100,true;";
- String nodesConfig = // n1 / n2 has no label
- "n1= res=50;" +
- "n2= res=50";
- String queuesConfig =
- // guaranteed,max,used,pending,reserved
- "root(=[100 100 99 9 9]);" + //root
- "-a(=[50 100 90 0]);" + // a
- "-b(=[50 100 9 9 9])"; // b
- String appsConfig=
- //queueName\t(priority,resource,host,expression,#repeat,reserved)
- "a\t" // app1 in a
- + "(1,2,n1,,24,false)" // 48 in n1
- + "(1,2,n2,,23,false);" + // 46 in n2
- "b\t" // app2 in b
- + "(1,9,n1,,1,true)"; // 1 container with size=9 reserved at n1
-
- buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
- policy.editSchedule();
-
- // Total 4 preempted from app1 at n1, don't preempt container from other
- // app/node
- verify(mDisp, times(4)).handle(argThat(
- new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n1", 1))));
- verify(mDisp, times(0)).handle(argThat(
- new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n2", 1))));
- }
-
- @Test
- public void testPreemptionForReservedContainerRespectAvailableResources()
- throws IOException {
- /**
- * Queue structure is:
- *
- * <pre>
- * root
- * / \
- * a b
- * </pre>
- *
- * Guaranteed resource of a/b are 50:50
- * Total cluster resource = 100, 4 nodes, 25 on each node
- * - A has 10 containers on every node, size of container is 2
- * - B reserves 1 container with size = 9 at n1,
- *
- * So even if we cannot allocate container for B now, no preemption should
- * happen since there're plenty of available resources.
- */
- String labelsConfig =
- "=100,true;";
- String nodesConfig =
- "n1= res=25;" +
- "n2= res=25;" +
- "n3= res=25;" +
- "n4= res=25;";
- String queuesConfig =
- // guaranteed,max,used,pending,reserved
- "root(=[100 100 89 9 9]);" + //root
- "-a(=[50 100 80 0]);" + // a
- "-b(=[50 100 9 9 9])"; // b
- String appsConfig=
- //queueName\t(priority,resource,host,expression,#repeat,reserved)
- "a\t" // app1 in a
- + "(1,2,n1,,10,false)" // 10 in n1
- + "(1,2,n2,,10,false)" // 10 in n2
- + "(1,2,n3,,10,false)" // 10 in n3
- + "(1,2,n4,,10,false);" + // 10 in n4
- "b\t" // app2 in b
- + "(1,9,n1,,1,true)"; // 1 container with size=5 reserved at n1
-
- buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
- policy.editSchedule();
-
- // No preemption should happen
- verify(mDisp, times(0)).handle(argThat(
- new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n1", 1))));
- verify(mDisp, times(0)).handle(argThat(
- new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n2", 1))));
- verify(mDisp, times(0)).handle(argThat(
- new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n3", 1))));
- verify(mDisp, times(0)).handle(argThat(
- new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a",
- NodeId.newInstance("n4", 1))));
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/48] hadoop git commit: HADOOP-13838. KMSTokenRenewer should close
providers (xiaochen via rkanter)
Posted by li...@apache.org.
HADOOP-13838. KMSTokenRenewer should close providers (xiaochen via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47ca9e26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47ca9e26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47ca9e26
Branch: refs/heads/HADOOP-13345
Commit: 47ca9e26fba4a639e43bee5bfc001ffc4b42330d
Parents: d60a60b
Author: Robert Kanter <rk...@cloudera.com>
Authored: Mon Nov 28 18:08:09 2016 -0800
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Mon Nov 28 18:08:09 2016 -0800
----------------------------------------------------------------------
.../crypto/key/kms/KMSClientProvider.java | 41 ++++++---
.../hadoop/crypto/key/kms/server/TestKMS.java | 97 +++++++++++++++-----
.../hdfs/server/namenode/FSNamesystem.java | 7 ++
3 files changed, 110 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47ca9e26/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 2b6ae9e..f184ea7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -173,14 +173,20 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
LOG.debug("Renewing delegation token {}", token);
KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
KeyProviderFactory.KEY_PROVIDER_PATH);
- if (!(keyProvider instanceof
- KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
- LOG.warn("keyProvider {} cannot renew dt.", keyProvider == null ?
- "null" : keyProvider.getClass());
- return 0;
+ try {
+ if (!(keyProvider instanceof
+ KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+ LOG.warn("keyProvider {} cannot renew dt.", keyProvider == null ?
+ "null" : keyProvider.getClass());
+ return 0;
+ }
+ return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+ keyProvider).renewDelegationToken(token);
+ } finally {
+ if (keyProvider != null) {
+ keyProvider.close();
+ }
}
- return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
- keyProvider).renewDelegationToken(token);
}
@Override
@@ -188,14 +194,20 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
LOG.debug("Canceling delegation token {}", token);
KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
KeyProviderFactory.KEY_PROVIDER_PATH);
- if (!(keyProvider instanceof
- KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
- LOG.warn("keyProvider {} cannot cancel dt.", keyProvider == null ?
- "null" : keyProvider.getClass());
- return;
+ try {
+ if (!(keyProvider instanceof
+ KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+ LOG.warn("keyProvider {} cannot cancel dt.", keyProvider == null ?
+ "null" : keyProvider.getClass());
+ return;
+ }
+ ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+ keyProvider).cancelDelegationToken(token);
+ } finally {
+ if (keyProvider != null) {
+ keyProvider.close();
+ }
}
- ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
- keyProvider).cancelDelegationToken(token);
}
}
@@ -1072,6 +1084,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
} finally {
if (sslFactory != null) {
sslFactory.destroy();
+ sslFactory = null;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47ca9e26/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index dac91e0..aaab172 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
+import com.google.common.base.Supplier;
import org.apache.curator.test.TestingServer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
@@ -74,12 +75,16 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
public class TestKMS {
private static final Logger LOG = LoggerFactory.getLogger(TestKMS.class);
+ private static final String SSL_RELOADER_THREAD_NAME =
+ "Truststore reloader thread";
+
@Rule
public final Timeout testTimeout = new Timeout(180000);
@@ -348,7 +353,7 @@ public class TestKMS {
Thread reloaderThread = null;
for (Thread thread : threads) {
if ((thread.getName() != null)
- && (thread.getName().contains("Truststore reloader thread"))) {
+ && (thread.getName().contains(SSL_RELOADER_THREAD_NAME))) {
reloaderThread = thread;
}
}
@@ -378,6 +383,7 @@ public class TestKMS {
.addDelegationTokens("myuser", new Credentials());
Assert.assertEquals(1, tokens.length);
Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
+ kp.close();
return null;
}
});
@@ -393,6 +399,7 @@ public class TestKMS {
.addDelegationTokens("myuser", new Credentials());
Assert.assertEquals(1, tokens.length);
Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
+ kp.close();
}
return null;
}
@@ -1755,34 +1762,63 @@ public class TestKMS {
});
}
+ private Configuration setupConfForKerberos(File confDir) throws Exception {
+ final Configuration conf = createBaseKMSConf(confDir, null);
+ conf.set("hadoop.security.authentication", "kerberos");
+ conf.set("hadoop.kms.authentication.type", "kerberos");
+ conf.set("hadoop.kms.authentication.kerberos.keytab",
+ keytab.getAbsolutePath());
+ conf.set("hadoop.kms.authentication.kerberos.principal",
+ "HTTP/localhost");
+ conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
+ return conf;
+ }
+
@Test
- public void testDelegationTokensOpsSimple() throws Exception {
- final Configuration conf = new Configuration();
- testDelegationTokensOps(conf, false);
+ public void testDelegationTokensOpsHttpPseudo() throws Exception {
+ testDelegationTokensOps(false, false);
}
@Test
- public void testDelegationTokensOpsKerberized() throws Exception {
- final Configuration conf = new Configuration();
- conf.set("hadoop.security.authentication", "kerberos");
- testDelegationTokensOps(conf, true);
+ public void testDelegationTokensOpsHttpKerberized() throws Exception {
+ testDelegationTokensOps(false, true);
}
- private void testDelegationTokensOps(Configuration conf,
- final boolean useKrb) throws Exception {
- File confDir = getTestDir();
- conf = createBaseKMSConf(confDir, conf);
- if (useKrb) {
- conf.set("hadoop.kms.authentication.type", "kerberos");
- conf.set("hadoop.kms.authentication.kerberos.keytab",
- keytab.getAbsolutePath());
- conf.set("hadoop.kms.authentication.kerberos.principal",
- "HTTP/localhost");
- conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
+ @Test
+ public void testDelegationTokensOpsHttpsPseudo() throws Exception {
+ testDelegationTokensOps(true, false);
+ }
+
+ @Test
+ public void testDelegationTokensOpsHttpsKerberized() throws Exception {
+ testDelegationTokensOps(true, true);
+ }
+
+ private void testDelegationTokensOps(final boolean ssl, final boolean kerb)
+ throws Exception {
+ final File confDir = getTestDir();
+ final Configuration conf;
+ if (kerb) {
+ conf = setupConfForKerberos(confDir);
+ } else {
+ conf = createBaseKMSConf(confDir, null);
+ }
+
+ final String keystore;
+ final String password;
+ if (ssl) {
+ final String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestKMS.class);
+ KeyStoreTestUtil.setupSSLConfig(confDir.getAbsolutePath(), sslConfDir,
+ conf, false);
+ keystore = confDir.getAbsolutePath() + "/serverKS.jks";
+ password = "serverP";
+ } else {
+ keystore = null;
+ password = null;
}
writeConf(confDir, conf);
- runServer(null, null, confDir, new KMSCallable<Void>() {
+ runServer(keystore, password, confDir, new KMSCallable<Void>() {
@Override
public Void call() throws Exception {
final Configuration clientConf = new Configuration();
@@ -1830,7 +1866,7 @@ public class TestKMS {
}
final UserGroupInformation otherUgi;
- if (useKrb) {
+ if (kerb) {
UserGroupInformation
.loginUserFromKeytab("client1", keytab.getAbsolutePath());
otherUgi = UserGroupInformation.getLoginUser();
@@ -1885,6 +1921,9 @@ public class TestKMS {
return null;
}
});
+ // Close the client provider. We will verify all providers'
+ // Truststore reloader threads are closed later.
+ kp.close();
return null;
} finally {
otherUgi.logoutUserFromKeytab();
@@ -1894,6 +1933,22 @@ public class TestKMS {
return null;
}
});
+
+ // verify that providers created by KMSTokenRenewer are closed.
+ if (ssl) {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ final Set<Thread> threadSet = Thread.getAllStackTraces().keySet();
+ for (Thread t : threadSet) {
+ if (t.getName().contains(SSL_RELOADER_THREAD_NAME)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }, 1000, 10000);
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47ca9e26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 06a0564..c78ef46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4569,6 +4569,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (blockManager != null) {
blockManager.shutdown();
}
+ if (provider != null) {
+ try {
+ provider.close();
+ } catch (IOException e) {
+ LOG.error("Failed to close provider.", e);
+ }
+ }
}
@Override // FSNamesystemMBean
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/48] hadoop git commit: YARN-4752. Improved preemption in
FairScheduler. (kasha)
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 8e6272a..992b75d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -17,14 +17,6 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import org.junit.Assert;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -39,7 +31,9 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -50,9 +44,17 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptM
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
public class FairSchedulerTestBase {
public final static String TEST_DIR =
new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath();
@@ -70,9 +72,14 @@ public class FairSchedulerTestBase {
private static final int SLEEP_DURATION = 10;
private static final int SLEEP_RETRIES = 1000;
+ /**
+ * The list of nodes added to the cluster using the {@link #addNode} method.
+ */
+ protected final List<RMNode> rmNodes = new ArrayList<>();
+
// Helper methods
public Configuration createConfiguration() {
- Configuration conf = new YarnConfiguration();
+ conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
ResourceScheduler.class);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
@@ -280,4 +287,18 @@ public class FairSchedulerTestBase {
Assert.assertEquals(resource.getVirtualCores(),
app.getCurrentConsumption().getVirtualCores());
}
+
+ /**
+ * Add a node to the cluster and track the nodes in {@link #rmNodes}.
+ * @param memory memory capacity of the node
+ * @param cores cpu capacity of the node
+ */
+ protected void addNode(int memory, int cores) {
+ int id = rmNodes.size() + 1;
+ RMNode node =
+ MockNodes.newNodeInfo(1, Resources.createResource(memory, cores), id,
+ "127.0.0." + id);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+ rmNodes.add(node);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerWithMockPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerWithMockPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerWithMockPreemption.java
new file mode 100644
index 0000000..25780cd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerWithMockPreemption.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class FairSchedulerWithMockPreemption extends FairScheduler {
+ @Override
+ protected void createPreemptionThread() {
+ preemptionThread = new MockPreemptionThread(this);
+ }
+
+ static class MockPreemptionThread extends FSPreemptionThread {
+ private Set<FSAppAttempt> appsAdded = new HashSet<>();
+ private int totalAppsAdded = 0;
+
+ MockPreemptionThread(FairScheduler scheduler) {
+ super(scheduler);
+ }
+
+ @Override
+ public void run() {
+ while (!Thread.interrupted()) {
+ try {
+ FSAppAttempt app = context.getStarvedApps().take();
+ appsAdded.add(app);
+ totalAppsAdded++;
+ } catch (InterruptedException e) {
+ return;
+ }
+ }
+ }
+
+ int uniqueAppsAdded() {
+ return appsAdded.size();
+ }
+
+ int totalAppsAdded() {
+ return totalAppsAdded;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
index 5a170cf..e802f42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
@@ -86,11 +86,6 @@ public class FakeSchedulable implements Schedulable {
}
@Override
- public RMContainer preemptContainer() {
- return null;
- }
-
- @Override
public Resource getFairShare() {
return this.fairShare;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
new file mode 100644
index 0000000..a5b2d86
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+
+import org.junit.After;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+/**
+ * Test class to verify identification of app starvation
+ */
+public class TestFSAppStarvation extends FairSchedulerTestBase {
+
+ private static final File ALLOC_FILE = new File(TEST_DIR, "test-QUEUES");
+
+ // Node Capacity = NODE_CAPACITY_MULTIPLE * (1 GB or 1 vcore)
+ private static final int NODE_CAPACITY_MULTIPLE = 4;
+ private static final String[] QUEUES =
+ {"no-preemption", "minshare", "fairshare.child", "drf.child"};
+
+ private FairSchedulerWithMockPreemption.MockPreemptionThread preemptionThread;
+
+ @Before
+ public void setup() {
+ createConfiguration();
+ conf.set(YarnConfiguration.RM_SCHEDULER,
+ FairSchedulerWithMockPreemption.class.getCanonicalName());
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ ALLOC_FILE.getAbsolutePath());
+ conf.setBoolean(FairSchedulerConfiguration.PREEMPTION, true);
+ conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
+ }
+
+ @After
+ public void teardown() {
+ ALLOC_FILE.delete();
+ conf = null;
+ if (resourceManager != null) {
+ resourceManager.stop();
+ resourceManager = null;
+ }
+ }
+
+ /*
+ * Test to verify application starvation is computed only when preemption
+ * is enabled.
+ */
+ @Test
+ public void testPreemptionDisabled() throws Exception {
+ conf.setBoolean(FairSchedulerConfiguration.PREEMPTION, false);
+
+ setupClusterAndSubmitJobs();
+
+ assertNull("Found starved apps even when preemption is turned off",
+ scheduler.getContext().getStarvedApps());
+ }
+
+ /*
+ * Test to verify application starvation is computed correctly when
+ * preemption is turned on.
+ */
+ @Test
+ public void testPreemptionEnabled() throws Exception {
+ setupClusterAndSubmitJobs();
+
+ assertNotNull("FSContext does not have an FSStarvedApps instance",
+ scheduler.getContext().getStarvedApps());
+ assertEquals("Expecting 3 starved applications, one each for the "
+ + "minshare and fairshare queues",
+ 3, preemptionThread.uniqueAppsAdded());
+
+ // Verify the apps get added again on a subsequent update
+ scheduler.update();
+ Thread.yield();
+
+ verifyLeafQueueStarvation();
+ assertTrue("Each app is marked as starved exactly once",
+ preemptionThread.totalAppsAdded() > preemptionThread.uniqueAppsAdded());
+ }
+
+ /*
+ * Test to verify app starvation is computed only when the cluster
+ * utilization threshold is over the preemption threshold.
+ */
+ @Test
+ public void testClusterUtilizationThreshold() throws Exception {
+ // Set preemption threshold to 1.1, so the utilization is always lower
+ conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 1.1f);
+
+ setupClusterAndSubmitJobs();
+
+ assertNotNull("FSContext does not have an FSStarvedApps instance",
+ scheduler.getContext().getStarvedApps());
+ assertEquals("Found starved apps when preemption threshold is over 100%", 0,
+ preemptionThread.totalAppsAdded());
+ }
+
+ private void verifyLeafQueueStarvation() {
+ for (String q : QUEUES) {
+ if (!q.equals("no-preemption")) {
+ boolean isStarved =
+ scheduler.getQueueManager().getLeafQueue(q, false).isStarved();
+ assertTrue(isStarved);
+ }
+ }
+ }
+
+ private void setupClusterAndSubmitJobs() throws Exception {
+ setupStarvedCluster();
+ submitAppsToEachLeafQueue();
+ sendEnoughNodeUpdatesToAssignFully();
+
+ // Sleep to hit the preemption timeouts
+ Thread.sleep(10);
+
+ // Scheduler update to populate starved apps
+ scheduler.update();
+
+ // Wait for apps to be processed by MockPreemptionThread
+ Thread.yield();
+ }
+
+ /**
+ * Setup the cluster for starvation testing:
+ * 1. Create FS allocation file
+ * 2. Create and start MockRM
+ * 3. Add two nodes to the cluster
+ * 4. Submit an app that uses up all resources on the cluster
+ */
+ private void setupStarvedCluster() throws IOException {
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+
+ // Default queue
+ out.println("<queue name=\"default\">");
+ out.println("</queue>");
+
+ // Queue with preemption disabled
+ out.println("<queue name=\"no-preemption\">");
+ out.println("<fairSharePreemptionThreshold>0" +
+ "</fairSharePreemptionThreshold>");
+ out.println("</queue>");
+
+ // Queue with minshare preemption enabled
+ out.println("<queue name=\"minshare\">");
+ out.println("<fairSharePreemptionThreshold>0" +
+ "</fairSharePreemptionThreshold>");
+ out.println("<minSharePreemptionTimeout>0" +
+ "</minSharePreemptionTimeout>");
+ out.println("<minResources>2048mb,2vcores</minResources>");
+ out.println("</queue>");
+
+ // FAIR queue with fairshare preemption enabled
+ out.println("<queue name=\"fairshare\">");
+ out.println("<fairSharePreemptionThreshold>1" +
+ "</fairSharePreemptionThreshold>");
+ out.println("<fairSharePreemptionTimeout>0" +
+ "</fairSharePreemptionTimeout>");
+ out.println("<schedulingPolicy>fair</schedulingPolicy>");
+ addChildQueue(out);
+ out.println("</queue>");
+
+ // DRF queue with fairshare preemption enabled
+ out.println("<queue name=\"drf\">");
+ out.println("<fairSharePreemptionThreshold>1" +
+ "</fairSharePreemptionThreshold>");
+ out.println("<fairSharePreemptionTimeout>0" +
+ "</fairSharePreemptionTimeout>");
+ out.println("<schedulingPolicy>drf</schedulingPolicy>");
+ addChildQueue(out);
+ out.println("</queue>");
+
+ out.println("</allocations>");
+ out.close();
+
+ assertTrue("Allocation file does not exist, not running the test",
+ ALLOC_FILE.exists());
+
+ resourceManager = new MockRM(conf);
+ resourceManager.start();
+ scheduler = (FairScheduler) resourceManager.getResourceScheduler();
+ preemptionThread = (FairSchedulerWithMockPreemption.MockPreemptionThread)
+ scheduler.preemptionThread;
+
+ // Create and add two nodes to the cluster
+ addNode(NODE_CAPACITY_MULTIPLE * 1024, NODE_CAPACITY_MULTIPLE);
+ addNode(NODE_CAPACITY_MULTIPLE * 1024, NODE_CAPACITY_MULTIPLE);
+
+ // Create an app that takes up all the resources on the cluster
+ ApplicationAttemptId app
+ = createSchedulingRequest(1024, 1, "root.default", "default", 8);
+
+ scheduler.update();
+ sendEnoughNodeUpdatesToAssignFully();
+
+ assertEquals(8, scheduler.getSchedulerApp(app).getLiveContainers().size());
+ }
+
+ private void addChildQueue(PrintWriter out) {
+ // Child queue under fairshare with same settings
+ out.println("<queue name=\"child\">");
+ out.println("<fairSharePreemptionThreshold>1" +
+ "</fairSharePreemptionThreshold>");
+ out.println("<fairSharePreemptionTimeout>0" +
+ "</fairSharePreemptionTimeout>");
+ out.println("</queue>");
+ }
+
+ private void submitAppsToEachLeafQueue() {
+ for (String queue : QUEUES) {
+ createSchedulingRequest(1024, 1, "root." + queue, "user", 1);
+ }
+ scheduler.update();
+ }
+
+ private void sendEnoughNodeUpdatesToAssignFully() {
+ for (RMNode node : rmNodes) {
+ NodeUpdateSchedulerEvent nodeUpdateSchedulerEvent =
+ new NodeUpdateSchedulerEvent(node);
+ for (int i = 0; i < NODE_CAPACITY_MULTIPLE; i++) {
+ scheduler.handle(nodeUpdateSchedulerEvent);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10468529/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 0a2ce81..98de8db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -106,12 +105,8 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<minResources>2048mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<minResources>2048mb,0vcores</minResources>");
- out.println("</queue>");
+ out.println("<queue name=\"queueA\"></queue>");
+ out.println("<queue name=\"queueB\"></queue>");
out.println("</allocations>");
out.close();
@@ -144,162 +139,6 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
scheduler.update();
Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
assertEquals(3, queues.size());
-
- // Queue A should be above min share, B below.
- FSLeafQueue queueA =
- scheduler.getQueueManager().getLeafQueue("queueA", false);
- FSLeafQueue queueB =
- scheduler.getQueueManager().getLeafQueue("queueB", false);
- assertFalse(queueA.isStarvedForMinShare());
- assertTrue(queueB.isStarvedForMinShare());
-
- // Node checks in again, should allocate for B
- scheduler.handle(nodeEvent2);
- // Now B should have min share ( = demand here)
- assertFalse(queueB.isStarvedForMinShare());
- }
-
- @Test (timeout = 5000)
- public void testIsStarvedForFairShare() throws Exception {
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>.2</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>.8</weight>");
- out.println("<fairSharePreemptionThreshold>.4</fairSharePreemptionThreshold>");
- out.println("<queue name=\"queueB1\">");
- out.println("</queue>");
- out.println("<queue name=\"queueB2\">");
- out.println("<fairSharePreemptionThreshold>.6</fairSharePreemptionThreshold>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>");
- out.println("</allocations>");
- out.close();
-
- resourceManager = new MockRM(conf);
- resourceManager.start();
- scheduler = (FairScheduler) resourceManager.getResourceScheduler();
-
- // Add one big node (only care about aggregate capacity)
- RMNode node1 =
- MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024, 10), 1,
- "127.0.0.1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
-
- scheduler.update();
-
- // Queue A wants 4 * 1024. Node update gives this all to A
- createSchedulingRequest(1 * 1024, "queueA", "user1", 4);
- scheduler.update();
- NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);
- for (int i = 0; i < 4; i ++) {
- scheduler.handle(nodeEvent2);
- }
-
- QueueManager queueMgr = scheduler.getQueueManager();
- FSLeafQueue queueA = queueMgr.getLeafQueue("queueA", false);
- assertEquals(4 * 1024, queueA.getResourceUsage().getMemorySize());
-
- // Both queue B1 and queue B2 want 3 * 1024
- createSchedulingRequest(1 * 1024, "queueB.queueB1", "user1", 3);
- createSchedulingRequest(1 * 1024, "queueB.queueB2", "user1", 3);
- scheduler.update();
- for (int i = 0; i < 4; i ++) {
- scheduler.handle(nodeEvent2);
- }
-
- FSLeafQueue queueB1 = queueMgr.getLeafQueue("queueB.queueB1", false);
- FSLeafQueue queueB2 = queueMgr.getLeafQueue("queueB.queueB2", false);
- assertEquals(2 * 1024, queueB1.getResourceUsage().getMemorySize());
- assertEquals(2 * 1024, queueB2.getResourceUsage().getMemorySize());
-
- // For queue B1, the fairSharePreemptionThreshold is 0.4, and the fair share
- // threshold is 1.6 * 1024
- assertFalse(queueB1.isStarvedForFairShare());
-
- // For queue B2, the fairSharePreemptionThreshold is 0.6, and the fair share
- // threshold is 2.4 * 1024
- assertTrue(queueB2.isStarvedForFairShare());
-
- // Node checks in again
- scheduler.handle(nodeEvent2);
- scheduler.handle(nodeEvent2);
- assertEquals(3 * 1024, queueB1.getResourceUsage().getMemorySize());
- assertEquals(3 * 1024, queueB2.getResourceUsage().getMemorySize());
-
- // Both queue B1 and queue B2 usages go to 3 * 1024
- assertFalse(queueB1.isStarvedForFairShare());
- assertFalse(queueB2.isStarvedForFairShare());
- }
-
- @Test (timeout = 5000)
- public void testIsStarvedForFairShareDRF() throws Exception {
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>.5</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>.5</weight>");
- out.println("</queue>");
- out.println("<defaultFairSharePreemptionThreshold>1</defaultFairSharePreemptionThreshold>");
- out.println("<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
-
- resourceManager = new MockRM(conf);
- resourceManager.start();
- scheduler = (FairScheduler) resourceManager.getResourceScheduler();
-
- // Add one big node (only care about aggregate capacity)
- RMNode node1 =
- MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024, 10), 1,
- "127.0.0.1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
-
- scheduler.update();
-
- // Queue A wants 7 * 1024, 1. Node update gives this all to A
- createSchedulingRequest(7 * 1024, 1, "queueA", "user1", 1);
- scheduler.update();
- NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);
- scheduler.handle(nodeEvent2);
-
- QueueManager queueMgr = scheduler.getQueueManager();
- FSLeafQueue queueA = queueMgr.getLeafQueue("queueA", false);
- assertEquals(7 * 1024, queueA.getResourceUsage().getMemorySize());
- assertEquals(1, queueA.getResourceUsage().getVirtualCores());
-
- // Queue B has 3 reqs :
- // 1) 2 * 1024, 5 .. which will be granted
- // 2) 1 * 1024, 1 .. which will be granted
- // 3) 1 * 1024, 1 .. which wont
- createSchedulingRequest(2 * 1024, 5, "queueB", "user1", 1);
- createSchedulingRequest(1 * 1024, 2, "queueB", "user1", 2);
- scheduler.update();
- for (int i = 0; i < 3; i ++) {
- scheduler.handle(nodeEvent2);
- }
-
- FSLeafQueue queueB = queueMgr.getLeafQueue("queueB", false);
- assertEquals(3 * 1024, queueB.getResourceUsage().getMemorySize());
- assertEquals(6, queueB.getResourceUsage().getVirtualCores());
-
- scheduler.update();
-
- // Verify that Queue us not starved for fair share..
- // Since the Starvation logic now uses DRF when the policy = drf, The
- // Queue should not be starved
- assertFalse(queueB.isStarvedForFairShare());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/48] hadoop git commit: HADOOP-13018. Make Kdiag check whether
hadoop.token.files points to existent and valid files. Contributed by Ravi
Prakash
Posted by li...@apache.org.
HADOOP-13018. Make Kdiag check whether hadoop.token.files points to existent and valid files. Contributed by Ravi Prakash
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abb9fa7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abb9fa7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abb9fa7f
Branch: refs/heads/HADOOP-13345
Commit: abb9fa7fc69ec7b25f1c44e17c4c7dd17f5de16a
Parents: eb0a483
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 24 16:41:35 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 24 16:41:59 2016 +0000
----------------------------------------------------------------------
.../java/org/apache/hadoop/security/KDiag.java | 67 ++++++++++++++++++++
.../apache/hadoop/security/TestKDiagNoKDC.java | 8 +++
2 files changed, 75 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/abb9fa7f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
index 266bba0..542a502 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
@@ -38,6 +38,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.crypto.Cipher;
+
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
@@ -61,6 +62,7 @@ import static org.apache.hadoop.security.UserGroupInformation.*;
import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
import static org.apache.hadoop.util.StringUtils.popOption;
import static org.apache.hadoop.util.StringUtils.popOptionWithArgument;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES;
/**
* Kerberos diagnostics
@@ -145,6 +147,7 @@ public class KDiag extends Configured implements Tool, Closeable {
public static final String CAT_OS = "JAAS";
public static final String CAT_SASL = "SASL";
public static final String CAT_UGI = "UGI";
+ public static final String CAT_TOKEN = "TOKEN";
public static final String ARG_KEYLEN = "--keylen";
public static final String ARG_KEYTAB = "--keytab";
@@ -371,6 +374,7 @@ public class KDiag extends Configured implements Tool, Closeable {
try {
UserGroupInformation.setConfiguration(conf);
+ validateHadoopTokenFiles(conf);
validateKrb5File();
printDefaultRealm();
validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS);
@@ -501,6 +505,47 @@ public class KDiag extends Configured implements Tool, Closeable {
}
/**
+ * Validate that hadoop.token.files (if specified) exist and are valid.
+ * @throws ClassNotFoundException
+ * @throws SecurityException
+ * @throws NoSuchMethodException
+ * @throws KerberosDiagsFailure
+ */
+ private void validateHadoopTokenFiles(Configuration conf)
+ throws ClassNotFoundException, KerberosDiagsFailure, NoSuchMethodException,
+ SecurityException {
+ title("Locating Hadoop token files");
+
+ String tokenFileLocation = System.getProperty(HADOOP_TOKEN_FILES);
+ if(tokenFileLocation != null) {
+ println("Found " + HADOOP_TOKEN_FILES + " in system properties : "
+ + tokenFileLocation);
+ }
+
+ if(conf.get(HADOOP_TOKEN_FILES) != null) {
+ println("Found " + HADOOP_TOKEN_FILES + " in hadoop configuration : "
+ + conf.get(HADOOP_TOKEN_FILES));
+ if(System.getProperty(HADOOP_TOKEN_FILES) != null) {
+ println(HADOOP_TOKEN_FILES + " in the system properties overrides the"
+ + " one specified in hadoop configuration");
+ } else {
+ tokenFileLocation = conf.get(HADOOP_TOKEN_FILES);
+ }
+ }
+
+ if (tokenFileLocation != null) {
+ for (String tokenFileName:
+ StringUtils.getTrimmedStrings(tokenFileLocation)) {
+ if (tokenFileName.length() > 0) {
+ File tokenFile = new File(tokenFileName);
+ verifyFileIsValid(tokenFile, CAT_TOKEN, "token");
+ verify(tokenFile, conf, CAT_TOKEN, "token");
+ }
+ }
+ }
+ }
+
+ /**
* Locate the {@code krb5.conf} file and dump it.
*
* No-op on windows.
@@ -929,6 +974,28 @@ public class KDiag extends Configured implements Tool, Closeable {
}
/**
+ * Verify that tokenFile contains valid Credentials.
+ *
+ * If not, an exception is raised, or, if {@link #nofail} is set,
+ * an error will be logged and the method return false.
+ *
+ */
+ private boolean verify(File tokenFile, Configuration conf, String category,
+ String message) throws KerberosDiagsFailure {
+ try {
+ Credentials.readTokenStorageFile(tokenFile, conf);
+ } catch(Exception e) {
+ if (!nofail) {
+ fail(category, message);
+ } else {
+ error(category, message);
+ }
+ return false;
+ }
+ return true;
+ }
+
+ /**
* Print a message as an error
* @param category error category
* @param message format string
http://git-wip-us.apache.org/repos/asf/hadoop/blob/abb9fa7f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiagNoKDC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiagNoKDC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiagNoKDC.java
index 9d4b87f..dbc40c5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiagNoKDC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiagNoKDC.java
@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.Properties;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
import static org.apache.hadoop.security.KDiag.ARG_KEYLEN;
import static org.apache.hadoop.security.KDiag.ARG_KEYTAB;
@@ -44,6 +45,7 @@ import static org.apache.hadoop.security.KDiag.ARG_SECURE;
import static org.apache.hadoop.security.KDiag.CAT_CONFIG;
import static org.apache.hadoop.security.KDiag.CAT_KERBEROS;
import static org.apache.hadoop.security.KDiag.CAT_LOGIN;
+import static org.apache.hadoop.security.KDiag.CAT_TOKEN;
import static org.apache.hadoop.security.KDiag.KerberosDiagsFailure;
import static org.apache.hadoop.security.KDiag.exec;
@@ -120,4 +122,10 @@ public class TestKDiagNoKDC extends Assert {
assertEquals(-1, kdiag("usage"));
}
+ @Test
+ public void testTokenFile() throws Throwable {
+ conf.set(HADOOP_TOKEN_FILES, "SomeNonExistentFile");
+ kdiagFailure(CAT_TOKEN, ARG_KEYLEN, KEYLEN);
+ conf.unset(HADOOP_TOKEN_FILES);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/48] hadoop git commit: YARN-5649. Add REST endpoints for updating
application timeouts. Contributed by Rohith Sharma K S
Posted by li...@apache.org.
YARN-5649. Add REST endpoints for updating application timeouts. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f12867a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f12867a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f12867a
Branch: refs/heads/HADOOP-13345
Commit: 1f12867a69544a1642aa986d4f9a8249be495434
Parents: 3d94da1
Author: Jian He <ji...@apache.org>
Authored: Wed Nov 23 16:25:29 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Wed Nov 23 16:25:39 2016 -0800
----------------------------------------------------------------------
.../server/resourcemanager/ClientRMService.java | 2 +-
.../server/resourcemanager/RMAuditLogger.java | 1 +
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../webapp/JAXBContextResolver.java | 3 +-
.../resourcemanager/webapp/RMWebServices.java | 186 +++++++++++++++++++
.../resourcemanager/webapp/dao/AppInfo.java | 24 +++
.../webapp/dao/AppTimeoutInfo.java | 71 +++++++
.../webapp/dao/AppTimeoutsInfo.java | 47 +++++
.../server/resourcemanager/rmapp/MockRMApp.java | 3 +-
.../webapp/TestRMWebServicesApps.java | 2 +-
.../TestRMWebServicesAppsModification.java | 137 +++++++++++++-
11 files changed, 472 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 4e36b6c..0db775f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1752,7 +1752,7 @@ public class ClientRMService extends AbstractService implements
RMAuditLogger.logFailure(callerUGI.getShortUserName(),
AuditConstants.UPDATE_APP_TIMEOUTS, "UNKNOWN", "ClientRMService",
ex.getMessage());
- throw RPCUtil.getRemoteException(ex);
+ throw ex;
}
RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
index d52e002..051d979 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
@@ -66,6 +66,7 @@ public class RMAuditLogger {
"Update Application Priority";
public static final String UPDATE_APP_TIMEOUTS =
"Update Application Timeouts";
+ public static final String GET_APP_TIMEOUTS = "Get Application Timeouts";
public static final String CHANGE_CONTAINER_RESOURCE =
"AM Changed Container Resource";
public static final String SIGNAL_CONTAINER = "Signal Container Request";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 98a34d9..45a1f30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -515,7 +515,7 @@ public class RMServerUtils {
String message =
"Expire time is not in ISO8601 format. ISO8601 supported "
+ "format is yyyy-MM-dd'T'HH:mm:ss.SSSZ";
- throw new YarnException(message);
+ throw new YarnException(message, ex);
}
if (expireTime < currentTimeMillis) {
String message =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
index b61072d..2f50a24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
@@ -54,7 +54,8 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
CapacitySchedulerQueueInfoList.class, ResourceInfo.class,
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
- FairSchedulerQueueInfoList.class};
+ FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
+ AppTimeoutInfo.class };
// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 852f937..a46fb81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -24,9 +24,11 @@ import java.nio.ByteBuffer;
import java.security.AccessControlException;
import java.security.Principal;
import java.security.PrivilegedExceptionAction;
+import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -96,10 +98,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -189,6 +193,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
import org.apache.hadoop.yarn.util.AdHocLogDumper;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -2428,4 +2433,185 @@ public class RMWebServices extends WebServices {
return Response.status(Status.OK).entity(resResponse).build();
}
+ @GET
+ @Path("/apps/{appid}/timeout/{type}")
+ @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ public AppTimeoutInfo getAppTimeout(@Context HttpServletRequest hsr,
+ @PathParam("appid") String appId, @PathParam("type") String type)
+ throws AuthorizationException {
+ init();
+ RMApp app = validateAppTimeoutRequest(hsr, appId);
+
+ ApplicationTimeoutType appTimeoutType = parseTimeoutType(type);
+ Long timeoutValue = app.getApplicationTimeouts().get(appTimeoutType);
+ AppTimeoutInfo timeout =
+ constructAppTimeoutDao(appTimeoutType, timeoutValue);
+ return timeout;
+ }
+
+ private RMApp validateAppTimeoutRequest(HttpServletRequest hsr,
+ String appId) {
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ String userName = "UNKNOWN-USER";
+ if (callerUGI != null) {
+ userName = callerUGI.getUserName();
+ }
+
+ if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
+ String msg = "The default static user cannot carry out this operation.";
+ RMAuditLogger.logFailure(userName, AuditConstants.GET_APP_TIMEOUTS,
+ "UNKNOWN", "RMWebService", msg);
+ throw new ForbiddenException(msg);
+ }
+
+ RMApp app = null;
+ try {
+ app = getRMAppForAppId(appId);
+ } catch (NotFoundException e) {
+ RMAuditLogger.logFailure(userName, AuditConstants.GET_APP_TIMEOUTS,
+ "UNKNOWN", "RMWebService",
+ "Trying to get timeouts of an absent application " + appId);
+ throw e;
+ }
+ return app;
+ }
+
+ @GET
+ @Path("/apps/{appid}/timeouts")
+ @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ public AppTimeoutsInfo getAppTimeouts(@Context HttpServletRequest hsr,
+ @PathParam("appid") String appId) throws AuthorizationException {
+ init();
+
+ RMApp app = validateAppTimeoutRequest(hsr, appId);
+
+ AppTimeoutsInfo timeouts = new AppTimeoutsInfo();
+ Map<ApplicationTimeoutType, Long> applicationTimeouts =
+ app.getApplicationTimeouts();
+ if (applicationTimeouts.isEmpty()) {
+ // If application is not set timeout, lifetime should be sent as default
+ // with expiryTime=UNLIMITED and remainingTime=-1
+ timeouts
+ .add(constructAppTimeoutDao(ApplicationTimeoutType.LIFETIME, null));
+ } else {
+ for (Entry<ApplicationTimeoutType, Long> timeout : app
+ .getApplicationTimeouts().entrySet()) {
+ AppTimeoutInfo timeoutInfo =
+ constructAppTimeoutDao(timeout.getKey(), timeout.getValue());
+ timeouts.add(timeoutInfo);
+ }
+ }
+ return timeouts;
+ }
+
+ private ApplicationTimeoutType parseTimeoutType(String type) {
+ try {
+ // enum string is in the uppercase
+ return ApplicationTimeoutType
+ .valueOf(StringUtils.toUpperCase(type.trim()));
+ } catch (RuntimeException e) {
+ ApplicationTimeoutType[] typeArray = ApplicationTimeoutType.values();
+ String allAppTimeoutTypes = Arrays.toString(typeArray);
+ throw new BadRequestException("Invalid application-state " + type.trim()
+ + " specified. It should be one of " + allAppTimeoutTypes);
+ }
+ }
+
+ private AppTimeoutInfo constructAppTimeoutDao(ApplicationTimeoutType type,
+ Long timeoutInMillis) {
+ AppTimeoutInfo timeout = new AppTimeoutInfo();
+ timeout.setTimeoutType(type);
+ if (timeoutInMillis != null) {
+ timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis.longValue()));
+ timeout.setRemainingTime(
+ Math.max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+ }
+ return timeout;
+ }
+
+ @PUT
+ @Path("/apps/{appid}/timeout")
+ @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response updateApplicationTimeout(AppTimeoutInfo appTimeout,
+ @Context HttpServletRequest hsr, @PathParam("appid") String appId)
+ throws AuthorizationException, YarnException, InterruptedException,
+ IOException {
+ init();
+
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ if (callerUGI == null) {
+ throw new AuthorizationException(
+ "Unable to obtain user name, user not authenticated");
+ }
+
+ if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
+ return Response.status(Status.FORBIDDEN)
+ .entity("The default static user cannot carry out this operation.")
+ .build();
+ }
+
+ String userName = callerUGI.getUserName();
+ RMApp app = null;
+ try {
+ app = getRMAppForAppId(appId);
+ } catch (NotFoundException e) {
+ RMAuditLogger.logFailure(userName, AuditConstants.UPDATE_APP_TIMEOUTS,
+ "UNKNOWN", "RMWebService",
+ "Trying to update timeout of an absent application " + appId);
+ throw e;
+ }
+
+ return updateApplicationTimeouts(app, callerUGI, appTimeout);
+ }
+
+ private Response updateApplicationTimeouts(final RMApp app,
+ UserGroupInformation callerUGI, final AppTimeoutInfo appTimeout)
+ throws IOException, InterruptedException {
+
+ if (appTimeout.getTimeoutType() == null) {
+ return Response.status(Status.BAD_REQUEST).entity("Timeout type is null.")
+ .build();
+ }
+
+ String userName = callerUGI.getUserName();
+ try {
+ callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException, YarnException {
+ UpdateApplicationTimeoutsRequest request =
+ UpdateApplicationTimeoutsRequest
+ .newInstance(app.getApplicationId(), Collections.singletonMap(
+ appTimeout.getTimeoutType(), appTimeout.getExpireTime()));
+ rm.getClientRMService().updateApplicationTimeouts(request);
+ return null;
+ }
+ });
+ } catch (UndeclaredThrowableException ue) {
+ // if the root cause is a permissions issue
+ // bubble that up to the user
+ if (ue.getCause() instanceof YarnException) {
+ YarnException ye = (YarnException) ue.getCause();
+ if (ye.getCause() instanceof AccessControlException) {
+ String appId = app.getApplicationId().toString();
+ String msg = "Unauthorized attempt to change timeout of app " + appId
+ + " by remote user " + userName;
+ return Response.status(Status.FORBIDDEN).entity(msg).build();
+ } else if (ye.getCause() instanceof ParseException) {
+ return Response.status(Status.BAD_REQUEST)
+ .entity(ye.getMessage()).build();
+ } else {
+ throw ue;
+ }
+ } else {
+ throw ue;
+ }
+ }
+ AppTimeoutInfo timeout = constructAppTimeoutDao(appTimeout.getTimeoutType(),
+ app.getApplicationTimeouts().get(appTimeout.getTimeoutType()));
+ return Response.status(Status.OK).entity(timeout).build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index d3ddf80..2d364f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -29,6 +30,7 @@ import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
@@ -114,6 +116,7 @@ public class AppInfo {
protected String amNodeLabelExpression;
protected ResourcesInfo resourceInfo = null;
+ protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo();
public AppInfo() {
} // JAXB needs this
@@ -240,6 +243,27 @@ public class AppInfo {
: null;
}
}
+
+ Map<ApplicationTimeoutType, Long> applicationTimeouts =
+ app.getApplicationTimeouts();
+ if (applicationTimeouts.isEmpty()) {
+ // If application is not set timeout, lifetime should be sent as default
+ // with expiryTime=UNLIMITED and remainingTime=-1
+ AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
+ timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
+ timeouts.add(timeoutInfo);
+ } else {
+ for (Map.Entry<ApplicationTimeoutType, Long> entry : app
+ .getApplicationTimeouts().entrySet()) {
+ AppTimeoutInfo timeout = new AppTimeoutInfo();
+ timeout.setTimeoutType(entry.getKey());
+ long timeoutInMillis = entry.getValue().longValue();
+ timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
+ timeout.setRemainingTime(Math
+ .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+ timeouts.add(timeout);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java
new file mode 100644
index 0000000..140857e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+
+/**
+ * DAO object to display Application timeout information.
+ */
+@XmlRootElement(name = "timeout")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AppTimeoutInfo {
+
+ @XmlElement(name = "type")
+ private ApplicationTimeoutType timeoutType;
+
+ @XmlElement(name = "expiryTime")
+ private String expiryTime;
+
+ @XmlElement(name = "remainingTimeInSeconds")
+ private long remainingTimeInSec;
+
+ public AppTimeoutInfo() {
+ expiryTime = "UNLIMITED";
+ remainingTimeInSec = -1;
+ }
+
+ public ApplicationTimeoutType getTimeoutType() {
+ return timeoutType;
+ }
+
+ public String getExpireTime() {
+ return expiryTime;
+ }
+
+ public long getRemainingTimeInSec() {
+ return remainingTimeInSec;
+ }
+
+ public void setTimeoutType(ApplicationTimeoutType type) {
+ this.timeoutType = type;
+ }
+
+ public void setExpiryTime(String expiryTime) {
+ this.expiryTime = expiryTime;
+ }
+
+ public void setRemainingTime(long remainingTime) {
+ this.remainingTimeInSec = remainingTime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java
new file mode 100644
index 0000000..256e55c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * This class hosts a set of AppTimeout DAO objects.
+ */
+@XmlRootElement(name = "timeouts")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AppTimeoutsInfo {
+
+ @XmlElement(name = "timeout")
+ private ArrayList<AppTimeoutInfo> timeouts = new ArrayList<AppTimeoutInfo>();
+
+ public AppTimeoutsInfo() {
+ } // JAXB needs this
+
+ public void add(AppTimeoutInfo timeoutInfo) {
+ timeouts.add(timeoutInfo);
+ }
+
+ public ArrayList<AppTimeoutInfo> getAppTimeouts() {
+ return timeouts;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 48a60b5..118b6bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import java.util.Collection;
+import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
@@ -319,7 +320,7 @@ public class MockRMApp implements RMApp {
@Override
public Map<ApplicationTimeoutType, Long> getApplicationTimeouts() {
- throw new UnsupportedOperationException("Not supported yet.");
+ return Collections.emptyMap();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index fffa55b..30f25e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1419,7 +1419,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
- int expectedNumberOfElements = 34;
+ int expectedNumberOfElements = 35;
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
if (app.getApplicationSubmissionContext()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f12867a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index e884a88..6458471 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHand
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -83,15 +84,17 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedule
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LogAggregationContextInfo;
-import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.After;
@@ -117,6 +120,7 @@ import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.client.filter.LoggingFilter;
+import com.sun.jersey.api.json.JSONConfiguration;
import com.sun.jersey.api.json.JSONJAXBContext;
import com.sun.jersey.api.json.JSONMarshaller;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
@@ -1295,4 +1299,135 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
assertEquals(queue, responseQueue);
}
+ @Test(timeout = 90000)
+ public void testUpdateAppTimeout() throws Exception {
+ client().addFilter(new LoggingFilter(System.out));
+
+ rm.start();
+ rm.registerNode("127.0.0.1:1234", 2048);
+ String[] mediaTypes =
+ { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
+ MediaType[] contentTypes =
+ { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE };
+ for (String mediaType : mediaTypes) {
+ for (MediaType contentType : contentTypes) {
+ // application submitted without timeout
+ RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
+
+ ClientResponse response =
+ this.constructWebResource("apps", app.getApplicationId().toString(),
+ "timeouts").accept(mediaType).get(ClientResponse.class);
+ if (mediaType.contains(MediaType.APPLICATION_JSON)) {
+ assertEquals(
+ MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject js =
+ response.getEntity(JSONObject.class).getJSONObject("timeouts");
+ JSONArray entity = js.getJSONArray("timeout");
+ verifyAppTimeoutJson(entity.getJSONObject(0),
+ ApplicationTimeoutType.LIFETIME, "UNLIMITED", -1);
+ }
+
+ AppTimeoutInfo timeoutUpdate = new AppTimeoutInfo();
+ long timeOutFromNow = 60;
+ String expireTime = Times
+ .formatISO8601(System.currentTimeMillis() + timeOutFromNow * 1000);
+ timeoutUpdate.setTimeoutType(ApplicationTimeoutType.LIFETIME);
+ timeoutUpdate.setExpiryTime(expireTime);
+
+ Object entity;
+ if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) {
+ entity = appTimeoutToJSON(timeoutUpdate);
+ } else {
+ entity = timeoutUpdate;
+ }
+ response = this
+ .constructWebResource("apps", app.getApplicationId().toString(),
+ "timeout")
+ .entity(entity, contentType).accept(mediaType)
+ .put(ClientResponse.class);
+
+ if (!isAuthenticationEnabled()) {
+ assertResponseStatusCode(Status.UNAUTHORIZED,
+ response.getStatusInfo());
+ continue;
+ }
+ assertResponseStatusCode(Status.OK, response.getStatusInfo());
+ if (mediaType.contains(MediaType.APPLICATION_JSON)) {
+ verifyAppTimeoutJson(response, ApplicationTimeoutType.LIFETIME,
+ expireTime, timeOutFromNow);
+ } else {
+ verifyAppTimeoutXML(response, ApplicationTimeoutType.LIFETIME,
+ expireTime, timeOutFromNow);
+ }
+
+ // invoke get
+ response =
+ this.constructWebResource("apps", app.getApplicationId().toString(),
+ "timeout", ApplicationTimeoutType.LIFETIME.toString())
+ .accept(mediaType).get(ClientResponse.class);
+ assertResponseStatusCode(Status.OK, response.getStatusInfo());
+ if (mediaType.contains(MediaType.APPLICATION_JSON)) {
+ verifyAppTimeoutJson(response, ApplicationTimeoutType.LIFETIME,
+ expireTime, timeOutFromNow);
+ }
+ }
+ }
+ rm.stop();
+ }
+
+ protected static void verifyAppTimeoutJson(ClientResponse response,
+ ApplicationTimeoutType type, String expireTime, long timeOutFromNow)
+ throws JSONException {
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject jsonTimeout = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, jsonTimeout.length());
+ JSONObject json = jsonTimeout.getJSONObject("timeout");
+ verifyAppTimeoutJson(json, type, expireTime, timeOutFromNow);
+ }
+
+ protected static void verifyAppTimeoutJson(JSONObject json,
+ ApplicationTimeoutType type, String expireTime, long timeOutFromNow)
+ throws JSONException {
+ assertEquals("incorrect number of elements", 3, json.length());
+ assertEquals(type.toString(), json.getString("type"));
+ assertEquals(expireTime, json.getString("expiryTime"));
+ assertTrue(json.getLong("remainingTimeInSeconds") <= timeOutFromNow);
+ }
+
+ protected static void verifyAppTimeoutXML(ClientResponse response,
+ ApplicationTimeoutType type, String expireTime, long timeOutFromNow)
+ throws ParserConfigurationException, IOException, SAXException {
+ assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList nodes = dom.getElementsByTagName("timeout");
+ assertEquals("incorrect number of elements", 1, nodes.getLength());
+ Element element = (Element) nodes.item(0);
+ assertEquals(type.toString(),
+ WebServicesTestUtils.getXmlString(element, "type"));
+ assertEquals(expireTime,
+ WebServicesTestUtils.getXmlString(element, "expiryTime"));
+ assertTrue(WebServicesTestUtils.getXmlLong(element,
+ "remainingTimeInSeconds") < timeOutFromNow);
+ }
+
+ protected static String appTimeoutToJSON(AppTimeoutInfo timeout)
+ throws Exception {
+ StringWriter sw = new StringWriter();
+ JSONJAXBContext ctx = new JSONJAXBContext(
+ JSONConfiguration.natural().rootUnwrapping(false).build(),
+ AppTimeoutInfo.class);
+ JSONMarshaller jm = ctx.createJSONMarshaller();
+ jm.marshallToJSON(timeout, sw);
+ jm.marshallToJSON(timeout, System.out);
+ return sw.toString();
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/48] hadoop git commit: YARN-5572. HBaseTimelineWriterImpl appears
to reference a bad property name. Contributed by Varun Saxena.
Posted by li...@apache.org.
YARN-5572. HBaseTimelineWriterImpl appears to reference a bad property name. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c06114d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c06114d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c06114d6
Branch: refs/heads/HADOOP-13345
Commit: c06114d6a360dddeb66c2dd9ad4fa5dae0cfbfb1
Parents: e2f77ca
Author: Naganarasimha <na...@apache.org>
Authored: Sun Nov 27 23:35:53 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Sun Nov 27 23:35:53 2016 +0530
----------------------------------------------------------------------
.../TestTimelineReaderWebServicesHBaseStorage.java | 2 +-
.../timelineservice/storage/DataGeneratorForTest.java | 4 ++--
.../storage/TestHBaseTimelineStorageApps.java | 8 ++++----
.../storage/TestHBaseTimelineStorageEntities.java | 6 +++---
.../storage/flow/TestHBaseStorageFlowActivity.java | 6 +++---
.../storage/flow/TestHBaseStorageFlowRun.java | 14 +++++++-------
.../flow/TestHBaseStorageFlowRunCompaction.java | 2 +-
.../storage/HBaseTimelineWriterImpl.java | 5 -----
8 files changed, 21 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c06114d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index 2ed5d96..17c01b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -334,7 +334,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, flowVersion, runid, entity.getId(), te);
hbi.write(cluster, user, flow, flowVersion, runid, entity1.getId(), te1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c06114d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
index 0938e9e..5cbb781 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
@@ -182,7 +182,7 @@ final class DataGeneratorForTest {
te2.addEntity(entity2);
HBaseTimelineWriterImpl hbi = null;
try {
- hbi = new HBaseTimelineWriterImpl(util.getConfiguration());
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(util.getConfiguration());
hbi.start();
String cluster = "cluster1";
@@ -360,7 +360,7 @@ final class DataGeneratorForTest {
te.addEntity(entity2);
HBaseTimelineWriterImpl hbi = null;
try {
- hbi = new HBaseTimelineWriterImpl(util.getConfiguration());
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(util.getConfiguration());
hbi.start();
String cluster = "cluster1";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c06114d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
index e70198a..b3e5197 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
@@ -155,7 +155,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "cluster_check_null_application";
@@ -275,7 +275,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_write_app";
@@ -502,7 +502,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_events";
@@ -620,7 +620,7 @@ public class TestHBaseTimelineStorageApps {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
// Writing application entity.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c06114d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
index 3076709..4b4c3e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
@@ -194,7 +194,7 @@ public class TestHBaseTimelineStorageEntities {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_write_entity";
@@ -391,7 +391,7 @@ public class TestHBaseTimelineStorageEntities {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "cluster_test_empty_eventkey";
@@ -496,7 +496,7 @@ public class TestHBaseTimelineStorageEntities {
HBaseTimelineWriterImpl hbi = null;
try {
Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.start();
String cluster = "clus!ter_\ttest_ev ents";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c06114d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index 1906574..3ddb230 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -119,7 +119,7 @@ public class TestHBaseStorageFlowActivity {
.getEntityMinStartTime(minStartTs);
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -228,7 +228,7 @@ public class TestHBaseStorageFlowActivity {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_1111999999_1234";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -341,7 +341,7 @@ public class TestHBaseStorageFlowActivity {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_11888888888_1111";
hbi.write(cluster, user, flow, flowVersion1, runid1, appName, te);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c06114d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index 74b9e50..377611f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -184,7 +184,7 @@ public class TestHBaseStorageFlowRun {
.getEntityMinStartTime(minStartTs);
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -290,7 +290,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_11111111111111_1111";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -559,7 +559,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_11111111111111_1111";
hbi.write(cluster, user, flow, flowVersion, 1002345678919L, appName, te);
@@ -644,7 +644,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_11111111111111_1111";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -737,7 +737,7 @@ public class TestHBaseStorageFlowRun {
TimelineEntity entityApp1 = null;
TimelineEntity entityApp2 = null;
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
for (int i = start; i < count; i++) {
@@ -826,7 +826,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L,
"application_11111111111111_1111", te);
@@ -908,7 +908,7 @@ public class TestHBaseStorageFlowRun {
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L,
"application_11111111111111_1111", te);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c06114d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
index 3094088..756d57b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
@@ -287,7 +287,7 @@ public class TestHBaseStorageFlowRunCompaction {
TimelineEntities te1 = null;
TimelineEntity entityApp1 = null;
try {
- hbi = new HBaseTimelineWriterImpl(c1);
+ hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
// now insert count * ( 100 + 100) metrics
// each call to getEntityMetricsApp1 brings back 100 values
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c06114d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 3511a2f..b94b85f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -101,11 +101,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
super(HBaseTimelineWriterImpl.class.getName());
}
- public HBaseTimelineWriterImpl(Configuration conf) throws IOException {
- super(conf.get("yarn.application.id",
- HBaseTimelineWriterImpl.class.getName()));
- }
-
/**
* initializes the hbase connection to write to the entity table.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/48] hadoop git commit: HADOOP-13605. Clean up FileSystem
javadocs, logging;
improve diagnostics on FS load. Contributed by Steve Loughran
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/860d49aa/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index b18b5f6..201d397 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -419,7 +419,7 @@ If the filesystem is not location aware, it SHOULD return
BlockLocation(["localhost:9866"] ,
["localhost"],
["/default/localhost"]
- 0, F.getLen())
+ 0, f.getLen())
] ;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/860d49aa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
index f232735..b84d66a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
@@ -21,14 +21,14 @@ import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.test.GenericTestUtils;
+
import org.junit.Test;
+import static org.apache.hadoop.test.LambdaTestUtils.*;
/**
* Test default URI related APIs in {@link FileSystem}.
@@ -69,15 +69,12 @@ public class TestDefaultUri {
}
@Test
- public void tetGetDefaultUriNoSchemeTrailingSlash() {
+ public void tetGetDefaultUriNoSchemeTrailingSlash() throws Exception {
conf.set(FS_DEFAULT_NAME_KEY, "nn_host/");
- try {
- FileSystem.getDefaultUri(conf);
- fail("Expect IAE: No scheme in default FS");
- } catch (IllegalArgumentException e) {
- GenericTestUtils.assertExceptionContains(
- "No scheme in default FS", e);
- }
+ intercept(IllegalArgumentException.class,
+ "No scheme in default FS",
+ () -> FileSystem.getDefaultUri(conf));
+
}
@Test
@@ -88,28 +85,19 @@ public class TestDefaultUri {
}
@Test
- public void tetFsGetNoScheme() throws IOException {
+ public void tetFsGetNoScheme() throws Exception {
// Bare host name or address indicates hdfs scheme
conf.set(FS_DEFAULT_NAME_KEY, "nn_host");
- try {
- FileSystem.get(conf);
- fail("Expect IOE: No FileSystem for scheme: hdfs");
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains(
- "No FileSystem for scheme: hdfs", e);
- }
+ intercept(UnsupportedFileSystemException.class, "hdfs",
+ () -> FileSystem.get(conf));
}
@Test
- public void tetFsGetNoSchemeTrailingSlash() throws IOException {
+ public void tetFsGetNoSchemeTrailingSlash() throws Exception {
// Bare host name or address with trailing slash is invalid
conf.set(FS_DEFAULT_NAME_KEY, "nn_host/");
- try {
- FileSystem.get(conf);
- fail("Expect IAE: No scheme in default FS");
- } catch (IllegalArgumentException e) {
- GenericTestUtils.assertExceptionContains(
- "No scheme in default FS", e);
- }
+ intercept(IllegalArgumentException.class,
+ "No scheme in default FS",
+ () -> FileSystem.get(conf));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/860d49aa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java
index 07b07dc..69ef71e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java
@@ -95,16 +95,14 @@ public class TestFileSystemCaching {
try {
fs = FileSystem.get(URI.create("//host"), conf);
fail("got fs with auth but no scheme");
- } catch (Exception e) {
- assertEquals("No FileSystem for scheme: null", e.getMessage());
+ } catch (UnsupportedFileSystemException e) {
}
-
+
// no scheme, different auth
try {
fs = FileSystem.get(URI.create("//host2"), conf);
fail("got fs with auth but no scheme");
- } catch (Exception e) {
- assertEquals("No FileSystem for scheme: null", e.getMessage());
+ } catch (UnsupportedFileSystemException e) {
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/48] hadoop git commit: HADOOP-13706. Update jackson from 1.9.13
to 2.x in hadoop-common-project.
Posted by li...@apache.org.
HADOOP-13706. Update jackson from 1.9.13 to 2.x in hadoop-common-project.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67d9f280
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67d9f280
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67d9f280
Branch: refs/heads/HADOOP-13345
Commit: 67d9f2808efb34b9a7b0b824cb4033b95ad33474
Parents: 47ca9e2
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 29 14:07:19 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 29 14:07:19 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/pom.xml | 14 ++++----------
.../java/org/apache/hadoop/conf/Configuration.java | 8 ++++----
.../hadoop/crypto/key/kms/KMSClientProvider.java | 2 +-
.../main/java/org/apache/hadoop/fs/shell/Display.java | 11 +----------
.../java/org/apache/hadoop/ipc/DecayRpcScheduler.java | 2 +-
.../src/main/java/org/apache/hadoop/ipc/Server.java | 2 +-
.../java/org/apache/hadoop/jmx/JMXJsonServlet.java | 6 +++---
.../src/main/java/org/apache/hadoop/log/Log4Json.java | 14 +++++++-------
.../web/DelegationTokenAuthenticationHandler.java | 6 +++---
.../delegation/web/DelegationTokenAuthenticator.java | 2 +-
.../org/apache/hadoop/util/HttpExceptionUtils.java | 2 +-
.../org/apache/hadoop/conf/TestConfiguration.java | 2 +-
.../test/java/org/apache/hadoop/log/TestLog4Json.java | 13 +++++--------
...DelegationTokenAuthenticationHandlerWithMocks.java | 3 ++-
.../token/delegation/web/TestWebDelegationToken.java | 4 +---
.../apache/hadoop/util/TestHttpExceptionUtils.java | 2 +-
hadoop-common-project/hadoop-kms/pom.xml | 4 ++++
.../hadoop/crypto/key/kms/server/KMSJSONReader.java | 3 ++-
.../hadoop/crypto/key/kms/server/KMSJSONWriter.java | 3 ++-
19 files changed, 45 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index f442b2c4..1d45452 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -192,16 +192,6 @@
<scope>runtime</scope>
</dependency>
<dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
@@ -324,6 +314,10 @@
<groupId>org.apache.kerby</groupId>
<artifactId>kerb-simplekdc</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index dbbc8ff..c0e42e5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.conf;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
import com.google.common.annotations.VisibleForTesting;
import java.io.BufferedInputStream;
@@ -91,8 +93,6 @@ import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
import org.w3c.dom.Attr;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
@@ -3028,7 +3028,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
propertyName + " not found");
} else {
JsonFactory dumpFactory = new JsonFactory();
- JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
+ JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("property");
appendJSONProperty(dumpGenerator, config, propertyName);
@@ -3066,7 +3066,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
public static void dumpConfiguration(Configuration config,
Writer out) throws IOException {
JsonFactory dumpFactory = new JsonFactory();
- JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
+ JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("properties");
dumpGenerator.writeStartArray();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index f184ea7..ff03db9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthentica
import org.apache.hadoop.util.HttpExceptionUtils;
import org.apache.hadoop.util.KMSUtil;
import org.apache.http.client.utils.URIBuilder;
-import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,6 +79,7 @@ import java.util.concurrent.ExecutionException;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
index e30e45c..d3b9a21 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
@@ -52,10 +52,6 @@ import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
-import org.codehaus.jackson.JsonEncoding;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.util.MinimalPrettyPrinter;
/**
* Display contents or checksums of files
@@ -277,12 +273,7 @@ class Display extends FsCommand {
Schema schema = fileReader.getSchema();
writer = new GenericDatumWriter<Object>(schema);
output = new ByteArrayOutputStream();
- JsonGenerator generator =
- new JsonFactory().createJsonGenerator(output, JsonEncoding.UTF8);
- MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter();
- prettyPrinter.setRootValueSeparator(System.getProperty("line.separator"));
- generator.setPrettyPrinter(prettyPrinter);
- encoder = EncoderFactory.get().jsonEncoder(schema, generator);
+ encoder = EncoderFactory.get().jsonEncoder(schema, output);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
index f40bd17..5547b1a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
@@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicLongArray;
import java.util.concurrent.atomic.AtomicReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.AtomicDoubleArray;
import org.apache.commons.lang.exception.ExceptionUtils;
@@ -47,7 +48,6 @@ import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.metrics2.util.Metrics2Util.NameValuePair;
import org.apache.hadoop.metrics2.util.Metrics2Util.TopN;
-import org.codehaus.jackson.map.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 5f9de8a..5ed6e15 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -118,11 +118,11 @@ import org.apache.htrace.core.SpanId;
import org.apache.htrace.core.TraceScope;
import org.apache.htrace.core.Tracer;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Message;
-import org.codehaus.jackson.map.ObjectMapper;
/** An abstract IPC service. IPC calls take a single {@link Writable} as a
* parameter, and return a {@link Writable} as their value. A service runs on
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
index 05434d6..a8d9fa7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
@@ -17,11 +17,11 @@
package org.apache.hadoop.jmx;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.http.HttpServer2;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
import javax.management.AttributeNotFoundException;
import javax.management.InstanceNotFoundException;
@@ -186,7 +186,7 @@ public class JMXJsonServlet extends HttpServlet {
response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, "GET");
response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
- jg = jsonFactory.createJsonGenerator(writer);
+ jg = jsonFactory.createGenerator(writer);
jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
jg.useDefaultPrettyPrinter();
jg.writeStartObject();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/Log4Json.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/Log4Json.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/Log4Json.java
index d998fbc..337846c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/Log4Json.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/Log4Json.java
@@ -19,16 +19,16 @@
package org.apache.hadoop.log;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.MappingJsonFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ContainerNode;
import org.apache.log4j.Layout;
import org.apache.log4j.helpers.ISO8601DateFormat;
import org.apache.log4j.spi.LoggingEvent;
import org.apache.log4j.spi.ThrowableInformation;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.MappingJsonFactory;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.node.ContainerNode;
import java.io.IOException;
import java.io.StringWriter;
@@ -197,7 +197,7 @@ public class Log4Json extends Layout {
final String threadName,
final String message,
final ThrowableInformation ti) throws IOException {
- JsonGenerator json = factory.createJsonGenerator(writer);
+ JsonGenerator json = factory.createGenerator(writer);
json.writeStartObject();
json.writeStringField(NAME, loggerName);
json.writeNumberField(TIME, timeStamp);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index 315c9d6..e8bf7f7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -48,10 +48,10 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdenti
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
import org.apache.hadoop.util.HttpExceptionUtils;
import org.apache.hadoop.util.StringUtils;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.map.ObjectMapper;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
index 2d60d4a..9f39569 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.security.token.delegation.web;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.SecurityUtil;
@@ -29,7 +30,6 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.util.HttpExceptionUtils;
import org.apache.hadoop.util.StringUtils;
-import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
index 7072d9a..52403d8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
@@ -17,9 +17,9 @@
*/
package org.apache.hadoop.util;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.codehaus.jackson.map.ObjectMapper;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 8a422f7..1f97786 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -40,6 +40,7 @@ import java.util.Set;
import java.util.regex.Pattern;
import static java.util.concurrent.TimeUnit.*;
+import com.fasterxml.jackson.databind.ObjectMapper;
import junit.framework.TestCase;
import static org.junit.Assert.assertArrayEquals;
@@ -52,7 +53,6 @@ import org.apache.hadoop.test.GenericTestUtils;
import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
-import org.codehaus.jackson.map.ObjectMapper;
import org.mockito.Mockito;
public class TestConfiguration extends TestCase {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLog4Json.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLog4Json.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLog4Json.java
index e875693..9fea50e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLog4Json.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLog4Json.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.log;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ContainerNode;
import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -31,10 +33,6 @@ import org.apache.log4j.spi.HierarchyEventListener;
import org.apache.log4j.spi.LoggerFactory;
import org.apache.log4j.spi.LoggerRepository;
import org.apache.log4j.spi.ThrowableInformation;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.MappingJsonFactory;
-import org.codehaus.jackson.node.ContainerNode;
import org.junit.Test;
import java.io.IOException;
@@ -47,7 +45,6 @@ import java.util.Vector;
public class TestLog4Json extends TestCase {
private static final Log LOG = LogFactory.getLog(TestLog4Json.class);
- private static final JsonFactory factory = new MappingJsonFactory();
@Test
public void testConstruction() throws Throwable {
@@ -106,7 +103,7 @@ public class TestLog4Json extends TestCase {
//rather than try and make assertions about the format of the text
//message equalling another ISO date, this test asserts that the hypen
//and colon characters are in the string.
- String dateText = node.getTextValue();
+ String dateText = node.textValue();
assertTrue("No '-' in " + dateText, dateText.contains("-"));
assertTrue("No '-' in " + dateText, dateText.contains(":"));
@@ -141,7 +138,7 @@ public class TestLog4Json extends TestCase {
void assertEntryEquals(ContainerNode rootNode, String key, String value) {
JsonNode node = assertNodeContains(rootNode, key);
- assertEquals(value, node.getTextValue());
+ assertEquals(value, node.textValue());
}
private JsonNode assertNodeContains(ContainerNode rootNode, String key) {
@@ -154,7 +151,7 @@ public class TestLog4Json extends TestCase {
void assertEntryEquals(ContainerNode rootNode, String key, long value) {
JsonNode node = assertNodeContains(rootNode, key);
- assertEquals(value, node.getNumberValue());
+ assertEquals(value, node.numberValue());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
index 361d7fd..f62a6ce 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.security.token.delegation.web;
import static org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.DelegationTokenOperation.*;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
@@ -28,7 +30,6 @@ import org.apache.hadoop.security.authentication.server.AuthenticationHandler;
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.Token;
-import org.codehaus.jackson.map.ObjectMapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
index 44241de..89f15da 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.security.token.delegation.web;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.minikdc.MiniKdc;
@@ -30,7 +31,6 @@ import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHa
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
-import org.codehaus.jackson.map.ObjectMapper;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.servlet.ServletContextHandler;
@@ -60,8 +60,6 @@ import java.io.File;
import java.io.IOException;
import java.io.Writer;
import java.net.HttpURLConnection;
-import java.net.InetAddress;
-import java.net.ServerSocket;
import java.net.URL;
import java.security.Principal;
import java.security.PrivilegedActionException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHttpExceptionUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHttpExceptionUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHttpExceptionUtils.java
index 3790c43..1e29a30 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHttpExceptionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHttpExceptionUtils.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.util;
-import org.codehaus.jackson.map.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 5d0d214..73b8339 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -197,6 +197,10 @@
<artifactId>bcprov-jdk16</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
index d3e0064..14a5451 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
@@ -17,8 +17,9 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
import org.apache.hadoop.classification.InterfaceAudience;
-import org.codehaus.jackson.map.ObjectMapper;
import javax.ws.rs.Consumes;
import javax.ws.rs.WebApplicationException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
index 4b8a5be..f826572 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
@@ -17,9 +17,10 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.http.JettyUtils;
-import org.codehaus.jackson.map.ObjectMapper;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/48] hadoop git commit: HADOOP-13506. Redundant groupid warning in
child projects. Contributed by Kai Sasaki.
Posted by li...@apache.org.
HADOOP-13506. Redundant groupid warning in child projects. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/209e8054
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/209e8054
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/209e8054
Branch: refs/heads/HADOOP-13345
Commit: 209e80543075ab7bf759886e1cb259516e34b792
Parents: 9e173d2
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Nov 28 14:34:57 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Nov 28 14:34:57 2016 +0900
----------------------------------------------------------------------
hadoop-assemblies/pom.xml | 1 -
hadoop-client/pom.xml | 1 -
hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml | 1 -
hadoop-cloud-storage-project/pom.xml | 1 -
hadoop-common-project/hadoop-annotations/pom.xml | 1 -
hadoop-common-project/hadoop-auth-examples/pom.xml | 1 -
hadoop-common-project/hadoop-auth/pom.xml | 1 -
hadoop-common-project/hadoop-common/pom.xml | 1 -
hadoop-common-project/hadoop-kms/pom.xml | 1 -
hadoop-common-project/hadoop-minikdc/pom.xml | 1 -
hadoop-common-project/hadoop-nfs/pom.xml | 1 -
hadoop-common-project/pom.xml | 1 -
hadoop-dist/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 -
hadoop-hdfs-project/pom.xml | 1 -
.../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml | 1 -
.../hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml | 1 -
.../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 1 -
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 1 -
.../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml | 1 -
.../hadoop-mapreduce-client-jobclient/pom.xml | 1 -
.../hadoop-mapreduce-client-nativetask/pom.xml | 1 -
.../hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml | 1 -
hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 1 -
hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml | 1 -
hadoop-mapreduce-project/pom.xml | 1 -
hadoop-maven-plugins/pom.xml | 1 -
hadoop-minicluster/pom.xml | 1 -
hadoop-project-dist/pom.xml | 1 -
hadoop-project/pom.xml | 1 -
hadoop-tools/hadoop-archive-logs/pom.xml | 1 -
hadoop-tools/hadoop-archives/pom.xml | 1 -
hadoop-tools/hadoop-azure-datalake/pom.xml | 1 -
hadoop-tools/hadoop-datajoin/pom.xml | 1 -
hadoop-tools/hadoop-distcp/pom.xml | 1 -
hadoop-tools/hadoop-extras/pom.xml | 1 -
hadoop-tools/hadoop-gridmix/pom.xml | 1 -
hadoop-tools/hadoop-pipes/pom.xml | 1 -
hadoop-tools/hadoop-rumen/pom.xml | 1 -
hadoop-tools/hadoop-sls/pom.xml | 1 -
hadoop-tools/hadoop-streaming/pom.xml | 1 -
hadoop-tools/hadoop-tools-dist/pom.xml | 1 -
hadoop-tools/pom.xml | 1 -
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml | 1 -
.../hadoop-yarn-applications-distributedshell/pom.xml | 1 -
.../hadoop-yarn-applications-unmanaged-am-launcher/pom.xml | 1 -
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml | 1 -
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml | 1 -
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml | 1 -
.../hadoop-yarn-server-applicationhistoryservice/pom.xml | 1 -
.../hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 1 -
.../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml | 1 -
.../hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml | 1 -
.../hadoop-yarn-server-sharedcachemanager/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml | 1 -
.../hadoop-yarn-server-timeline-pluginstorage/pom.xml | 1 -
.../hadoop-yarn-server-timelineservice-hbase-tests/pom.xml | 1 -
.../hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml | 1 -
.../hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml | 1 -
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml | 1 -
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml | 1 -
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml | 1 -
hadoop-yarn-project/hadoop-yarn/pom.xml | 1 -
hadoop-yarn-project/pom.xml | 1 -
68 files changed, 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-assemblies/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml
index 4ff11f6..75f83de 100644
--- a/hadoop-assemblies/pom.xml
+++ b/hadoop-assemblies/pom.xml
@@ -26,7 +26,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-assemblies</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop Assemblies</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml
index 988dd03..dbd4fac 100644
--- a/hadoop-client/pom.xml
+++ b/hadoop-client/pom.xml
@@ -21,7 +21,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project-dist</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
index 7993b83..14cfc76 100644
--- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
@@ -21,7 +21,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-cloud-storage</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-cloud-storage-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cloud-storage-project/pom.xml b/hadoop-cloud-storage-project/pom.xml
index 94d4c02..15f4346 100644
--- a/hadoop-cloud-storage-project/pom.xml
+++ b/hadoop-cloud-storage-project/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-cloud-storage-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Cloud Storage Project</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-common-project/hadoop-annotations/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index 0afa7e3..a75c291 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Annotations</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-common-project/hadoop-auth-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml
index d841ace..8c480ea 100644
--- a/hadoop-common-project/hadoop-auth-examples/pom.xml
+++ b/hadoop-common-project/hadoop-auth-examples/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth-examples</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>war</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-common-project/hadoop-auth/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index 5df4bdb..f3a03a4 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 596f872..f442b2c4 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Common</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 3eaaf52..5d0d214 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -25,7 +25,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>war</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-common-project/hadoop-minikdc/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-minikdc/pom.xml b/hadoop-common-project/hadoop-minikdc/pom.xml
index e75bc7f..ade1c03 100644
--- a/hadoop-common-project/hadoop-minikdc/pom.xml
+++ b/hadoop-common-project/hadoop-minikdc/pom.xml
@@ -22,7 +22,6 @@
<relativePath>../../hadoop-project</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop MiniKDC</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-common-project/hadoop-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml
index 027cf34..639a9e1 100644
--- a/hadoop-common-project/hadoop-nfs/pom.xml
+++ b/hadoop-common-project/hadoop-nfs/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-nfs</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-common-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml
index db54ae5..dfc87ea 100644
--- a/hadoop-common-project/pom.xml
+++ b/hadoop-common-project/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Common Project</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index ad30db9..30e6472 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-dist</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Distribution</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index 8ce5f3f..69be56f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -23,7 +23,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS Client</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 928ada9..5580e68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -25,7 +25,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-httpfs</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>war</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index fa38719..031b8a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -23,7 +23,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-native-client</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS Native Client</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index 90236c9..4eb35e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -23,7 +23,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-nfs</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS-NFS</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 5427db8..770dd39 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -23,7 +23,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-hdfs-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml
index a0993c9..3a7a7ad 100644
--- a/hadoop-hdfs-project/pom.xml
+++ b/hadoop-hdfs-project/pom.xml
@@ -23,7 +23,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS Project</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index 8634903..ed51084 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce App</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index f78e83b..9b33945 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Common</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 3c16c3d..38a7725 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Core</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
index eca7571..90569e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs-plugins</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce HistoryServer Plugins</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
index 1bcaf73..5df9494 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce HistoryServer</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index cd181fe..40030d8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce JobClient</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 0520245..89b23cc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-nativetask</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce NativeTask</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
index 41a5220..4ce7cb8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Shuffle</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
index b5041bd..5ebaeca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Client</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
index 3fbd487..20e3766 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-examples</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop MapReduce Examples</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-mapreduce-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index a1b437e..4b63162 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -21,7 +21,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index f82d750..51b8f91 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<packaging>maven-plugin</packaging>
<name>Apache Hadoop Maven Plugins</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-minicluster/pom.xml b/hadoop-minicluster/pom.xml
index 473ee35..23b086d 100644
--- a/hadoop-minicluster/pom.xml
+++ b/hadoop-minicluster/pom.xml
@@ -21,7 +21,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 7908d1a..e0d2be9 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Project Dist POM</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 112fa18..fa43afd 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -22,7 +22,6 @@
<artifactId>hadoop-main</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Project POM</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-archive-logs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/pom.xml b/hadoop-tools/hadoop-archive-logs/pom.xml
index 714f0d9..3efdb4b 100644
--- a/hadoop-tools/hadoop-archive-logs/pom.xml
+++ b/hadoop-tools/hadoop-archive-logs/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-archive-logs</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Archive Logs</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-archives/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml
index d7def7b..b6c8bf9 100644
--- a/hadoop-tools/hadoop-archives/pom.xml
+++ b/hadoop-tools/hadoop-archives/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-archives</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Archives</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-azure-datalake/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index eba0979..7f872c3 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure-datalake</artifactId>
<name>Apache Hadoop Azure Data Lake support</name>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-datajoin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml
index 40b165d..cf8796c 100644
--- a/hadoop-tools/hadoop-datajoin/pom.xml
+++ b/hadoop-tools/hadoop-datajoin/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-datajoin</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Data Join</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-distcp/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml
index a07406b..4f141e2 100644
--- a/hadoop-tools/hadoop-distcp/pom.xml
+++ b/hadoop-tools/hadoop-distcp/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Distributed Copy</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-extras/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml
index fbb75a7..ab73cbd 100644
--- a/hadoop-tools/hadoop-extras/pom.xml
+++ b/hadoop-tools/hadoop-extras/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-extras</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Extras</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-gridmix/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml
index a1a8b0c..9e5be96 100644
--- a/hadoop-tools/hadoop-gridmix/pom.xml
+++ b/hadoop-tools/hadoop-gridmix/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-gridmix</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Gridmix</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-pipes/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml
index 5be72e7..b16ffc1 100644
--- a/hadoop-tools/hadoop-pipes/pom.xml
+++ b/hadoop-tools/hadoop-pipes/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-pipes</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Pipes</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-rumen/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml
index bc873f7..ff4cbcb 100644
--- a/hadoop-tools/hadoop-rumen/pom.xml
+++ b/hadoop-tools/hadoop-rumen/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-rumen</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Rumen</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-sls/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml
index b84f5fc..f31e415 100644
--- a/hadoop-tools/hadoop-sls/pom.xml
+++ b/hadoop-tools/hadoop-sls/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-sls</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Scheduler Load Simulator</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml
index b0eb5df..b97fbc0 100644
--- a/hadoop-tools/hadoop-streaming/pom.xml
+++ b/hadoop-tools/hadoop-streaming/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-streaming</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop MapReduce Streaming</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 14fa9f0..a8e2066 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-tools-dist</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Tools Dist</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index e7e876b..9dffe01 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-tools</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Tools</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index d36ae58..f3fdf09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN API</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index 0f696d8..e23dbae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications-distributedshell</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN DistributedShell</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
index ea3f294..c326c99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications-unmanaged-am-launcher</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Unmanaged Am Launcher</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index a1ff36d..233a353 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Applications</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index a53ec57..5118303 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -19,7 +19,6 @@
<groupId>org.apache.hadoop</groupId>
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Client</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 12b53af..c76e775 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Common</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
index 15f6511..165798e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
@@ -25,7 +25,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN ApplicationHistoryService</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index ee892d1..43ea7ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Server Common</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index acc1d50..23ceaaf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN NodeManager</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 0fcd535..f7d22a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN ResourceManager</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
index 4918276..5d6e46c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
@@ -19,7 +19,6 @@
<groupId>org.apache.hadoop</groupId>
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-sharedcachemanager</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN SharedCacheManager</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
index 359c706..97e92dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
@@ -21,7 +21,6 @@
<groupId>org.apache.hadoop</groupId>
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Server Tests</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
index 90cf55b..f7cd9be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
@@ -25,7 +25,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Timeline Plugin Storage</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
index c4786c7..026ef75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -25,7 +25,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice-hbase-tests</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Timeline Service HBase tests</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index 0092aed..6b535c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -25,7 +25,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Timeline Service</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
index cb9759b..c66aa73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Web Proxy</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index 416c511..a35e7c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Server</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
index 6436109..7fe88fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
@@ -22,7 +22,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-site</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Site</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
index 5211fd7..88648ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
@@ -23,7 +23,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-ui</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN UI</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 70b68d7..c929292 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -19,7 +19,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209e8054/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 380b515..a20ef55 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -21,7 +21,6 @@
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>pom</packaging>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/48] hadoop git commit: HADOOP-13820. Replace ugi.getUsername()
with ugi.getShortUserName() in viewFS. Contributed By Brahma Reddy Battula.
Posted by li...@apache.org.
HADOOP-13820. Replace ugi.getUsername() with ugi.getShortUserName() in viewFS. Contributed By Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee3d437a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee3d437a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee3d437a
Branch: refs/heads/HADOOP-13345
Commit: ee3d437a33c2106ea416078e722ff6184106a0bc
Parents: bb1b5e0
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Fri Nov 25 19:54:50 2016 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Fri Nov 25 19:54:50 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 8 +++----
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 +++++------
.../fs/viewfs/ViewFileSystemBaseTest.java | 20 ++++++++++++++++++
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 22 ++++++++++++++++++++
4 files changed, 52 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3d437a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index fa9f712..9061b2a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -951,7 +951,7 @@ public class ViewFileSystem extends FileSystem {
public FileStatus getFileStatus(Path f) throws IOException {
checkPathIsSlash(f);
return new FileStatus(0, true, 0, 0, creationTime, creationTime,
- PERMISSION_555, ugi.getUserName(), ugi.getPrimaryGroupName(),
+ PERMISSION_555, ugi.getShortUserName(), ugi.getPrimaryGroupName(),
new Path(theInternalDir.fullPath).makeQualified(
myUri, ROOT_PATH));
@@ -972,14 +972,14 @@ public class ViewFileSystem extends FileSystem {
result[i++] = new FileStatus(0, false, 0, 0,
creationTime, creationTime, PERMISSION_555,
- ugi.getUserName(), ugi.getPrimaryGroupName(),
+ ugi.getShortUserName(), ugi.getPrimaryGroupName(),
link.getTargetLink(),
new Path(inode.fullPath).makeQualified(
myUri, null));
} else {
result[i++] = new FileStatus(0, true, 0, 0,
creationTime, creationTime, PERMISSION_555,
- ugi.getUserName(), ugi.getGroupNames()[0],
+ ugi.getShortUserName(), ugi.getGroupNames()[0],
new Path(inode.fullPath).makeQualified(
myUri, null));
}
@@ -1103,7 +1103,7 @@ public class ViewFileSystem extends FileSystem {
@Override
public AclStatus getAclStatus(Path path) throws IOException {
checkPathIsSlash(path);
- return new AclStatus.Builder().owner(ugi.getUserName())
+ return new AclStatus.Builder().owner(ugi.getShortUserName())
.group(ugi.getPrimaryGroupName())
.addEntries(AclUtil.getMinimalAcl(PERMISSION_555))
.stickyBit(false).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3d437a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 6195540..d5f8e81 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -845,7 +845,7 @@ public class ViewFs extends AbstractFileSystem {
public FileStatus getFileStatus(final Path f) throws IOException {
checkPathIsSlash(f);
return new FileStatus(0, true, 0, 0, creationTime, creationTime,
- PERMISSION_555, ugi.getUserName(), ugi.getPrimaryGroupName(),
+ PERMISSION_555, ugi.getShortUserName(), ugi.getPrimaryGroupName(),
new Path(theInternalDir.fullPath).makeQualified(
myUri, null));
}
@@ -865,13 +865,13 @@ public class ViewFs extends AbstractFileSystem {
INodeLink<AbstractFileSystem> inodelink =
(INodeLink<AbstractFileSystem>) inode;
result = new FileStatus(0, false, 0, 0, creationTime, creationTime,
- PERMISSION_555, ugi.getUserName(), ugi.getPrimaryGroupName(),
+ PERMISSION_555, ugi.getShortUserName(), ugi.getPrimaryGroupName(),
inodelink.getTargetLink(),
new Path(inode.fullPath).makeQualified(
myUri, null));
} else {
result = new FileStatus(0, true, 0, 0, creationTime, creationTime,
- PERMISSION_555, ugi.getUserName(), ugi.getPrimaryGroupName(),
+ PERMISSION_555, ugi.getShortUserName(), ugi.getPrimaryGroupName(),
new Path(inode.fullPath).makeQualified(
myUri, null));
}
@@ -910,14 +910,14 @@ public class ViewFs extends AbstractFileSystem {
result[i++] = new FileStatus(0, false, 0, 0,
creationTime, creationTime,
- PERMISSION_555, ugi.getUserName(), ugi.getPrimaryGroupName(),
+ PERMISSION_555, ugi.getShortUserName(), ugi.getPrimaryGroupName(),
link.getTargetLink(),
new Path(inode.fullPath).makeQualified(
myUri, null));
} else {
result[i++] = new FileStatus(0, true, 0, 0,
creationTime, creationTime,
- PERMISSION_555, ugi.getUserName(), ugi.getGroupNames()[0],
+ PERMISSION_555, ugi.getShortUserName(), ugi.getGroupNames()[0],
new Path(inode.fullPath).makeQualified(
myUri, null));
}
@@ -1043,7 +1043,7 @@ public class ViewFs extends AbstractFileSystem {
@Override
public AclStatus getAclStatus(Path path) throws IOException {
checkPathIsSlash(path);
- return new AclStatus.Builder().owner(ugi.getUserName())
+ return new AclStatus.Builder().owner(ugi.getShortUserName())
.group(ugi.getPrimaryGroupName())
.addEntries(AclUtil.getMinimalAcl(PERMISSION_555))
.stickyBit(false).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3d437a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index 407b885..06f9868 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.viewfs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
+import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
@@ -1088,4 +1089,23 @@ abstract public class ViewFileSystemBaseTest {
// Case 7: Verify FsStatus of a non MountPoint path throws exception
ViewFileSystemUtil.getStatus(fileSystem, new Path("/non-existing"));
}
+
+ @Test
+ public void testCheckOwnerWithFileStatus()
+ throws IOException, InterruptedException {
+ final UserGroupInformation userUgi = UserGroupInformation
+ .createUserForTesting("user@HADOOP.COM", new String[]{"hadoop"});
+ userUgi.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws IOException {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String doAsUserName = ugi.getUserName();
+ assertEquals(doAsUserName, "user@HADOOP.COM");
+ FileSystem vfs = FileSystem.get(FsConstants.VIEWFS_URI, conf);
+ FileStatus stat = vfs.getFileStatus(new Path("/internalDir"));
+ assertEquals(userUgi.getShortUserName(), stat.getOwner());
+ return null;
+ }
+ });
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3d437a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
index ceebb26..16d2644 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
@@ -29,6 +29,8 @@ import static org.junit.Assert.assertFalse;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
@@ -798,4 +800,24 @@ abstract public class ViewFsBaseTest {
public void testInternalDeleteSnapshot() throws IOException {
fcView.deleteSnapshot(new Path("/internalDir"), "snap1");
}
+
+ @Test
+ public void testOwnerForInternalDir()
+ throws IOException, InterruptedException, URISyntaxException {
+ final UserGroupInformation userUgi = UserGroupInformation
+ .createUserForTesting("user@HADOOP.COM", new String[]{"hadoop"});
+ userUgi.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws IOException, URISyntaxException {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String doAsUserName = ugi.getUserName();
+ assertEquals(doAsUserName, "user@HADOOP.COM");
+ FileContext
+ viewFS = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf);
+ FileStatus stat = viewFS.getFileStatus(new Path("/internalDir"));
+ assertEquals(userUgi.getShortUserName(), stat.getOwner());
+ return null;
+ }
+ });
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/48] hadoop git commit: HADOOP-13766. Fix a typo in the comments
of RPC.getProtocolVersion. Contributed by Ethan Li.
Posted by li...@apache.org.
HADOOP-13766. Fix a typo in the comments of RPC.getProtocolVersion. Contributed by Ethan Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7a5f290
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7a5f290
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7a5f290
Branch: refs/heads/HADOOP-13345
Commit: c7a5f2906fcfd073a402b6981b091bd6d9b80294
Parents: 860d49a
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Nov 23 17:08:38 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Nov 23 17:08:38 2016 -0800
----------------------------------------------------------------------
.../hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5f290/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index a62748e..e16a8f5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -158,8 +158,9 @@ public class RPC {
/**
* Get the protocol version from protocol class.
- * If the protocol class has a ProtocolAnnotation, then get the protocol
- * name from the annotation; otherwise the class name is the protocol name.
+ * If the protocol class has a ProtocolAnnotation,
+ * then get the protocol version from the annotation;
+ * otherwise get it from the versionID field of the protocol class.
*/
static public long getProtocolVersion(Class<?> protocol) {
if (protocol == null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org