You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/01/11 18:19:17 UTC
[01/50] [abbrv] hadoop git commit: HDFS-11279. Cleanup unused
DataNode#checkDiskErrorAsync(). Contributed by Hanisha Koneru
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 a5cc315db -> 0c61010de
HDFS-11279. Cleanup unused DataNode#checkDiskErrorAsync(). Contributed by Hanisha Koneru
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87bb1c49
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87bb1c49
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87bb1c49
Branch: refs/heads/HADOOP-13345
Commit: 87bb1c49bb25f75b040028b1cebe3bc5251836d1
Parents: 8fadd69
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Jan 3 18:25:46 2017 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Jan 3 18:25:46 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/DataNode.java | 19 ------
.../datanode/checker/DatasetVolumeChecker.java | 69 --------------------
.../hdfs/server/datanode/DataNodeTestUtils.java | 15 +++++
.../datanode/TestDataNodeHotSwapVolumes.java | 18 +----
.../datanode/TestDataNodeVolumeFailure.java | 17 ++---
.../checker/TestDatasetVolumeChecker.java | 47 -------------
6 files changed, 26 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87bb1c49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index e893c5e..28d627a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -2051,25 +2051,6 @@ public class DataNode extends ReconfigurableBase
* Check if there is a disk failure asynchronously
* and if so, handle the error.
*/
- @VisibleForTesting
- public void checkDiskErrorAsync() {
- volumeChecker.checkAllVolumesAsync(
- data, (healthyVolumes, failedVolumes) -> {
- if (failedVolumes.size() > 0) {
- LOG.warn("checkDiskErrorAsync callback got {} failed volumes: {}",
- failedVolumes.size(), failedVolumes);
- } else {
- LOG.debug("checkDiskErrorAsync: no volume failures detected");
- }
- lastDiskErrorCheck = Time.monotonicNow();
- handleVolumeFailures(failedVolumes);
- });
- }
-
- /**
- * Check if there is a disk failure asynchronously
- * and if so, handle the error.
- */
public void checkDiskErrorAsync(FsVolumeSpi volume) {
volumeChecker.checkVolume(
volume, (healthyVolumes, failedVolumes) -> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87bb1c49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
index cab6122..9ad47f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
@@ -233,68 +233,6 @@ public class DatasetVolumeChecker {
}
/**
- * Start checks against all volumes of a dataset, invoking the
- * given callback when the operation has completed. The function
- * does not wait for the checks to complete.
- *
- * If a volume cannot be referenced then it is already closed and
- * cannot be checked. No error is propagated to the callback for that
- * volume.
- *
- * @param dataset - FsDatasetSpi to be checked.
- * @param callback - Callback to be invoked when the checks are complete.
- * @return true if the check was scheduled and the callback will be invoked.
- * false if the check was not scheduled and the callback will not be
- * invoked.
- */
- public boolean checkAllVolumesAsync(
- final FsDatasetSpi<? extends FsVolumeSpi> dataset,
- Callback callback) {
- final long gap = timer.monotonicNow() - lastAllVolumesCheck;
- if (gap < minDiskCheckGapMs) {
- numSkippedChecks.incrementAndGet();
- LOG.trace(
- "Skipped checking all volumes, time since last check {} is less " +
- "than the minimum gap between checks ({} ms).",
- gap, minDiskCheckGapMs);
- return false;
- }
-
- final FsDatasetSpi.FsVolumeReferences references =
- dataset.getFsVolumeReferences();
-
- if (references.size() == 0) {
- LOG.warn("checkAllVolumesAsync - no volumes can be referenced");
- return false;
- }
-
- lastAllVolumesCheck = timer.monotonicNow();
- final Set<FsVolumeSpi> healthyVolumes = new HashSet<>();
- final Set<FsVolumeSpi> failedVolumes = new HashSet<>();
- final AtomicLong numVolumes = new AtomicLong(references.size());
- boolean added = false;
-
- LOG.info("Checking {} volumes", references.size());
- for (int i = 0; i < references.size(); ++i) {
- final FsVolumeReference reference = references.getReference(i);
- // The context parameter is currently ignored.
- Optional<ListenableFuture<VolumeCheckResult>> olf =
- delegateChecker.schedule(reference.getVolume(), IGNORED_CONTEXT);
- if (olf.isPresent()) {
- added = true;
- Futures.addCallback(olf.get(),
- new ResultHandler(reference, healthyVolumes, failedVolumes,
- numVolumes, callback));
- } else {
- IOUtils.cleanup(null, reference);
- numVolumes.decrementAndGet();
- }
- }
- numAsyncDatasetChecks.incrementAndGet();
- return added;
- }
-
- /**
* A callback interface that is supplied the result of running an
* async disk check on multiple volumes.
*/
@@ -489,13 +427,6 @@ public class DatasetVolumeChecker {
}
/**
- * Return the number of {@link #checkAllVolumesAsync} invocations.
- */
- public long getNumAsyncDatasetChecks() {
- return numAsyncDatasetChecks.get();
- }
-
- /**
* Return the number of checks skipped because the minimum gap since the
* last check had not elapsed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87bb1c49/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
index 5a1ad87..cf5b724 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
import org.mockito.Mockito;
@@ -242,4 +243,18 @@ public class DataNodeTestUtils {
LOG.warn("Could not reconfigure DataNode.", e);
}
}
+
+ /** Get the FsVolume on the given basePath. */
+ public static FsVolumeImpl getVolume(DataNode dn, File basePath) throws
+ IOException {
+ try (FsDatasetSpi.FsVolumeReferences volumes = dn.getFSDataset()
+ .getFsVolumeReferences()) {
+ for (FsVolumeSpi vol : volumes) {
+ if (vol.getBaseURI().equals(basePath.toURI())) {
+ return (FsVolumeImpl) vol;
+ }
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87bb1c49/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 0401a81..80ca0ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -899,20 +899,6 @@ public class TestDataNodeHotSwapVolumes {
is(dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY)));
}
- /** Get the FsVolume on the given basePath */
- private FsVolumeImpl getVolume(DataNode dn, File basePath)
- throws IOException {
- try (FsDatasetSpi.FsVolumeReferences volumes =
- dn.getFSDataset().getFsVolumeReferences()) {
- for (FsVolumeSpi vol : volumes) {
- if (vol.getBaseURI().equals(basePath.toURI())) {
- return (FsVolumeImpl) vol;
- }
- }
- }
- return null;
- }
-
/**
* Verify that {@link DataNode#checkDiskError()} removes all metadata in
* DataNode upon a volume failure. Thus we can run reconfig on the same
@@ -933,7 +919,7 @@ public class TestDataNodeHotSwapVolumes {
final String oldDataDir = dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY);
File dirToFail = new File(cluster.getDataDirectory(), "data1");
- FsVolumeImpl failedVolume = getVolume(dn, dirToFail);
+ FsVolumeImpl failedVolume = DataNodeTestUtils.getVolume(dn, dirToFail);
assertTrue("No FsVolume was found for " + dirToFail,
failedVolume != null);
long used = failedVolume.getDfsUsed();
@@ -957,7 +943,7 @@ public class TestDataNodeHotSwapVolumes {
is(dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY)));
createFile(new Path("/test2"), 32, (short)2);
- FsVolumeImpl restoredVolume = getVolume(dn, dirToFail);
+ FsVolumeImpl restoredVolume = DataNodeTestUtils.getVolume(dn, dirToFail);
assertTrue(restoredVolume != null);
assertTrue(restoredVolume != failedVolume);
// More data has been written to this volume.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87bb1c49/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
index 06e2871..970b83b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
@@ -237,7 +237,7 @@ public class TestDataNodeVolumeFailure {
File dn0Vol1 = new File(dataDir, "data" + (2 * 0 + 1));
DataNodeTestUtils.injectDataDirFailure(dn0Vol1);
DataNode dn0 = cluster.getDataNodes().get(0);
- checkDiskErrorSync(dn0);
+ checkDiskErrorSync(dn0, DataNodeTestUtils.getVolume(dn0, dn0Vol1));
// Verify dn0Vol1 has been completely removed from DN0.
// 1. dn0Vol1 is removed from DataStorage.
@@ -284,10 +284,10 @@ public class TestDataNodeVolumeFailure {
assertFalse(dataDirStrs[0].contains(dn0Vol1.getAbsolutePath()));
}
- private static void checkDiskErrorSync(DataNode dn)
+ private static void checkDiskErrorSync(DataNode dn, FsVolumeSpi volume)
throws InterruptedException {
final long lastDiskErrorCheck = dn.getLastDiskErrorCheck();
- dn.checkDiskErrorAsync();
+ dn.checkDiskErrorAsync(volume);
// Wait 10 seconds for checkDiskError thread to finish and discover volume
// failures.
int count = 100;
@@ -311,7 +311,8 @@ public class TestDataNodeVolumeFailure {
final File dn0Vol2 = new File(dataDir, "data" + (2 * 0 + 2));
DataNodeTestUtils.injectDataDirFailure(dn0Vol1, dn0Vol2);
DataNode dn0 = cluster.getDataNodes().get(0);
- checkDiskErrorSync(dn0);
+ checkDiskErrorSync(dn0, DataNodeTestUtils.getVolume(dn0, dn0Vol1));
+ checkDiskErrorSync(dn0, DataNodeTestUtils.getVolume(dn0, dn0Vol2));
// DN0 should stop after the number of failure disks exceed tolerated
// value (1).
@@ -332,7 +333,7 @@ public class TestDataNodeVolumeFailure {
// Fail dn0Vol1 first.
DataNodeTestUtils.injectDataDirFailure(dn0Vol1);
- checkDiskErrorSync(dn0);
+ checkDiskErrorSync(dn0, DataNodeTestUtils.getVolume(dn0, dn0Vol1));
// Hot swap out the failure volume.
String dataDirs = dn0Vol2.getPath();
@@ -351,7 +352,7 @@ public class TestDataNodeVolumeFailure {
// Fail dn0Vol2. Now since dn0Vol1 has been fixed, DN0 has sufficient
// resources, thus it should keep running.
DataNodeTestUtils.injectDataDirFailure(dn0Vol2);
- checkDiskErrorSync(dn0);
+ checkDiskErrorSync(dn0, DataNodeTestUtils.getVolume(dn0, dn0Vol2));
assertTrue(dn0.shouldRun());
}
@@ -378,12 +379,12 @@ public class TestDataNodeVolumeFailure {
// Fail dn0Vol1 first and hot swap it.
DataNodeTestUtils.injectDataDirFailure(dn0Vol1);
- checkDiskErrorSync(dn0);
+ checkDiskErrorSync(dn0, DataNodeTestUtils.getVolume(dn0, dn0Vol1));
assertTrue(dn0.shouldRun());
// Fail dn0Vol2, now dn0 should stop, because we only tolerate 1 disk failure.
DataNodeTestUtils.injectDataDirFailure(dn0Vol2);
- checkDiskErrorSync(dn0);
+ checkDiskErrorSync(dn0, DataNodeTestUtils.getVolume(dn0, dn0Vol2));
assertFalse(dn0.shouldRun());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87bb1c49/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java
index f5bb807..b37cc75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java
@@ -162,53 +162,6 @@ public class TestDatasetVolumeChecker {
}
/**
- * Unit test for {@link DatasetVolumeChecker#checkAllVolumesAsync}.
- *
- * @throws Exception
- */
- @Test(timeout=10000)
- public void testCheckAllVolumesAsync() throws Exception {
- LOG.info("Executing {}", testName.getMethodName());
-
- final List<FsVolumeSpi> volumes = makeVolumes(
- NUM_VOLUMES, expectedVolumeHealth);
- final FsDatasetSpi<FsVolumeSpi> dataset = makeDataset(volumes);
- final DatasetVolumeChecker checker =
- new DatasetVolumeChecker(new HdfsConfiguration(), new FakeTimer());
- checker.setDelegateChecker(new DummyChecker());
- final AtomicLong numCallbackInvocations = new AtomicLong(0);
-
- boolean result = checker.checkAllVolumesAsync(
- dataset, new DatasetVolumeChecker.Callback() {
- @Override
- public void call(
- Set<FsVolumeSpi> healthyVolumes,
- Set<FsVolumeSpi> failedVolumes) {
- LOG.info("Got back {} failed volumes", failedVolumes.size());
- if (expectedVolumeHealth == null ||
- expectedVolumeHealth == FAILED) {
- assertThat(healthyVolumes.size(), is(0));
- assertThat(failedVolumes.size(), is(NUM_VOLUMES));
- } else {
- assertThat(healthyVolumes.size(), is(NUM_VOLUMES));
- assertThat(failedVolumes.size(), is(0));
- }
- numCallbackInvocations.incrementAndGet();
- }
- });
-
- // The callback should be invoked exactly once.
- if (result) {
- assertThat(numCallbackInvocations.get(), is(1L));
- }
-
- // Ensure each volume's check() method was called exactly once.
- for (FsVolumeSpi volume : volumes) {
- verify(volume, times(1)).check(anyObject());
- }
- }
-
- /**
* A checker to wraps the result of {@link FsVolumeSpi#check} in
* an ImmediateFuture.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-6074. FlowRunEntity does not
deserialize long values correctly (Rohith Sharma K S via Varun Saxena)
Posted by st...@apache.org.
YARN-6074. FlowRunEntity does not deserialize long values correctly (Rohith Sharma K S via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db490ecc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db490ecc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db490ecc
Branch: refs/heads/HADOOP-13345
Commit: db490eccced3c42ac27253ca6cbaf10a77e0e116
Parents: f59e36b
Author: Varun Saxena <va...@apache.org>
Authored: Mon Jan 9 18:54:22 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Mon Jan 9 18:54:22 2017 +0530
----------------------------------------------------------------------
.../hadoop/yarn/api/records/timelineservice/FlowRunEntity.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db490ecc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
index 410a1bf..5c666cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
@@ -100,7 +100,7 @@ public class FlowRunEntity extends HierarchicalTimelineEntity {
public long getRunId() {
Object runId = getInfo().get(FLOW_RUN_ID_INFO_KEY);
- return runId == null ? 0L : (Long) runId;
+ return runId == null ? 0L : ((Number) runId).longValue();
}
public void setRunId(long runId) {
@@ -117,7 +117,7 @@ public class FlowRunEntity extends HierarchicalTimelineEntity {
public long getMaxEndTime() {
Object time = getInfo().get(FLOW_RUN_END_TIME);
- return time == null ? 0L : (Long)time;
+ return time == null ? 0L : ((Number) time).longValue();
}
public void setMaxEndTime(long endTime) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDFS-9391. Update webUI/JMX to
display maintenance state info. (Manoj Govindassamy via mingma)
Posted by st...@apache.org.
HDFS-9391. Update webUI/JMX to display maintenance state info. (Manoj Govindassamy via mingma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/467f5f17
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/467f5f17
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/467f5f17
Branch: refs/heads/HADOOP-13345
Commit: 467f5f1735494c5ef74e6591069884d3771c17e4
Parents: 4db119b
Author: Ming Ma <mi...@apache.org>
Authored: Tue Jan 10 20:12:42 2017 -0800
Committer: Ming Ma <mi...@apache.org>
Committed: Tue Jan 10 20:12:42 2017 -0800
----------------------------------------------------------------------
.../blockmanagement/DatanodeDescriptor.java | 12 +-
.../blockmanagement/DecommissionManager.java | 9 +-
.../server/blockmanagement/NumberReplicas.java | 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 29 ++++-
.../hdfs/server/namenode/NameNodeMXBean.java | 9 +-
.../src/main/webapps/hdfs/dfshealth.html | 32 ++++-
.../src/main/webapps/hdfs/dfshealth.js | 11 +-
.../server/namenode/TestNameNodeMXBean.java | 125 +++++++++++++++++--
8 files changed, 205 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/467f5f17/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 320c680..8ab4bba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -730,7 +730,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
// Super implementation is sufficient
return super.hashCode();
}
-
+
@Override
public boolean equals(Object obj) {
// Sufficient to use super equality as datanodes are uniquely identified
@@ -745,14 +745,14 @@ public class DatanodeDescriptor extends DatanodeInfo {
private int underReplicatedInOpenFiles;
private long startTime;
- synchronized void set(int underRep,
- int onlyRep, int underConstruction) {
+ synchronized void set(int underRepInOpenFiles, int underRepBlocks,
+ int outOfServiceOnlyRep) {
if (!isDecommissionInProgress() && !isEnteringMaintenance()) {
return;
}
- underReplicatedBlocks = underRep;
- outOfServiceOnlyReplicas = onlyRep;
- underReplicatedInOpenFiles = underConstruction;
+ underReplicatedInOpenFiles = underRepInOpenFiles;
+ underReplicatedBlocks = underRepBlocks;
+ outOfServiceOnlyReplicas = outOfServiceOnlyRep;
}
/** @return the number of under-replicated blocks */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/467f5f17/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index b1cfd78..ae79826 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -634,9 +634,12 @@ public class DecommissionManager {
final List<BlockInfo> insufficientList,
boolean pruneReliableBlocks) {
boolean firstReplicationLog = true;
+ // Low redundancy in UC Blocks only
+ int lowRedundancyInOpenFiles = 0;
+ // All low redundancy blocks. Includes lowRedundancyInOpenFiles.
int lowRedundancyBlocks = 0;
+ // All maintenance and decommission replicas.
int outOfServiceOnlyReplicas = 0;
- int lowRedundancyInOpenFiles = 0;
while (it.hasNext()) {
if (insufficientList == null
&& numBlocksCheckedPerLock >= numBlocksPerCheck) {
@@ -726,8 +729,8 @@ public class DecommissionManager {
}
}
- datanode.getLeavingServiceStatus().set(lowRedundancyBlocks,
- outOfServiceOnlyReplicas, lowRedundancyInOpenFiles);
+ datanode.getLeavingServiceStatus().set(lowRedundancyInOpenFiles,
+ lowRedundancyBlocks, outOfServiceOnlyReplicas);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/467f5f17/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/NumberReplicas.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/NumberReplicas.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/NumberReplicas.java
index be984f9..e61df87 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/NumberReplicas.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/NumberReplicas.java
@@ -77,7 +77,7 @@ public class NumberReplicas extends EnumCounters<NumberReplicas.StoredReplicaSta
* @return decommissioned and decommissioning replicas
*/
public int decommissionedAndDecommissioning() {
- return (int) (get(DECOMMISSIONED) + get(DECOMMISSIONING));
+ return decommissioned() + decommissioning();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/467f5f17/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 249324b..2a1f4d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5500,6 +5500,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
Map<String, Object> innerinfo = ImmutableMap.<String, Object>builder()
.put("lastContact", getLastContact(node))
.put("decommissioned", node.isDecommissioned())
+ .put("adminState", node.getAdminState().toString())
.put("xferaddr", node.getXferAddr())
.build();
info.put(node.getHostName() + ":" + node.getXferPort(), innerinfo);
@@ -5524,7 +5525,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
.put("xferaddr", node.getXferAddr())
.put("underReplicatedBlocks",
node.getLeavingServiceStatus().getUnderReplicatedBlocks())
- // TODO use another property name for outOfServiceOnlyReplicas.
.put("decommissionOnlyReplicas",
node.getLeavingServiceStatus().getOutOfServiceOnlyReplicas())
.put("underReplicateInOpenFiles",
@@ -5535,6 +5535,33 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return JSON.toString(info);
}
+ /**
+ * Returned information is a JSON representation of map with host name of
+ * nodes entering maintenance as the key and value as a map of various node
+ * attributes to its values.
+ */
+ @Override // NameNodeMXBean
+ public String getEnteringMaintenanceNodes() {
+ final Map<String, Map<String, Object>> nodesMap =
+ new HashMap<String, Map<String, Object>>();
+ final List<DatanodeDescriptor> enteringMaintenanceNodeList =
+ blockManager.getDatanodeManager().getEnteringMaintenanceNodes();
+ for (DatanodeDescriptor node : enteringMaintenanceNodeList) {
+ Map<String, Object> attrMap = ImmutableMap
+ .<String, Object> builder()
+ .put("xferaddr", node.getXferAddr())
+ .put("underReplicatedBlocks",
+ node.getLeavingServiceStatus().getUnderReplicatedBlocks())
+ .put("maintenanceOnlyReplicas",
+ node.getLeavingServiceStatus().getOutOfServiceOnlyReplicas())
+ .put("underReplicateInOpenFiles",
+ node.getLeavingServiceStatus().getUnderReplicatedInOpenFiles())
+ .build();
+ nodesMap.put(node.getHostName() + ":" + node.getXferPort(), attrMap);
+ }
+ return JSON.toString(nodesMap);
+ }
+
private long getLastContact(DatanodeDescriptor alivenode) {
return (monotonicNow() - alivenode.getLastUpdateMonotonic())/1000;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/467f5f17/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
index e796122..82cec33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
@@ -190,7 +190,14 @@ public interface NameNodeMXBean {
* @return the decommissioning node information
*/
public String getDecomNodes();
-
+
+ /**
+ * Gets the information on nodes entering maintenance.
+ *
+ * @return the information on nodes entering maintenance
+ */
+ String getEnteringMaintenanceNodes();
+
/**
* Gets the cluster id.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/467f5f17/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index d71e798..94516a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -171,9 +171,10 @@
{/nn}
{#fs}
- <tr><th><a href="#tab-datanode">Live Nodes</a></th><td>{NumLiveDataNodes} (Decommissioned: {NumDecomLiveDataNodes})</td></tr>
- <tr><th><a href="#tab-datanode">Dead Nodes</a></th><td>{NumDeadDataNodes} (Decommissioned: {NumDecomDeadDataNodes})</td></tr>
+ <tr><th><a href="#tab-datanode">Live Nodes</a></th><td>{NumLiveDataNodes} (Decommissioned: {NumDecomLiveDataNodes}, In Maintenance: {NumInMaintenanceLiveDataNodes})</td></tr>
+ <tr><th><a href="#tab-datanode">Dead Nodes</a></th><td>{NumDeadDataNodes} (Decommissioned: {NumDecomDeadDataNodes}, In Maintenance: {NumInMaintenanceDeadDataNodes})</td></tr>
<tr><th><a href="#tab-datanode">Decommissioning Nodes</a></th><td>{NumDecommissioningDataNodes}</td></tr>
+ <tr><th><a href="#tab-datanode">Entering Maintenance Nodes</a></th><td> {NumEnteringMaintenanceDataNodes}</td></tr>
<tr><th><a href="#tab-datanode-volume-failures">Total Datanode Volume Failures</a></th><td>{VolumeFailuresTotal} ({EstimatedCapacityLostTotal|fmt_bytes})</td></tr>
{@eq key=nnstat.State value="active"}
<tr><th title="Excludes missing blocks.">Number of Under-Replicated Blocks</th><td>{UnderReplicatedBlocks}</td></tr>
@@ -295,6 +296,7 @@
<li class="dfshealth-node-icon dfshealth-node-down">Down</li>
<li class="dfshealth-node-icon dfshealth-node-decommissioned">Decommissioned</li>
<li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Decommissioned & dead</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-maintenance">In Maintenance & dead</li>
</ul>
</div>
<div class="page-header"><h1><small>In operation</small></h1></div>
@@ -344,6 +346,32 @@
</table>
</small>
+<div class="page-header"><h1><small>Entering Maintenance</small></h1></div>
+<small>
+ {?EnteringMaintenanceNodes}
+ <table class="table">
+ <thead>
+ <tr>
+ <th>Node</th>
+ <th>Under replicated blocks</th>
+ <th>Blocks with no live replicas</th>
+ <th>Under Replicated Blocks <br/>In files under construction</th>
+ </tr>
+ </thead>
+ {#EnteringMaintenanceNodes}
+ <tr>
+ <td>{name} ({xferaddr})</td>
+ <td>{underReplicatedBlocks}</td>
+ <td>{maintenanceOnlyReplicas}</td>
+ <td>{underReplicateInOpenFiles}</td>
+ </tr>
+ {/EnteringMaintenanceNodes}
+ </table>
+ {:else}
+ No nodes are entering maintenance.
+ {/EnteringMaintenanceNodes}
+</small>
+
<div class="page-header"><h1><small>Decommissioning</small></h1></div>
<small>
{?DecomNodes}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/467f5f17/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
index 02aa895..e460381 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
@@ -223,17 +223,23 @@
if (n.adminState === "In Service") {
n.state = "alive";
} else if (nodes[i].adminState === "Decommission In Progress") {
- n.state = "decommisioning";
+ n.state = "decommissioning";
} else if (nodes[i].adminState === "Decommissioned") {
n.state = "decommissioned";
+ } else if (nodes[i].adminState === "Entering Maintenance") {
+ n.state = "entering-maintenance";
+ } else if (nodes[i].adminState === "In Maintenance") {
+ n.state = "in-maintenance";
}
}
}
function augment_dead_nodes(nodes) {
for (var i = 0, e = nodes.length; i < e; ++i) {
- if (nodes[i].decommissioned) {
+ if (nodes[i].adminState === "Decommissioned") {
nodes[i].state = "down-decommissioned";
+ } else if (nodes[i].adminState === "In Maintenance") {
+ nodes[i].state = "down-maintenance";
} else {
nodes[i].state = "down";
}
@@ -245,6 +251,7 @@
r.DeadNodes = node_map_to_array(JSON.parse(r.DeadNodes));
augment_dead_nodes(r.DeadNodes);
r.DecomNodes = node_map_to_array(JSON.parse(r.DecomNodes));
+ r.EnteringMaintenanceNodes = node_map_to_array(JSON.parse(r.EnteringMaintenanceNodes));
return r;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/467f5f17/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 4287a92..7322cba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -37,8 +37,10 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.CombinedHostFileManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.HostConfigManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
@@ -50,10 +52,13 @@ import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator;
import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.VersionInfo;
import org.junit.Assert;
import org.junit.Test;
import org.eclipse.jetty.util.ajax.JSON;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.management.MBeanServer;
import javax.management.ObjectName;
@@ -64,6 +69,7 @@ import java.net.BindException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -80,6 +86,9 @@ import static org.junit.Assert.fail;
*/
public class TestNameNodeMXBean {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestNameNodeMXBean.class);
+
/**
* Used to assert equality between doubles
*/
@@ -180,10 +189,10 @@ public class TestNameNodeMXBean {
assertFalse(xferAddr.equals(dnXferAddrInMaintenance) ^ inMaintenance);
}
assertEquals(fsn.getLiveNodes(), alivenodeinfo);
- // get attribute deadnodeinfo
- String deadnodeinfo = (String) (mbs.getAttribute(mxbeanName,
+ // get attributes DeadNodes
+ String deadNodeInfo = (String) (mbs.getAttribute(mxbeanName,
"DeadNodes"));
- assertEquals(fsn.getDeadNodes(), deadnodeinfo);
+ assertEquals(fsn.getDeadNodes(), deadNodeInfo);
// get attribute NodeUsage
String nodeUsage = (String) (mbs.getAttribute(mxbeanName,
"NodeUsage"));
@@ -295,16 +304,16 @@ public class TestNameNodeMXBean {
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
}
- // get attribute deadnodeinfo
- String deadnodeinfo = (String) (mbs.getAttribute(mxbeanName,
+ // get attribute DeadNodes
+ String deadNodeInfo = (String) (mbs.getAttribute(mxbeanName,
"DeadNodes"));
- assertEquals(fsn.getDeadNodes(), deadnodeinfo);
+ assertEquals(fsn.getDeadNodes(), deadNodeInfo);
Map<String, Map<String, Object>> deadNodes =
- (Map<String, Map<String, Object>>) JSON.parse(deadnodeinfo);
+ (Map<String, Map<String, Object>>) JSON.parse(deadNodeInfo);
assertTrue(deadNodes.size() > 0);
for (Map<String, Object> deadNode : deadNodes.values()) {
assertTrue(deadNode.containsKey("lastContact"));
- assertTrue(deadNode.containsKey("decommissioned"));
+ assertTrue(deadNode.containsKey("adminState"));
assertTrue(deadNode.containsKey("xferaddr"));
}
} finally {
@@ -415,6 +424,106 @@ public class TestNameNodeMXBean {
}
}
+ @Test (timeout = 120000)
+ public void testMaintenanceNodes() throws Exception {
+ LOG.info("Starting testMaintenanceNodes");
+ int expirationInMs = 30 * 1000;
+ Configuration conf = new Configuration();
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
+ expirationInMs);
+ conf.setClass(DFSConfigKeys.DFS_NAMENODE_HOSTS_PROVIDER_CLASSNAME_KEY,
+ CombinedHostFileManager.class, HostConfigManager.class);
+ MiniDFSCluster cluster = null;
+ HostsFileWriter hostsFileWriter = new HostsFileWriter();
+ hostsFileWriter.initialize(conf, "temp/TestNameNodeMXBean");
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+ cluster.waitActive();
+
+ FSNamesystem fsn = cluster.getNameNode().namesystem;
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName mxbeanName = new ObjectName(
+ "Hadoop:service=NameNode,name=NameNodeInfo");
+
+ List<String> hosts = new ArrayList<>();
+ for(DataNode dn : cluster.getDataNodes()) {
+ hosts.add(dn.getDisplayName());
+ }
+ hostsFileWriter.initIncludeHosts(hosts.toArray(
+ new String[hosts.size()]));
+ fsn.getBlockManager().getDatanodeManager().refreshNodes(conf);
+
+ // 1. Verify nodes for DatanodeReportType.LIVE state
+ String liveNodesInfo = (String) (mbs.getAttribute(mxbeanName,
+ "LiveNodes"));
+ LOG.info("Live Nodes: " + liveNodesInfo);
+ Map<String, Map<String, Object>> liveNodes =
+ (Map<String, Map<String, Object>>) JSON.parse(liveNodesInfo);
+ assertEquals(fsn.getLiveNodes(), liveNodesInfo);
+ assertEquals(fsn.getNumLiveDataNodes(), liveNodes.size());
+
+ for (Map<String, Object> liveNode : liveNodes.values()) {
+ assertTrue(liveNode.containsKey("lastContact"));
+ assertTrue(liveNode.containsKey("xferaddr"));
+ }
+
+ // Add the 1st DataNode to Maintenance list
+ Map<String, Long> maintenanceNodes = new HashMap<>();
+ maintenanceNodes.put(cluster.getDataNodes().get(0).getDisplayName(),
+ Time.monotonicNow() + expirationInMs);
+ hostsFileWriter.initOutOfServiceHosts(null, maintenanceNodes);
+ fsn.getBlockManager().getDatanodeManager().refreshNodes(conf);
+
+ boolean recheck = true;
+ while (recheck) {
+ // 2. Verify nodes for DatanodeReportType.ENTERING_MAINTENANCE state
+ String enteringMaintenanceNodesInfo =
+ (String) (mbs.getAttribute(mxbeanName, "EnteringMaintenanceNodes"));
+ Map<String, Map<String, Object>> enteringMaintenanceNodes =
+ (Map<String, Map<String, Object>>) JSON.parse(
+ enteringMaintenanceNodesInfo);
+ if (enteringMaintenanceNodes.size() <= 0) {
+ LOG.info("Waiting for a node to Enter Maintenance state!");
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+ continue;
+ }
+ LOG.info("Nodes entering Maintenance: " + enteringMaintenanceNodesInfo);
+ recheck = false;
+ assertEquals(fsn.getEnteringMaintenanceNodes(),
+ enteringMaintenanceNodesInfo);
+ assertEquals(fsn.getNumEnteringMaintenanceDataNodes(),
+ enteringMaintenanceNodes.size());
+ assertEquals(0, fsn.getNumInMaintenanceLiveDataNodes());
+ assertEquals(0, fsn.getNumInMaintenanceDeadDataNodes());
+ }
+
+ // Wait for the DecommissionManager to complete check
+ // and perform state transition
+ while (fsn.getNumInMaintenanceLiveDataNodes() != 1) {
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+ }
+
+ // 3. Verify nodes for AdminStates.IN_MAINTENANCE state
+ String enteringMaintenanceNodesInfo =
+ (String) (mbs.getAttribute(mxbeanName, "EnteringMaintenanceNodes"));
+ Map<String, Map<String, Object>> enteringMaintenanceNodes =
+ (Map<String, Map<String, Object>>) JSON.parse(
+ enteringMaintenanceNodesInfo);
+ assertEquals(0, enteringMaintenanceNodes.size());
+ assertEquals(fsn.getEnteringMaintenanceNodes(),
+ enteringMaintenanceNodesInfo);
+ assertEquals(1, fsn.getNumInMaintenanceLiveDataNodes());
+ assertEquals(0, fsn.getNumInMaintenanceDeadDataNodes());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ hostsFileWriter.cleanup();
+ }
+ }
+
@Test(timeout=120000)
@SuppressWarnings("unchecked")
public void testTopUsers() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HDFS-11273. Move
TransferFsImage#doGetUrl function to a Util class. Contributed by Hanisha
Koneru.
Posted by st...@apache.org.
HDFS-11273. Move TransferFsImage#doGetUrl function to a Util class. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ec609b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ec609b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ec609b2
Branch: refs/heads/HADOOP-13345
Commit: 7ec609b28989303fe0cc36812f225028b0251b32
Parents: 511d39e
Author: Jing Zhao <ji...@apache.org>
Authored: Mon Jan 9 18:00:14 2017 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Mon Jan 9 18:05:33 2017 -0800
----------------------------------------------------------------------
.../server/common/HttpGetFailedException.java | 39 +++
.../server/common/HttpPutFailedException.java | 37 +++
.../apache/hadoop/hdfs/server/common/Util.java | 258 +++++++++++++++-
.../server/namenode/EditLogFileInputStream.java | 3 +-
.../hdfs/server/namenode/ImageServlet.java | 15 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 4 +-
.../hdfs/server/namenode/TransferFsImage.java | 295 ++-----------------
7 files changed, 366 insertions(+), 285 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ec609b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HttpGetFailedException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HttpGetFailedException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HttpGetFailedException.java
new file mode 100644
index 0000000..f36aca3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HttpGetFailedException.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+
+@InterfaceAudience.Private
+public class HttpGetFailedException extends IOException {
+ private static final long serialVersionUID = 1L;
+ private final int responseCode;
+
+ public HttpGetFailedException(String msg, HttpURLConnection connection)
+ throws IOException {
+ super(msg);
+ this.responseCode = connection.getResponseCode();
+ }
+
+ public int getResponseCode() {
+ return responseCode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ec609b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HttpPutFailedException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HttpPutFailedException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HttpPutFailedException.java
new file mode 100644
index 0000000..5fad77d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HttpPutFailedException.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.io.IOException;
+
+@InterfaceAudience.Private
+public class HttpPutFailedException extends IOException {
+ private static final long serialVersionUID = 1L;
+ private final int responseCode;
+
+ public HttpPutFailedException(String msg, int responseCode) throws IOException {
+ super(msg);
+ this.responseCode = responseCode;
+ }
+
+ public int getResponseCode() {
+ return responseCode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ec609b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
index 6ec686f..f08c3fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
@@ -18,30 +18,66 @@
package org.apache.hadoop.hdfs.server.common;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URL;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.server.namenode.ImageServlet;
+import org.apache.hadoop.hdfs.util.DataTransferThrottler;
+import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+
@InterfaceAudience.Private
public final class Util {
private final static Log LOG = LogFactory.getLog(Util.class.getName());
+ public final static String FILE_LENGTH = "File-Length";
+ public final static String CONTENT_LENGTH = "Content-Length";
+ public final static String MD5_HEADER = "X-MD5-Digest";
+ public final static String CONTENT_TYPE = "Content-Type";
+ public final static String CONTENT_TRANSFER_ENCODING = "Content-Transfer-Encoding";
+
+ public final static int IO_FILE_BUFFER_SIZE;
+ private static final boolean isSpnegoEnabled;
+ public static final URLConnectionFactory connectionFactory;
+
+ static {
+ Configuration conf = new Configuration();
+ connectionFactory = URLConnectionFactory
+ .newDefaultURLConnectionFactory(conf);
+ isSpnegoEnabled = UserGroupInformation.isSecurityEnabled();
+ IO_FILE_BUFFER_SIZE = DFSUtilClient.getIoFileBufferSize(conf);
+ }
+
/**
* Interprets the passed string as a URI. In case of error it
* assumes the specified string is a file.
*
* @param s the string to interpret
- * @return the resulting URI
- * @throws IOException
+ * @return the resulting URI
*/
- public static URI stringAsURI(String s) throws IOException {
+ static URI stringAsURI(String s) throws IOException {
URI u = null;
// try to make a URI
try {
@@ -67,7 +103,6 @@ public final class Util {
*
* @param f the file to convert
* @return the resulting URI
- * @throws IOException
*/
public static URI fileAsURI(File f) throws IOException {
URI u = f.getCanonicalFile().toURI();
@@ -92,7 +127,7 @@ public final class Util {
*/
public static List<URI> stringCollectionAsURIs(
Collection<String> names) {
- List<URI> uris = new ArrayList<URI>(names.size());
+ List<URI> uris = new ArrayList<>(names.size());
for(String name : names) {
try {
uris.add(stringAsURI(name));
@@ -102,4 +137,217 @@ public final class Util {
}
return uris;
}
+
+ /**
+ * Downloads the files at the specified url location into destination
+ * storage.
+ */
+ public static MD5Hash doGetUrl(URL url, List<File> localPaths,
+ Storage dstStorage, boolean getChecksum, int timeout) throws IOException {
+ HttpURLConnection connection;
+ try {
+ connection = (HttpURLConnection)
+ connectionFactory.openConnection(url, isSpnegoEnabled);
+ } catch (AuthenticationException e) {
+ throw new IOException(e);
+ }
+
+ setTimeout(connection, timeout);
+
+ if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
+ throw new HttpGetFailedException("Image transfer servlet at " + url +
+ " failed with status code " + connection.getResponseCode() +
+ "\nResponse message:\n" + connection.getResponseMessage(),
+ connection);
+ }
+
+ long advertisedSize;
+ String contentLength = connection.getHeaderField(CONTENT_LENGTH);
+ if (contentLength != null) {
+ advertisedSize = Long.parseLong(contentLength);
+ } else {
+ throw new IOException(CONTENT_LENGTH + " header is not provided " +
+ "by the namenode when trying to fetch " + url);
+ }
+ MD5Hash advertisedDigest = parseMD5Header(connection);
+ String fsImageName = connection
+ .getHeaderField(ImageServlet.HADOOP_IMAGE_EDITS_HEADER);
+ InputStream stream = connection.getInputStream();
+
+ return receiveFile(url.toExternalForm(), localPaths, dstStorage,
+ getChecksum, advertisedSize, advertisedDigest, fsImageName, stream,
+ null);
+ }
+
+ /**
+ * Receives file at the url location from the input stream and puts them in
+ * the specified destination storage location.
+ */
+ public static MD5Hash receiveFile(String url, List<File> localPaths,
+ Storage dstStorage, boolean getChecksum, long advertisedSize,
+ MD5Hash advertisedDigest, String fsImageName, InputStream stream,
+ DataTransferThrottler throttler) throws
+ IOException {
+ long startTime = Time.monotonicNow();
+ Map<FileOutputStream, File> streamPathMap = new HashMap<>();
+ StringBuilder xferStats = new StringBuilder();
+ double xferCombined = 0;
+ if (localPaths != null) {
+ // If the local paths refer to directories, use the server-provided header
+ // as the filename within that directory
+ List<File> newLocalPaths = new ArrayList<>();
+ for (File localPath : localPaths) {
+ if (localPath.isDirectory()) {
+ if (fsImageName == null) {
+ throw new IOException("No filename header provided by server");
+ }
+ newLocalPaths.add(new File(localPath, fsImageName));
+ } else {
+ newLocalPaths.add(localPath);
+ }
+ }
+ localPaths = newLocalPaths;
+ }
+
+
+ long received = 0;
+ MessageDigest digester = null;
+ if (getChecksum) {
+ digester = MD5Hash.getDigester();
+ stream = new DigestInputStream(stream, digester);
+ }
+ boolean finishedReceiving = false;
+
+ List<FileOutputStream> outputStreams = Lists.newArrayList();
+
+ try {
+ if (localPaths != null) {
+ for (File f : localPaths) {
+ try {
+ if (f.exists()) {
+ LOG.warn("Overwriting existing file " + f
+ + " with file downloaded from " + url);
+ }
+ FileOutputStream fos = new FileOutputStream(f);
+ outputStreams.add(fos);
+ streamPathMap.put(fos, f);
+ } catch (IOException ioe) {
+ LOG.warn("Unable to download file " + f, ioe);
+ // This will be null if we're downloading the fsimage to a file
+ // outside of an NNStorage directory.
+ if (dstStorage != null &&
+ (dstStorage instanceof StorageErrorReporter)) {
+ ((StorageErrorReporter)dstStorage).reportErrorOnFile(f);
+ }
+ }
+ }
+
+ if (outputStreams.isEmpty()) {
+ throw new IOException(
+ "Unable to download to any storage directory");
+ }
+ }
+
+ int num = 1;
+ byte[] buf = new byte[IO_FILE_BUFFER_SIZE];
+ while (num > 0) {
+ num = stream.read(buf);
+ if (num > 0) {
+ received += num;
+ for (FileOutputStream fos : outputStreams) {
+ fos.write(buf, 0, num);
+ }
+ if (throttler != null) {
+ throttler.throttle(num);
+ }
+ }
+ }
+ finishedReceiving = true;
+ double xferSec = Math.max(
+ ((float)(Time.monotonicNow() - startTime)) / 1000.0, 0.001);
+ long xferKb = received / 1024;
+ xferCombined += xferSec;
+ xferStats.append(
+ String.format(" The fsimage download took %.2fs at %.2f KB/s.",
+ xferSec, xferKb / xferSec));
+ } finally {
+ stream.close();
+ for (FileOutputStream fos : outputStreams) {
+ long flushStartTime = Time.monotonicNow();
+ fos.getChannel().force(true);
+ fos.close();
+ double writeSec = Math.max(((float)
+ (flushStartTime - Time.monotonicNow())) / 1000.0, 0.001);
+ xferCombined += writeSec;
+ xferStats.append(String
+ .format(" Synchronous (fsync) write to disk of " +
+ streamPathMap.get(fos).getAbsolutePath() +
+ " took %.2fs.", writeSec));
+ }
+
+ // Something went wrong and did not finish reading.
+ // Remove the temporary files.
+ if (!finishedReceiving) {
+ deleteTmpFiles(localPaths);
+ }
+
+ if (finishedReceiving && received != advertisedSize) {
+ // only throw this exception if we think we read all of it on our end
+ // -- otherwise a client-side IOException would be masked by this
+ // exception that makes it look like a server-side problem!
+ deleteTmpFiles(localPaths);
+ throw new IOException("File " + url + " received length " + received +
+ " is not of the advertised size " +
+ advertisedSize);
+ }
+ }
+ xferStats.insert(0, String.format("Combined time for fsimage download and" +
+ " fsync to all disks took %.2fs.", xferCombined));
+ LOG.info(xferStats.toString());
+
+ if (digester != null) {
+ MD5Hash computedDigest = new MD5Hash(digester.digest());
+
+ if (advertisedDigest != null &&
+ !computedDigest.equals(advertisedDigest)) {
+ deleteTmpFiles(localPaths);
+ throw new IOException("File " + url + " computed digest " +
+ computedDigest + " does not match advertised digest " +
+ advertisedDigest);
+ }
+ return computedDigest;
+ } else {
+ return null;
+ }
+ }
+
+ private static void deleteTmpFiles(List<File> files) {
+ if (files == null) {
+ return;
+ }
+
+ LOG.info("Deleting temporary files: " + files);
+ for (File file : files) {
+ if (!file.delete()) {
+ LOG.warn("Deleting " + file + " has failed");
+ }
+ }
+ }
+
+ /**
+ * Sets a timeout value in millisecods for the Http connection.
+ * @param connection the Http connection for which timeout needs to be set
+ * @param timeout value to be set as timeout in milliseconds
+ */
+ public static void setTimeout(HttpURLConnection connection, int timeout) {
+ if (timeout > 0) {
+ connection.setConnectTimeout(timeout);
+ connection.setReadTimeout(timeout);
+ }
+ }
+
+ private static MD5Hash parseMD5Header(HttpURLConnection connection) {
+ String header = connection.getHeaderField(MD5_HEADER);
+ return (header != null) ? new MD5Hash(header) : null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ec609b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
index 48df8d6..36c2232 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
@@ -36,9 +36,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.LayoutFlags;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.common.HttpGetFailedException;
import org.apache.hadoop.hdfs.server.common.Storage;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation;
-import org.apache.hadoop.hdfs.server.namenode.TransferFsImage.HttpGetFailedException;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.SecurityUtil;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ec609b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
index 4e4028d..7a26df9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import org.apache.hadoop.hdfs.server.common.Util;
import static org.apache.hadoop.util.Time.monotonicNow;
import java.net.HttpURLConnection;
@@ -304,11 +305,12 @@ public class ImageServlet extends HttpServlet {
*/
public static void setVerificationHeadersForGet(HttpServletResponse response,
File file) throws IOException {
- response.setHeader(TransferFsImage.CONTENT_LENGTH,
+ response.setHeader(
+ Util.CONTENT_LENGTH,
String.valueOf(file.length()));
MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
if (hash != null) {
- response.setHeader(TransferFsImage.MD5_HEADER, hash.toString());
+ response.setHeader(Util.MD5_HEADER, hash.toString());
}
}
@@ -437,12 +439,13 @@ public class ImageServlet extends HttpServlet {
*/
static void setVerificationHeadersForPut(HttpURLConnection connection,
File file) throws IOException {
- connection.setRequestProperty(TransferFsImage.CONTENT_LENGTH,
+ connection.setRequestProperty(
+ Util.CONTENT_LENGTH,
String.valueOf(file.length()));
MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
if (hash != null) {
connection
- .setRequestProperty(TransferFsImage.MD5_HEADER, hash.toString());
+ .setRequestProperty(Util.MD5_HEADER, hash.toString());
}
}
@@ -462,7 +465,7 @@ public class ImageServlet extends HttpServlet {
params.put(STORAGEINFO_PARAM, storage.toColonSeparatedString());
// setting the length of the file to be uploaded in separate property as
// Content-Length only supports up to 2GB
- params.put(TransferFsImage.FILE_LENGTH, Long.toString(imageFileSize));
+ params.put(Util.FILE_LENGTH, Long.toString(imageFileSize));
params.put(IMAGE_FILE_TYPE, nnf.name());
return params;
}
@@ -586,7 +589,7 @@ public class ImageServlet extends HttpServlet {
txId = ServletUtil.parseLongParam(request, TXID_PARAM);
storageInfoString = ServletUtil.getParameter(request, STORAGEINFO_PARAM);
fileSize = ServletUtil.parseLongParam(request,
- TransferFsImage.FILE_LENGTH);
+ Util.FILE_LENGTH);
String imageType = ServletUtil.getParameter(request, IMAGE_FILE_TYPE);
nnf = imageType == null ? NameNodeFile.IMAGE : NameNodeFile
.valueOf(imageType);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ec609b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 0fc3e60..735b2c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -28,6 +28,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.MAX_PATH_DEPTH;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.MAX_PATH_LENGTH;
+
import static org.apache.hadoop.util.Time.now;
import java.io.FileNotFoundException;
@@ -136,6 +137,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerFaultInjector;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
+import org.apache.hadoop.hdfs.server.common.HttpGetFailedException;
import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
@@ -2104,7 +2106,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
} catch (FileNotFoundException e) {
LOG.debug("Tried to read from deleted or moved edit log segment", e);
return null;
- } catch (TransferFsImage.HttpGetFailedException e) {
+ } catch (HttpGetFailedException e) {
LOG.debug("Tried to read from deleted edit log segment", e);
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ec609b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index e4b95ee..5821353 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -19,18 +19,12 @@ package org.apache.hadoop.hdfs.server.namenode;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URISyntaxException;
import java.net.URL;
-import java.security.DigestInputStream;
-import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -44,17 +38,16 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.common.HttpPutFailedException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
-import org.apache.hadoop.hdfs.server.common.StorageErrorReporter;
+import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.util.Canceler;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
-import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.security.UserGroupInformation;
@@ -66,6 +59,9 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import org.eclipse.jetty.io.EofException;
+import static org.apache.hadoop.hdfs.server.common.Util.IO_FILE_BUFFER_SIZE;
+import static org.apache.hadoop.hdfs.server.common.Util.connectionFactory;
+
/**
* This class provides fetching a specified file from the NameNode.
*/
@@ -82,43 +78,23 @@ public class TransferFsImage {
private final int response;
private final boolean shouldReThrowException;
- private TransferResult(int response, boolean rethrow) {
+ TransferResult(int response, boolean rethrow) {
this.response = response;
this.shouldReThrowException = rethrow;
}
public static TransferResult getResultForCode(int code){
- TransferResult ret = UNEXPECTED_FAILURE;
for(TransferResult result:TransferResult.values()){
if(result.response == code){
return result;
}
}
- return ret;
+ return UNEXPECTED_FAILURE;
}
}
- public final static String CONTENT_LENGTH = "Content-Length";
- public final static String FILE_LENGTH = "File-Length";
- public final static String MD5_HEADER = "X-MD5-Digest";
-
- private final static String CONTENT_TYPE = "Content-Type";
- private final static String CONTENT_TRANSFER_ENCODING = "Content-Transfer-Encoding";
- private final static int IO_FILE_BUFFER_SIZE;
-
@VisibleForTesting
static int timeout = 0;
- private static final URLConnectionFactory connectionFactory;
- private static final boolean isSpnegoEnabled;
-
- static {
- Configuration conf = new Configuration();
- connectionFactory = URLConnectionFactory
- .newDefaultURLConnectionFactory(conf);
- isSpnegoEnabled = UserGroupInformation.isSecurityEnabled();
- IO_FILE_BUFFER_SIZE = DFSUtilClient.getIoFileBufferSize(conf);
- }
-
private static final Log LOG = LogFactory.getLog(TransferFsImage.class);
public static void downloadMostRecentImageToDirectory(URL infoServer,
@@ -159,7 +135,7 @@ public class TransferFsImage {
}
MD5Hash advertisedDigest = parseMD5Header(request);
- MD5Hash hash = receiveFile(fileName, dstFiles, dstStorage, true,
+ MD5Hash hash = Util.receiveFile(fileName, dstFiles, dstStorage, true,
advertisedSize, advertisedDigest, fileName, stream, throttler);
LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size "
+ dstFiles.get(0).length() + " bytes.");
@@ -226,8 +202,9 @@ public class TransferFsImage {
* @param txid the transaction ID of the image to be uploaded
* @throws IOException if there is an I/O error
*/
- public static TransferResult uploadImageFromStorage(URL fsName, Configuration conf,
- NNStorage storage, NameNodeFile nnf, long txid) throws IOException {
+ static TransferResult uploadImageFromStorage(URL fsName,
+ Configuration conf, NNStorage storage, NameNodeFile nnf, long txid)
+ throws IOException {
return uploadImageFromStorage(fsName, conf, storage, nnf, txid, null);
}
@@ -323,9 +300,7 @@ public class TransferFsImage {
responseCode, urlWithParams, connection.getResponseMessage()),
responseCode);
}
- } catch (AuthenticationException e) {
- throw new IOException(e);
- } catch (URISyntaxException e) {
+ } catch (AuthenticationException | URISyntaxException e) {
throw new IOException(e);
} finally {
if (connection != null) {
@@ -336,9 +311,9 @@ public class TransferFsImage {
private static void writeFileToPutRequest(Configuration conf,
HttpURLConnection connection, File imageFile, Canceler canceler)
- throws FileNotFoundException, IOException {
- connection.setRequestProperty(CONTENT_TYPE, "application/octet-stream");
- connection.setRequestProperty(CONTENT_TRANSFER_ENCODING, "binary");
+ throws IOException {
+ connection.setRequestProperty(Util.CONTENT_TYPE, "application/octet-stream");
+ connection.setRequestProperty(Util.CONTENT_TRANSFER_ENCODING, "binary");
OutputStream output = connection.getOutputStream();
FileInputStream input = new FileInputStream(imageFile);
try {
@@ -414,7 +389,7 @@ public class TransferFsImage {
* Copies the response from the URL to a list of local files.
* @param dstStorage if an error occurs writing to one of the files,
* this storage object will be notified.
- * @Return a digest of the received file if getChecksum is true
+ * @return a digest of the received file if getChecksum is true
*/
static MD5Hash getFileClient(URL infoServer,
String queryString, List<File> localPaths,
@@ -426,40 +401,12 @@ public class TransferFsImage {
public static MD5Hash doGetUrl(URL url, List<File> localPaths,
Storage dstStorage, boolean getChecksum) throws IOException {
- HttpURLConnection connection;
- try {
- connection = (HttpURLConnection)
- connectionFactory.openConnection(url, isSpnegoEnabled);
- } catch (AuthenticationException e) {
- throw new IOException(e);
- }
-
- setTimeout(connection);
+ return Util.doGetUrl(url, localPaths, dstStorage, getChecksum, timeout);
+ }
- if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
- throw new HttpGetFailedException(
- "Image transfer servlet at " + url +
- " failed with status code " + connection.getResponseCode() +
- "\nResponse message:\n" + connection.getResponseMessage(),
- connection);
- }
-
- long advertisedSize;
- String contentLength = connection.getHeaderField(CONTENT_LENGTH);
- if (contentLength != null) {
- advertisedSize = Long.parseLong(contentLength);
- } else {
- throw new IOException(CONTENT_LENGTH + " header is not provided " +
- "by the namenode when trying to fetch " + url);
- }
- MD5Hash advertisedDigest = parseMD5Header(connection);
- String fsImageName = connection
- .getHeaderField(ImageServlet.HADOOP_IMAGE_EDITS_HEADER);
- InputStream stream = connection.getInputStream();
-
- return receiveFile(url.toExternalForm(), localPaths, dstStorage,
- getChecksum, advertisedSize, advertisedDigest, fsImageName, stream,
- null);
+ private static MD5Hash parseMD5Header(HttpServletRequest request) {
+ String header = request.getHeader(Util.MD5_HEADER);
+ return (header != null) ? new MD5Hash(header) : null;
}
private static void setTimeout(HttpURLConnection connection) {
@@ -467,204 +414,10 @@ public class TransferFsImage {
Configuration conf = new HdfsConfiguration();
timeout = conf.getInt(DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_KEY,
DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_DEFAULT);
- LOG.info("Image Transfer timeout configured to " + timeout
- + " milliseconds");
- }
-
- if (timeout > 0) {
- connection.setConnectTimeout(timeout);
- connection.setReadTimeout(timeout);
- }
- }
-
- private static MD5Hash receiveFile(String url, List<File> localPaths,
- Storage dstStorage, boolean getChecksum, long advertisedSize,
- MD5Hash advertisedDigest, String fsImageName, InputStream stream,
- DataTransferThrottler throttler) throws IOException {
- long startTime = Time.monotonicNow();
- Map<FileOutputStream, File> streamPathMap = new HashMap<>();
- StringBuilder xferStats = new StringBuilder();
- double xferCombined = 0;
- if (localPaths != null) {
- // If the local paths refer to directories, use the server-provided header
- // as the filename within that directory
- List<File> newLocalPaths = new ArrayList<File>();
- for (File localPath : localPaths) {
- if (localPath.isDirectory()) {
- if (fsImageName == null) {
- throw new IOException("No filename header provided by server");
- }
- newLocalPaths.add(new File(localPath, fsImageName));
- } else {
- newLocalPaths.add(localPath);
- }
- }
- localPaths = newLocalPaths;
- }
-
-
- long received = 0;
- MessageDigest digester = null;
- if (getChecksum) {
- digester = MD5Hash.getDigester();
- stream = new DigestInputStream(stream, digester);
- }
- boolean finishedReceiving = false;
-
- List<FileOutputStream> outputStreams = Lists.newArrayList();
-
- try {
- if (localPaths != null) {
- for (File f : localPaths) {
- try {
- if (f.exists()) {
- LOG.warn("Overwriting existing file " + f
- + " with file downloaded from " + url);
- }
- FileOutputStream fos = new FileOutputStream(f);
- outputStreams.add(fos);
- streamPathMap.put(fos, f);
- } catch (IOException ioe) {
- LOG.warn("Unable to download file " + f, ioe);
- // This will be null if we're downloading the fsimage to a file
- // outside of an NNStorage directory.
- if (dstStorage != null &&
- (dstStorage instanceof StorageErrorReporter)) {
- ((StorageErrorReporter)dstStorage).reportErrorOnFile(f);
- }
- }
- }
-
- if (outputStreams.isEmpty()) {
- throw new IOException(
- "Unable to download to any storage directory");
- }
- }
-
- int num = 1;
- byte[] buf = new byte[IO_FILE_BUFFER_SIZE];
- while (num > 0) {
- num = stream.read(buf);
- if (num > 0) {
- received += num;
- for (FileOutputStream fos : outputStreams) {
- fos.write(buf, 0, num);
- }
- if (throttler != null) {
- throttler.throttle(num);
- }
- }
- }
- finishedReceiving = true;
- double xferSec = Math.max(
- ((float)(Time.monotonicNow() - startTime)) / 1000.0, 0.001);
- long xferKb = received / 1024;
- xferCombined += xferSec;
- xferStats.append(
- String.format(" The fsimage download took %.2fs at %.2f KB/s.",
- xferSec, xferKb / xferSec));
- } finally {
- stream.close();
- for (FileOutputStream fos : outputStreams) {
- long flushStartTime = Time.monotonicNow();
- fos.getChannel().force(true);
- fos.close();
- double writeSec = Math.max(((float)
- (flushStartTime - Time.monotonicNow())) / 1000.0, 0.001);
- xferCombined += writeSec;
- xferStats.append(String
- .format(" Synchronous (fsync) write to disk of " +
- streamPathMap.get(fos).getAbsolutePath() +
- " took %.2fs.", writeSec));
- }
-
- // Something went wrong and did not finish reading.
- // Remove the temporary files.
- if (!finishedReceiving) {
- deleteTmpFiles(localPaths);
- }
-
- if (finishedReceiving && received != advertisedSize) {
- // only throw this exception if we think we read all of it on our end
- // -- otherwise a client-side IOException would be masked by this
- // exception that makes it look like a server-side problem!
- deleteTmpFiles(localPaths);
- throw new IOException("File " + url + " received length " + received +
- " is not of the advertised size " +
- advertisedSize);
- }
- }
- xferStats.insert(
- 0, String.format(
- "Combined time for fsimage download and fsync " +
- "to all disks took %.2fs.", xferCombined));
- LOG.info(xferStats.toString());
-
- if (digester != null) {
- MD5Hash computedDigest = new MD5Hash(digester.digest());
-
- if (advertisedDigest != null &&
- !computedDigest.equals(advertisedDigest)) {
- deleteTmpFiles(localPaths);
- throw new IOException("File " + url + " computed digest " +
- computedDigest + " does not match advertised digest " +
- advertisedDigest);
- }
- return computedDigest;
- } else {
- return null;
- }
- }
-
- private static void deleteTmpFiles(List<File> files) {
- if (files == null) {
- return;
- }
-
- LOG.info("Deleting temporary files: " + files);
- for (File file : files) {
- if (!file.delete()) {
- LOG.warn("Deleting " + file + " has failed");
- }
+ LOG.info("Image Transfer timeout configured to " + timeout +
+ " milliseconds");
}
- }
- private static MD5Hash parseMD5Header(HttpURLConnection connection) {
- String header = connection.getHeaderField(MD5_HEADER);
- return (header != null) ? new MD5Hash(header) : null;
- }
-
- private static MD5Hash parseMD5Header(HttpServletRequest request) {
- String header = request.getHeader(MD5_HEADER);
- return (header != null) ? new MD5Hash(header) : null;
+ Util.setTimeout(connection, timeout);
}
-
- public static class HttpGetFailedException extends IOException {
- private static final long serialVersionUID = 1L;
- private final int responseCode;
-
- HttpGetFailedException(String msg, HttpURLConnection connection) throws IOException {
- super(msg);
- this.responseCode = connection.getResponseCode();
- }
-
- public int getResponseCode() {
- return responseCode;
- }
- }
-
- public static class HttpPutFailedException extends IOException {
- private static final long serialVersionUID = 1L;
- private final int responseCode;
-
- HttpPutFailedException(String msg, int responseCode) throws IOException {
- super(msg);
- this.responseCode = responseCode;
- }
-
- public int getResponseCode() {
- return responseCode;
- }
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: MAPREDUCE-6711. JobImpl fails to
handle preemption events on state COMMITTING. Contributed by Prabhu Joseph.
Posted by st...@apache.org.
MAPREDUCE-6711. JobImpl fails to handle preemption events on state COMMITTING. Contributed by Prabhu Joseph.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/679478d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/679478d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/679478d0
Branch: refs/heads/HADOOP-13345
Commit: 679478d0c643defb3759a2ecd3c704315edee124
Parents: ac16400
Author: Junping Du <ju...@apache.org>
Authored: Sat Jan 7 22:04:18 2017 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Sat Jan 7 22:04:18 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java | 4 +++-
.../apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java | 8 ++++++++
2 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/679478d0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index 5127a43..4d155d0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -419,7 +419,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
.addTransition(JobStateInternal.COMMITTING,
JobStateInternal.COMMITTING,
EnumSet.of(JobEventType.JOB_UPDATED_NODES,
- JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
+ JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE,
+ JobEventType.JOB_TASK_ATTEMPT_COMPLETED,
+ JobEventType.JOB_MAP_TASK_RESCHEDULED))
// Transitions from SUCCEEDED state
.addTransition(JobStateInternal.SUCCEEDED, JobStateInternal.SUCCEEDED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/679478d0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
index eaa5af7..79b6281 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
@@ -218,6 +218,14 @@ public class TestJobImpl {
completeJobTasks(job);
assertJobState(job, JobStateInternal.COMMITTING);
+ job.handle(new JobEvent(job.getID(),
+ JobEventType.JOB_TASK_ATTEMPT_COMPLETED));
+ assertJobState(job, JobStateInternal.COMMITTING);
+
+ job.handle(new JobEvent(job.getID(),
+ JobEventType.JOB_MAP_TASK_RESCHEDULED));
+ assertJobState(job, JobStateInternal.COMMITTING);
+
// let the committer complete and verify the job succeeds
syncBarrier.await();
assertJobState(job, JobStateInternal.SUCCEEDED);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into
s3guard/HADOOP-13345
Posted by st...@apache.org.
Merge branch 'trunk' into s3guard/HADOOP-13345
# Conflicts:
# hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c61010d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c61010d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c61010d
Branch: refs/heads/HADOOP-13345
Commit: 0c61010deec872a56149e58055fc90aba08fbb83
Parents: a5cc315 e648b6e
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jan 11 18:18:49 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jan 11 18:18:49 2017 +0000
----------------------------------------------------------------------
.gitignore | 1 +
BUILDING.txt | 25 +
LICENSE.txt | 1322 +++++++++++-------
NOTICE.txt | 129 +-
dev-support/bin/dist-copynativelibs | 13 +
dev-support/bin/dist-layout-stitching | 8 +-
.../resources/assemblies/hadoop-kms-dist.xml | 25 +-
hadoop-client-modules/hadoop-client-api/pom.xml | 216 +++
.../hadoop-client-check-invariants/pom.xml | 124 ++
.../hadoop-client-check-test-invariants/pom.xml | 132 ++
.../hadoop-client-integration-tests/pom.xml | 130 ++
.../apache/hadoop/example/ITUseMiniCluster.java | 113 ++
.../src/test/resources/hdfs-site.xml | 34 +
.../src/test/resources/log4j.properties | 24 +
.../hadoop-client-minicluster/pom.xml | 745 ++++++++++
.../hadoop-client-runtime/pom.xml | 318 +++++
hadoop-client-modules/hadoop-client/pom.xml | 337 +++++
hadoop-client-modules/pom.xml | 45 +
hadoop-client/pom.xml | 319 -----
.../server/AuthenticationFilter.java | 6 +-
.../server/KerberosAuthenticationHandler.java | 254 +++-
.../authentication/util/KerberosName.java | 4 +-
.../authentication/util/TestKerberosName.java | 22 +
hadoop-common-project/hadoop-common/pom.xml | 24 +-
.../hadoop-common/src/CMakeLists.txt | 29 +
.../hadoop-common/src/config.h.cmake | 1 +
.../src/main/bin/hadoop-functions.sh | 51 +-
.../hadoop/conf/ConfigurationWithLogging.java | 113 ++
.../hadoop/fs/CommonConfigurationKeys.java | 16 +
.../main/java/org/apache/hadoop/fs/FsShell.java | 7 +-
.../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 70 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 5 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 11 +
.../apache/hadoop/ha/ZKFailoverController.java | 7 +-
.../org/apache/hadoop/http/HttpServer2.java | 163 ++-
.../apache/hadoop/io/compress/Decompressor.java | 2 +-
.../hadoop/io/compress/ZStandardCodec.java | 242 ++++
.../io/compress/zstd/ZStandardCompressor.java | 305 ++++
.../io/compress/zstd/ZStandardDecompressor.java | 323 +++++
.../hadoop/io/compress/zstd/package-info.java | 22 +
.../org/apache/hadoop/io/nativeio/NativeIO.java | 40 +-
.../hadoop/metrics2/MetricsJsonBuilder.java | 125 ++
.../hadoop/metrics2/lib/MutableQuantiles.java | 10 +
.../lib/MutableRatesWithAggregation.java | 40 +-
.../hadoop/metrics2/lib/RollingAverages.java | 251 ++++
.../apache/hadoop/security/ssl/SSLFactory.java | 64 +-
.../hadoop/util/DiskValidatorFactory.java | 5 +-
.../hadoop/util/GenericOptionsParser.java | 14 +-
.../apache/hadoop/util/NativeCodeLoader.java | 5 +
.../hadoop/util/NativeLibraryChecker.java | 13 +-
.../hadoop/util/ReadWriteDiskValidator.java | 95 ++
.../util/ReadWriteDiskValidatorMetrics.java | 170 +++
.../main/java/org/apache/hadoop/util/Shell.java | 24 +
.../io/compress/zstd/ZStandardCompressor.c | 259 ++++
.../io/compress/zstd/ZStandardDecompressor.c | 218 +++
.../zstd/org_apache_hadoop_io_compress_zstd.h | 34 +
.../org/apache/hadoop/util/NativeCodeLoader.c | 11 +
...g.apache.hadoop.io.compress.CompressionCodec | 1 +
.../src/main/resources/core-default.xml | 68 +-
.../src/site/markdown/ClusterSetup.md | 1 +
.../src/site/markdown/CommandsManual.md | 15 +-
.../hadoop-common/src/site/markdown/Metrics.md | 37 +
.../src/site/markdown/NativeLibraries.md.vm | 1 +
.../src/site/markdown/SingleCluster.md.vm | 12 +-
.../conf/TestCommonConfigurationFields.java | 11 +-
.../java/org/apache/hadoop/fs/TestFsShell.java | 67 +-
.../apache/hadoop/fs/ftp/TestFTPFileSystem.java | 56 +-
.../fs/viewfs/ViewFileSystemBaseTest.java | 73 +
.../apache/hadoop/io/compress/TestCodec.java | 12 +
.../io/compress/TestCompressionStreamReuse.java | 8 +
.../TestZStandardCompressorDecompressor.java | 485 +++++++
.../hadoop/metrics2/impl/MetricsRecords.java | 16 +
.../metrics2/lib/TestRollingAverages.java | 123 ++
.../delegation/web/TestWebDelegationToken.java | 4 +
.../apache/hadoop/test/GenericTestUtils.java | 72 +-
.../org/apache/hadoop/test/MetricsAsserts.java | 7 +
.../hadoop/util/TestReadWriteDiskValidator.java | 161 +++
.../java/org/apache/hadoop/util/TestShell.java | 52 +
.../src/test/resources/zstd/test_file.txt | 71 +
.../src/test/resources/zstd/test_file.txt.zst | Bin 0 -> 3690 bytes
.../src/test/scripts/hadoop_mkdir.bats | 42 +
.../src/test/scripts/hadoop_using_envvar.bats | 33 +
.../dev-support/findbugsExcludeFile.xml | 2 +-
hadoop-common-project/hadoop-kms/pom.xml | 160 +--
.../hadoop-kms/src/main/conf/kms-env.sh | 53 +-
.../src/main/conf/kms-log4j.properties | 3 +-
.../hadoop-kms/src/main/conf/kms-site.xml | 167 +--
.../hadoop/crypto/key/kms/server/KMSACLs.java | 36 +-
.../hadoop/crypto/key/kms/server/KMSAudit.java | 30 +-
.../crypto/key/kms/server/KMSAuditLogger.java | 21 +-
.../crypto/key/kms/server/KMSConfiguration.java | 16 +
.../crypto/key/kms/server/KMSJMXServlet.java | 36 -
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 23 +-
.../crypto/key/kms/server/KMSWebServer.java | 155 ++
.../hadoop-kms/src/main/libexec/kms-config.sh | 77 -
.../main/libexec/shellprofile.d/hadoop-kms.sh | 57 +
.../src/main/resources/kms-default.xml | 248 ++++
.../main/resources/webapps/kms/WEB-INF/web.xml | 68 +
.../main/resources/webapps/static/index.html | 35 +
.../hadoop-kms/src/main/sbin/kms.sh | 123 +-
.../src/main/tomcat/ROOT/WEB-INF/web.xml | 16 -
.../hadoop-kms/src/main/tomcat/ROOT/index.html | 27 -
.../src/main/tomcat/logging.properties | 67 -
.../hadoop-kms/src/main/tomcat/server.xml | 155 --
.../src/main/tomcat/ssl-server.xml.conf | 137 --
.../hadoop-kms/src/main/webapp/WEB-INF/web.xml | 78 --
.../hadoop-kms/src/site/configuration.xsl | 49 +
.../hadoop-kms/src/site/markdown/index.md.vm | 122 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 118 +-
hadoop-dist/pom.xml | 17 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 16 +-
.../hadoop/hdfs/DistributedFileSystem.java | 29 +
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 10 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 7 +
.../datatransfer/BlockPinningException.java | 33 +
.../datatransfer/DataTransferProtoUtil.java | 17 +-
.../ClientNamenodeProtocolTranslatorPB.java | 15 +
.../server/datanode/BlockMetadataHeader.java | 29 +-
.../server/datanode/DiskBalancerWorkItem.java | 2 +-
.../server/datanode/DiskBalancerWorkStatus.java | 4 +-
.../hdfs/util/CombinedHostsFileReader.java | 2 +-
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 55 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 73 +-
...onfRefreshTokenBasedAccessTokenProvider.java | 2 +-
.../CredentialBasedAccessTokenProvider.java | 2 +-
.../hdfs/web/resources/CreateParentParam.java | 2 +-
.../hadoop/hdfs/web/resources/GetOpParam.java | 12 +-
.../hdfs/web/resources/OverwriteParam.java | 2 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +
.../src/main/proto/datatransfer.proto | 1 +
.../src/main/proto/erasurecoding.proto | 7 +
.../dev-support/findbugsExcludeFile.xml | 27 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 25 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 14 +
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 15 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 62 +-
.../blockmanagement/DatanodeDescriptor.java | 17 +-
.../server/blockmanagement/DatanodeManager.java | 8 +-
.../blockmanagement/DecommissionManager.java | 9 +-
.../server/blockmanagement/NumberReplicas.java | 2 +-
.../hdfs/server/common/HdfsServerConstants.java | 9 +-
.../server/common/HttpGetFailedException.java | 39 +
.../server/common/HttpPutFailedException.java | 37 +
.../hadoop/hdfs/server/common/Storage.java | 12 +-
.../apache/hadoop/hdfs/server/common/Util.java | 258 +++-
.../hdfs/server/datanode/BPServiceActor.java | 10 +-
.../server/datanode/BlockPoolSliceStorage.java | 8 +-
.../hdfs/server/datanode/BlockReceiver.java | 84 +-
.../hdfs/server/datanode/BlockScanner.java | 10 +-
.../hdfs/server/datanode/BlockSender.java | 30 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 207 +--
.../server/datanode/DataNodeFaultInjector.java | 25 +-
.../hdfs/server/datanode/DataNodeMXBean.java | 18 +-
.../hdfs/server/datanode/DataStorage.java | 16 +-
.../hdfs/server/datanode/DataXceiver.java | 17 +-
.../hdfs/server/datanode/DatanodeUtil.java | 19 +-
.../datanode/FaultInjectorFileIoEvents.java | 55 +
.../hdfs/server/datanode/FileIoProvider.java | 1051 ++++++++++++++
.../hdfs/server/datanode/FinalizedReplica.java | 29 +
.../hdfs/server/datanode/LocalReplica.java | 133 +-
.../server/datanode/LocalReplicaInPipeline.java | 41 +-
.../server/datanode/ProfilingFileIoEvents.java | 124 ++
.../hdfs/server/datanode/ReplicaInPipeline.java | 4 +-
.../hdfs/server/datanode/ReplicaInfo.java | 17 +-
.../server/datanode/ReplicaUnderRecovery.java | 2 +-
.../hdfs/server/datanode/VolumeScanner.java | 9 +-
.../server/datanode/checker/AsyncChecker.java | 5 +-
.../datanode/checker/DatasetVolumeChecker.java | 178 ++-
.../checker/StorageLocationChecker.java | 8 +-
.../datanode/checker/ThrottledAsyncChecker.java | 19 +-
.../erasurecode/StripedBlockReader.java | 10 +-
.../erasurecode/StripedBlockReconstructor.java | 7 +-
.../erasurecode/StripedBlockWriter.java | 1 +
.../datanode/erasurecode/StripedReader.java | 4 +
.../erasurecode/StripedReconstructor.java | 31 +
.../datanode/erasurecode/StripedWriter.java | 4 +
.../fsdataset/DataNodeVolumeMetrics.java | 289 ++++
.../server/datanode/fsdataset/FsDatasetSpi.java | 3 +-
.../server/datanode/fsdataset/FsVolumeSpi.java | 16 +
.../datanode/fsdataset/ReplicaInputStreams.java | 11 +-
.../fsdataset/ReplicaOutputStreams.java | 72 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 123 +-
.../impl/FsDatasetAsyncDiskService.java | 6 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 53 +-
.../datanode/fsdataset/impl/FsDatasetUtil.java | 5 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 133 +-
.../fsdataset/impl/FsVolumeImplBuilder.java | 12 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 25 +-
.../datanode/metrics/DataNodeMetrics.java | 24 +-
.../datanode/metrics/DataNodePeerMetrics.java | 117 ++
.../server/datanode/web/webhdfs/HdfsWriter.java | 19 +-
.../server/diskbalancer/command/Command.java | 2 +-
.../connectors/JsonNodeConnector.java | 2 +-
.../datamodel/DiskBalancerCluster.java | 2 +-
.../datamodel/DiskBalancerVolume.java | 2 +-
.../server/diskbalancer/planner/NodePlan.java | 2 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 26 +-
.../server/namenode/EditLogFileInputStream.java | 3 +-
.../server/namenode/EncryptionZoneManager.java | 8 +-
.../hadoop/hdfs/server/namenode/FSDirAclOp.java | 6 +-
.../hdfs/server/namenode/FSDirAttrOp.java | 16 +-
.../server/namenode/FSDirErasureCodingOp.java | 115 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 18 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 3 -
.../server/namenode/FSImageFormatProtobuf.java | 4 +-
...FSImagePreTransactionalStorageInspector.java | 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 65 +-
.../hdfs/server/namenode/ImageServlet.java | 15 +-
.../hadoop/hdfs/server/namenode/NNStorage.java | 14 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 5 +-
.../hdfs/server/namenode/NameNodeMXBean.java | 15 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 20 +-
.../server/namenode/NameNodeStatusMXBean.java | 8 +-
.../hdfs/server/namenode/NamenodeFsck.java | 61 +-
.../hdfs/server/namenode/TransferFsImage.java | 295 +---
.../server/namenode/ha/BootstrapStandby.java | 10 +-
.../web/resources/NamenodeWebHdfsMethods.java | 16 +
.../apache/hadoop/hdfs/tools/CacheAdmin.java | 8 +-
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 8 +-
.../org/apache/hadoop/hdfs/tools/DFSck.java | 4 +
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 6 +-
.../hdfs/tools/erasurecode/ECCommand.java | 34 +
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 33 +
.../src/main/proto/DatanodeProtocol.proto | 10 -
.../hadoop-hdfs/src/main/proto/HdfsServer.proto | 15 +-
.../src/main/resources/hdfs-default.xml | 25 +
.../src/main/webapps/hdfs/dfshealth.html | 36 +-
.../src/main/webapps/hdfs/dfshealth.js | 11 +-
.../src/site/markdown/HDFSCommands.md | 6 +-
.../src/site/markdown/HDFSErasureCoding.md | 5 +
.../src/site/markdown/HdfsRollingUpgrade.md | 8 +-
.../hadoop-hdfs/src/site/markdown/LibHdfs.md | 2 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 197 ++-
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 3 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 15 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 24 +
.../TestClientProtocolForPipelineRecovery.java | 3 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 5 +-
.../org/apache/hadoop/hdfs/TestFileAppend.java | 2 +-
.../org/apache/hadoop/hdfs/TestSafeMode.java | 1 +
.../TestUnsetAndChangeDirectoryEcPolicy.java | 366 +++++
.../blockmanagement/BlockManagerTestUtil.java | 4 +-
.../blockmanagement/TestBlockStatsMXBean.java | 4 +
.../hdfs/server/common/StorageAdapter.java | 2 +-
.../hdfs/server/datanode/DataNodeTestUtils.java | 82 ++
.../server/datanode/SimulatedFSDataset.java | 56 +-
.../server/datanode/TestBPOfferService.java | 1 +
.../datanode/TestBlockPoolSliceStorage.java | 2 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 2 +-
.../server/datanode/TestBlockReplacement.java | 70 +-
.../hdfs/server/datanode/TestBlockScanner.java | 100 ++
.../TestDataNodeErasureCodingMetrics.java | 168 ++-
.../datanode/TestDataNodeFaultInjector.java | 173 +++
.../datanode/TestDataNodeHotSwapVolumes.java | 177 ++-
.../server/datanode/TestDataNodeLifeline.java | 33 +-
.../datanode/TestDataNodePeerMetrics.java | 92 ++
.../datanode/TestDataNodeVolumeFailure.java | 20 +-
.../TestDataNodeVolumeFailureReporting.java | 66 +-
.../TestDataNodeVolumeFailureToleration.java | 3 +
.../datanode/TestDataNodeVolumeMetrics.java | 182 +++
.../server/datanode/TestDirectoryScanner.java | 17 +
.../hdfs/server/datanode/TestDiskError.java | 24 +-
.../server/datanode/TestSimulatedFSDataset.java | 2 +-
.../checker/TestDatasetVolumeChecker.java | 97 +-
.../TestDatasetVolumeCheckerFailures.java | 60 +-
.../checker/TestThrottledAsyncChecker.java | 128 +-
.../extdataset/ExternalDatasetImpl.java | 5 +-
.../extdataset/ExternalReplicaInPipeline.java | 6 +-
.../datanode/extdataset/ExternalVolumeImpl.java | 19 +
.../fsdataset/impl/TestFsDatasetImpl.java | 84 +-
.../fsdataset/impl/TestFsVolumeList.java | 37 -
.../hadoop/hdfs/server/mover/TestMover.java | 163 ++-
.../namenode/TestEncryptionZoneManager.java | 138 ++
.../hdfs/server/namenode/TestFSDirAttrOp.java | 76 +
.../hdfs/server/namenode/TestFSImage.java | 12 +-
.../hdfs/server/namenode/TestFileTruncate.java | 31 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 340 ++++-
.../server/namenode/TestNameNodeMXBean.java | 225 ++-
.../apache/hadoop/hdfs/web/TestJsonUtil.java | 8 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 315 +++++
.../hdfs/web/TestWebHdfsFileSystemContract.java | 40 +
.../hadoop/tools/TestHdfsConfigFields.java | 4 +
.../src/test/resources/testCryptoConf.xml | 98 +-
.../test/resources/testErasureCodingConf.xml | 64 +-
.../jobhistory/JobHistoryCopyService.java | 5 +-
.../hadoop/mapreduce/v2/app/AppContext.java | 4 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 2 +-
.../mapreduce/v2/app/job/impl/JobImpl.java | 4 +-
.../v2/app/rm/RMContainerAllocator.java | 20 +-
.../v2/app/speculate/DefaultSpeculator.java | 3 +-
.../mapred/TestLocalContainerLauncher.java | 2 +-
.../mapred/TestTaskAttemptFinishingMonitor.java | 2 +-
.../mapred/TestTaskAttemptListenerImpl.java | 17 +-
.../jobhistory/TestJobHistoryEventHandler.java | 10 +-
.../hadoop/mapreduce/v2/app/MockAppContext.java | 4 +-
.../mapreduce/v2/app/MockEventHandler.java | 6 +-
.../v2/app/TestCheckpointPreemptionPolicy.java | 4 +-
.../mapreduce/v2/app/TestFetchFailure.java | 17 +-
.../v2/app/TestKillAMPreemptionPolicy.java | 4 +-
.../mapreduce/v2/app/TestRuntimeEstimators.java | 3 +-
.../app/commit/TestCommitterEventHandler.java | 2 +-
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 8 +
.../app/launcher/TestContainerLauncherImpl.java | 12 +-
.../app/local/TestLocalContainerAllocator.java | 4 +-
.../v2/app/rm/TestRMContainerAllocator.java | 12 +-
.../apache/hadoop/mapreduce/TypeConverter.java | 7 +-
.../org/apache/hadoop/mapred/CleanupQueue.java | 7 +-
.../java/org/apache/hadoop/mapred/JobConf.java | 2 +-
.../java/org/apache/hadoop/mapred/MapTask.java | 10 +-
.../java/org/apache/hadoop/mapred/Mapper.java | 2 +-
.../java/org/apache/hadoop/mapred/Reducer.java | 2 +-
.../org/apache/hadoop/mapreduce/JobStatus.java | 18 +-
.../apache/hadoop/mapreduce/JobSubmitter.java | 2 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 6 +-
.../mapreduce/lib/output/TextOutputFormat.java | 15 +-
.../task/reduce/ShuffleSchedulerImpl.java | 40 +-
.../src/site/markdown/MapredCommands.md | 6 +
.../hadoop/mapreduce/v2/hs/JobHistory.java | 3 +-
.../resources/job_1329348432655_0001_conf.xml | 2 -
.../apache/hadoop/mapred/JobClientUnitTest.java | 6 +-
.../hadoop/mapred/TestMROpportunisticMaps.java | 2 +-
.../mapred/pipes/TestPipeApplication.java | 4 +-
.../hadoop-mapreduce-client-nativetask/pom.xml | 8 +
.../hadoop/examples/dancing/Pentomino.java | 5 +
.../hadoop/examples/terasort/TeraScheduler.java | 16 +-
hadoop-maven-plugins/pom.xml | 22 +
.../resource/ServicesResourceTransformer.java | 164 +++
.../plugin/shade/resource/package-info.java | 23 +
hadoop-project-dist/pom.xml | 8 +-
hadoop-project/pom.xml | 130 +-
.../hadoop/tools/TestHadoopArchiveLogs.java | 12 +-
.../tools/TestHadoopArchiveLogsRunner.java | 11 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 14 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 16 +-
.../apache/hadoop/fs/s3a/S3AInputStream.java | 33 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 100 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 153 +-
.../tools/hadoop-aws/troubleshooting_s3a.md | 52 +
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 1 -
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 130 ++
.../apache/hadoop/fs/s3a/S3ATestConstants.java | 12 -
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 61 +-
.../scale/ITestS3AInputStreamPerformance.java | 57 +-
.../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 28 +-
hadoop-tools/hadoop-azure-datalake/pom.xml | 10 +-
.../fs/azure/AzureNativeFileSystemStore.java | 91 +-
.../org/apache/hadoop/fs/azure/KeyProvider.java | 3 +-
.../fs/azure/LocalSASKeyGeneratorImpl.java | 263 ++++
.../hadoop/fs/azure/NativeAzureFileSystem.java | 31 +-
.../hadoop/fs/azure/NativeFileSystemStore.java | 5 +-
.../fs/azure/RemoteSASKeyGeneratorImpl.java | 296 ++++
.../fs/azure/SASKeyGenerationException.java | 40 +
.../hadoop/fs/azure/SASKeyGeneratorImpl.java | 61 +
.../fs/azure/SASKeyGeneratorInterface.java | 66 +
.../hadoop/fs/azure/SecureModeException.java | 40 +
.../fs/azure/SecureStorageInterfaceImpl.java | 565 ++++++++
.../hadoop/fs/azure/SelfRenewingLease.java | 2 +-
.../hadoop/fs/azure/SendRequestIntercept.java | 3 +-
.../hadoop/fs/azure/StorageInterface.java | 28 +-
.../fs/azure/WasbRemoteCallException.java | 41 +
.../hadoop/fs/azure/WasbRemoteCallHelper.java | 93 ++
.../metrics/AzureFileSystemInstrumentation.java | 2 +
.../hadoop-azure/src/site/markdown/index.md | 48 +
.../hadoop/fs/azure/TestContainerChecks.java | 12 +-
.../fs/azure/TestWasbUriAndConfiguration.java | 17 +-
.../src/test/resources/azure-test.xml | 15 +-
.../src/site/markdown/DistCp.md.vm | 1 +
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 4 +-
.../src/main/data/2jobs2min-rumen-jh.json | 6 -
.../sls/scheduler/ResourceSchedulerWrapper.java | 11 +-
.../sls/scheduler/SLSCapacityScheduler.java | 53 +-
.../hadoop-yarn/bin/stop-yarn.sh | 16 +-
.../jdiff/Apache_Hadoop_YARN_API_2.6.0.xml | 2 +-
.../jdiff/Apache_Hadoop_YARN_API_2.7.2.xml | 2 +-
.../api/records/AbstractResourceRequest.java | 55 -
.../yarn/api/records/ApplicationTimeout.java | 12 +-
.../hadoop/yarn/api/records/ContainerState.java | 2 +
.../yarn/api/records/ContainerUpdateType.java | 9 +-
.../hadoop/yarn/api/records/QueueState.java | 11 +-
.../yarn/api/records/ResourceRequest.java | 19 +-
.../yarn/api/records/UpdateContainerError.java | 6 +
.../api/records/UpdateContainerRequest.java | 39 +-
.../records/timelineservice/FlowRunEntity.java | 4 +-
.../org/apache/hadoop/yarn/conf/HAUtil.java | 4 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 136 +-
.../src/main/proto/yarn_protos.proto | 1 +
.../src/main/proto/yarn_service_protos.proto | 3 +-
.../distributedshell/ApplicationMaster.java | 2 +-
.../UnmanagedAMLauncher.java | 10 +-
.../api/async/impl/AMRMClientAsyncImpl.java | 30 +-
.../hadoop/yarn/client/cli/ApplicationCLI.java | 16 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 2 +-
.../hadoop/yarn/client/cli/RMAdminCLI.java | 2 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 15 +-
...stHedgingRequestRMFailoverProxyProvider.java | 57 +-
...nagerAdministrationProtocolPBClientImpl.java | 33 +-
.../api/async/impl/TestAMRMClientAsync.java | 2 +-
.../yarn/client/api/impl/TestAMRMClient.java | 2 +-
.../api/impl/TestAMRMClientOnRMRestart.java | 11 +-
.../yarn/client/api/impl/TestAMRMProxy.java | 37 +-
.../api/impl/TestDistributedScheduling.java | 1 -
.../hadoop/yarn/client/cli/TestLogsCLI.java | 15 +
.../hadoop/yarn/client/cli/TestYarnCLI.java | 77 +-
.../impl/pb/AllocateResponsePBImpl.java | 4 +-
.../client/api/impl/TimelineClientImpl.java | 2 +-
.../hadoop/yarn/event/AsyncDispatcher.java | 20 +-
.../apache/hadoop/yarn/event/Dispatcher.java | 2 +-
.../nodelabels/CommonNodeLabelsManager.java | 2 +-
.../apache/hadoop/yarn/security/AccessType.java | 2 +
.../org/apache/hadoop/yarn/state/Graph.java | 2 +-
.../yarn/state/VisualizeStateMachine.java | 4 +-
.../yarn/util/ProcfsBasedProcessTree.java | 5 +-
.../src/main/resources/yarn-default.xml | 95 +-
.../hadoop/yarn/api/BasePBImplRecordsTest.java | 2 +-
.../hadoop/yarn/event/DrainDispatcher.java | 6 +-
.../hadoop/yarn/event/InlineDispatcher.java | 2 +-
.../nodelabels/TestCommonNodeLabelsManager.java | 2 +-
.../server/timeline/LeveldbTimelineStore.java | 33 +-
.../timeline/TestLeveldbTimelineStore.java | 39 +
.../server/api/protocolrecords/RemoteNode.java | 7 +
.../OpportunisticContainerAllocator.java | 77 +-
.../OpportunisticContainerContext.java | 40 +-
.../server/scheduler/SchedulerRequestKey.java | 162 +++
.../hadoop/yarn/server/webapp/AppBlock.java | 9 +
.../src/CMakeLists.txt | 16 +
.../server/nodemanager/ContainerExecutor.java | 11 +-
.../yarn/server/nodemanager/NodeManager.java | 4 +-
.../nodemanager/amrmproxy/AMRMProxyService.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 6 +-
.../application/ApplicationImpl.java | 2 +-
.../container/ContainerImpl.java | 2 +-
.../linux/resources/TrafficController.java | 13 +-
.../localizer/LocalizedResource.java | 2 +-
.../localizer/ResourceLocalizationService.java | 1 +
.../logaggregation/AppLogAggregatorImpl.java | 4 +-
.../scheduler/ContainerScheduler.java | 12 +-
.../nodemanager/metrics/NodeManagerMetrics.java | 65 +-
.../scheduler/DistributedScheduler.java | 11 +-
.../timelineservice/NMTimelinePublisher.java | 2 +-
.../nodemanager/webapp/NMWebAppFilter.java | 8 +
.../server/nodemanager/webapp/WebServer.java | 35 +-
.../impl/container-executor.c | 12 +-
.../nodemanager/TestNodeHealthService.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 4 +-
.../launcher/TestContainerLaunch.java | 73 +-
.../TestResourceLocalizationService.java | 2 +-
.../TestAppLogAggregatorImpl.java | 4 +-
.../TestLogAggregationService.java | 2 +-
.../nodemanager/webapp/TestNMWebFilter.java | 176 +++
.../conf/capacity-scheduler.xml | 9 +
...ActiveStandbyElectorBasedElectorService.java | 271 ++++
.../server/resourcemanager/AdminService.java | 83 +-
.../ApplicationMasterService.java | 444 +++---
.../server/resourcemanager/ClientRMService.java | 13 +-
.../CuratorBasedElectorService.java | 139 ++
.../server/resourcemanager/EmbeddedElector.java | 41 +
.../resourcemanager/EmbeddedElectorService.java | 260 ----
.../resourcemanager/LeaderElectorService.java | 129 --
.../resourcemanager/NMLivelinessMonitor.java | 3 +-
...pportunisticContainerAllocatorAMService.java | 137 +-
.../server/resourcemanager/RMAppManager.java | 40 +-
.../yarn/server/resourcemanager/RMContext.java | 6 +-
.../server/resourcemanager/RMContextImpl.java | 15 +-
.../server/resourcemanager/RMServerUtils.java | 144 +-
.../server/resourcemanager/ResourceManager.java | 41 +-
.../ahs/RMApplicationHistoryWriter.java | 4 +-
.../metrics/AbstractSystemMetricsPublisher.java | 4 +-
.../recovery/FileSystemRMStateStore.java | 2 +-
.../recovery/MemoryRMStateStore.java | 2 +-
.../resourcemanager/recovery/RMStateStore.java | 2 +-
.../reservation/InMemoryPlan.java | 4 +-
.../reservation/ReservationInputValidator.java | 2 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 31 +-
.../rmapp/attempt/AMLivelinessMonitor.java | 3 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 33 +-
.../rmapp/monitor/RMAppLifetimeMonitor.java | 4 +-
.../rmcontainer/RMContainer.java | 3 +-
.../rmcontainer/RMContainerImpl.java | 34 +-
.../rmcontainer/RMContainerReservedEvent.java | 2 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 2 +-
.../scheduler/AbstractYarnScheduler.java | 138 +-
.../resourcemanager/scheduler/Allocation.java | 26 +-
.../scheduler/AppSchedulingInfo.java | 563 +++-----
.../scheduler/ContainerUpdateContext.java | 267 ++++
.../scheduler/ContainerUpdates.java | 68 +
.../scheduler/QueueStateManager.java | 100 ++
.../scheduler/SchedulerApplicationAttempt.java | 216 ++-
.../scheduler/SchedulerNode.java | 89 +-
.../scheduler/SchedulerQueue.java | 69 +
.../scheduler/SchedulerQueueManager.java | 3 +-
.../scheduler/SchedulerRequestKey.java | 122 --
.../scheduler/SchedulerUtils.java | 42 +-
.../scheduler/YarnScheduler.java | 24 +-
.../scheduler/capacity/AbstractCSQueue.java | 164 ++-
.../AppPriorityACLConfigurationParser.java | 219 +++
.../scheduler/capacity/AppPriorityACLGroup.java | 108 ++
.../scheduler/capacity/CSQueue.java | 4 +-
.../scheduler/capacity/CapacityScheduler.java | 178 ++-
.../CapacitySchedulerConfiguration.java | 44 +-
.../capacity/CapacitySchedulerContext.java | 9 +
.../capacity/CapacitySchedulerQueueManager.java | 36 +-
.../scheduler/capacity/LeafQueue.java | 44 +-
.../scheduler/capacity/ParentQueue.java | 27 +-
.../allocator/AbstractContainerAllocator.java | 4 +
.../allocator/IncreaseContainerAllocator.java | 2 +-
.../allocator/RegularContainerAllocator.java | 138 +-
.../scheduler/common/PendingAsk.java | 57 +
.../scheduler/common/SchedulerContainer.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 71 +-
.../common/fica/FiCaSchedulerNode.java | 6 +-
.../distributed/NodeQueueLoadMonitor.java | 16 +-
.../fair/AllocationFileLoaderService.java | 2 +-
.../scheduler/fair/FSAppAttempt.java | 166 +--
.../scheduler/fair/FSPreemptionThread.java | 11 +-
.../scheduler/fair/FSQueueMetrics.java | 2 +-
.../scheduler/fair/FSSchedulerNode.java | 2 +-
.../scheduler/fair/FairScheduler.java | 50 +-
.../fair/policies/FairSharePolicy.java | 19 +-
.../scheduler/fifo/FifoAppAttempt.java | 18 +-
.../scheduler/fifo/FifoScheduler.java | 141 +-
.../LocalitySchedulingPlacementSet.java | 414 ++++++
.../placement/SchedulingPlacementSet.java | 94 +-
.../security/AppPriorityACLsManager.java | 230 +++
.../webapp/CapacitySchedulerPage.java | 2 +-
.../resourcemanager/webapp/NodesPage.java | 6 +-
.../server/resourcemanager/webapp/RMWebApp.java | 3 +-
.../resourcemanager/webapp/dao/AppInfo.java | 8 +-
.../resourcemanager/webapp/dao/ClusterInfo.java | 2 +-
.../resourcemanager/webapp/dao/NodeInfo.java | 10 +-
.../server/resourcemanager/ACLsTestBase.java | 5 +-
.../server/resourcemanager/Application.java | 6 +-
.../yarn/server/resourcemanager/MockAM.java | 7 +
.../yarn/server/resourcemanager/MockNM.java | 6 +
.../yarn/server/resourcemanager/MockRM.java | 35 +-
.../server/resourcemanager/RMHATestBase.java | 4 +-
.../yarn/server/resourcemanager/Task.java | 2 +-
.../TestApplicationMasterService.java | 5 +-
.../resourcemanager/TestClientRMService.java | 13 +-
.../TestContainerResourceUsage.java | 8 +-
.../TestLeaderElectorService.java | 17 +-
...pportunisticContainerAllocatorAMService.java | 456 +++++-
.../yarn/server/resourcemanager/TestRM.java | 7 +-
.../resourcemanager/TestRMAdminService.java | 56 +-
.../resourcemanager/TestRMEmbeddedElector.java | 49 +-
.../yarn/server/resourcemanager/TestRMHA.java | 45 +-
.../resourcemanager/TestRMNodeTransitions.java | 2 +-
.../server/resourcemanager/TestRMRestart.java | 14 +-
...alCapacityPreemptionPolicyMockFramework.java | 8 +-
...estProportionalCapacityPreemptionPolicy.java | 6 +-
.../recovery/RMStateStoreTestBase.java | 2 +-
.../TestReservationInputValidator.java | 16 +-
.../rmapp/TestNodesListManager.java | 3 +-
.../attempt/TestRMAppAttemptTransitions.java | 19 +-
.../rmcontainer/TestRMContainerImpl.java | 7 +-
.../scheduler/TestAbstractYarnScheduler.java | 129 +-
.../scheduler/TestAppSchedulingInfo.java | 5 +-
.../TestSchedulerApplicationAttempt.java | 5 +-
.../scheduler/TestSchedulerUtils.java | 8 +-
.../TestApplicationLimitsByPartition.java | 6 +
.../capacity/TestApplicationPriority.java | 25 +-
...TestApplicationPriorityACLConfiguration.java | 120 ++
.../capacity/TestApplicationPriorityACLs.java | 206 +++
.../capacity/TestCapacityScheduler.java | 279 ++--
.../scheduler/capacity/TestChildQueueOrder.java | 12 +-
.../capacity/TestContainerAllocation.java | 50 +
.../scheduler/capacity/TestLeafQueue.java | 80 +-
.../TestNodeLabelContainerAllocation.java | 65 +-
.../scheduler/capacity/TestParentQueue.java | 6 +-
.../scheduler/capacity/TestQueueState.java | 104 ++
.../capacity/TestQueueStateManager.java | 162 +++
.../scheduler/capacity/TestReservations.java | 33 +-
.../scheduler/capacity/TestUtils.java | 5 +-
.../scheduler/fair/FairSchedulerTestBase.java | 14 +-
.../fair/TestContinuousScheduling.java | 11 +-
.../scheduler/fair/TestFSAppAttempt.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 35 +-
.../scheduler/fifo/TestFifoScheduler.java | 48 +-
.../resourcemanager/webapp/TestRMWebApp.java | 2 +-
.../webapp/TestRMWebServiceAppsNodelabel.java | 2 +-
.../webapp/TestRMWebServicesNodes.java | 10 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 10 +-
.../hadoop/yarn/server/TestMiniYarnCluster.java | 80 +-
.../pom.xml | 142 +-
...stTimelineReaderWebServicesHBaseStorage.java | 6 +-
...TestPhoenixOfflineAggregationWriterImpl.java | 161 ---
.../flow/TestHBaseStorageFlowActivity.java | 12 +-
.../storage/flow/TestHBaseStorageFlowRun.java | 14 +-
.../flow/TestHBaseStorageFlowRunCompaction.java | 44 +-
.../hadoop-yarn-server-timelineservice/pom.xml | 30 +-
.../collector/TimelineCollectorManager.java | 26 +-
.../reader/TimelineReaderServer.java | 26 +-
.../PhoenixOfflineAggregationWriterImpl.java | 358 -----
.../storage/TimelineSchemaCreator.java | 22 -
.../storage/common/AppIdKeyConverter.java | 5 +-
.../common/HBaseTimelineStorageUtils.java | 243 ++++
.../storage/common/TimelineStorageUtils.java | 207 ---
.../storage/flow/FlowActivityColumnPrefix.java | 10 +-
.../storage/flow/FlowActivityRowKey.java | 4 +-
.../storage/flow/FlowRunColumn.java | 6 +-
.../storage/flow/FlowRunColumnPrefix.java | 6 +-
.../storage/flow/FlowRunCoprocessor.java | 6 +-
.../storage/flow/FlowScanner.java | 13 +-
.../collector/TestTimelineCollectorManager.java | 74 +
.../reader/TestTimelineReaderServer.java | 43 +
.../storage/common/TestRowKeys.java | 2 +-
.../site/markdown/OpportunisticContainers.md | 225 +++
.../src/site/markdown/ResourceManagerRest.md | 252 +++-
.../src/site/markdown/TimelineServiceV2.md | 2 +-
.../src/site/markdown/YarnCommands.md | 7 +
.../webapp/app/components/app-timeout-bar.js | 60 +
.../src/main/webapp/app/models/yarn-app.js | 2 +
.../src/main/webapp/app/serializers/yarn-app.js | 21 +-
.../templates/components/app-timeout-bar.hbs | 34 +
.../src/main/webapp/app/templates/yarn-app.hbs | 10 +-
.../src/main/webapp/app/utils/converter.js | 3 +
.../components/app-timeout-bar-test.js | 34 +
pom.xml | 4 +-
617 files changed, 26395 insertions(+), 8108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-project/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 15604c6,567bacb..fa55586
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@@ -21,13 -21,12 +21,15 @@@ package org.apache.hadoop.fs.s3a
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase;
import org.junit.Assert;
+ import org.junit.Assume;
import org.junit.internal.AssumptionViolatedException;
import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
@@@ -532,61 -502,46 +522,104 @@@ public final class S3ATestUtils
private S3ATestUtils() {
}
+ public static void verifyFileStatus(FileStatus status, long size,
+ long blockSize, long modTime) {
+ verifyFileStatus(status, size, 0, modTime, 0, blockSize, null, null, null);
+ }
+
+ public static void verifyFileStatus(FileStatus status, long size,
+ int replication,
+ long modTime,
+ long accessTime,
+ long blockSize,
+ String owner,
+ String group,
+ FsPermission permission) {
+ String details = status.toString();
+ assertFalse("Not a dir: " + details, status.isDirectory());
+ assertEquals("Mod time: " + details, modTime, status.getModificationTime());
+ assertEquals("File size: " + details, size, status.getLen());
+ assertEquals("Block size: " + details, blockSize, status.getBlockSize());
+ if (replication > 0) {
+ assertEquals("Replication value: " + details, replication,
+ status.getReplication());
+ }
+ if (accessTime != 0) {
+ assertEquals("Access time: " + details, accessTime,
+ status.getAccessTime());
+ }
+ if (owner != null) {
+ assertEquals("Owner: " + details, owner, status.getOwner());
+ }
+ if (group != null) {
+ assertEquals("Group: " + details, group, status.getGroup());
+ }
+ if (permission != null) {
+ assertEquals("Permission: " + details, permission,
+ status.getPermission());
+ }
+ }
+
+ public static void verifyDirStatus(FileStatus status,
+ int replication,
+ long modTime,
+ long accessTime,
+ String owner,
+ String group,
+ FsPermission permission) {
+ String details = status.toString();
+ assertTrue("Is a dir: " + details, status.isDirectory());
+ assertEquals("zero length: " + details, 0, status.getLen());
+
+ assertEquals("Mod time: " + details, modTime, status.getModificationTime());
+ assertEquals("Replication value: " + details, replication,
+ status.getReplication());
+ assertEquals("Access time: " + details, accessTime, status.getAccessTime());
+ assertEquals("Owner: " + details, owner, status.getOwner());
+ assertEquals("Group: " + details, group, status.getGroup());
+ assertEquals("Permission: " + details, permission, status.getPermission());
+ }
++
+ /**
+ * Set a bucket specific property to a particular value.
+ * If the generic key passed in has an {@code fs.s3a. prefix},
+ * that's stripped off, so that when the the bucket properties are propagated
+ * down to the generic values, that value gets copied down.
+ * @param conf configuration to set
+ * @param bucket bucket name
+ * @param genericKey key; can start with "fs.s3a."
+ * @param value value to set
+ */
+ public static void setBucketOption(Configuration conf, String bucket,
+ String genericKey, String value) {
+ final String baseKey = genericKey.startsWith(FS_S3A_PREFIX) ?
+ genericKey.substring(FS_S3A_PREFIX.length())
+ : genericKey;
+ conf.set(FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey, value);
+ }
+
+ /**
+ * Assert that a configuration option matches the expected value.
+ * @param conf configuration
+ * @param key option key
+ * @param expected expected value
+ */
+ public static void assertOptionEquals(Configuration conf,
+ String key,
+ String expected) {
+ assertEquals("Value of " + key, expected, conf.get(key));
+ }
+
+ /**
+ * Assume that a condition is met. If not: log at WARN and
+ * then throw an {@link AssumptionViolatedException}.
+ * @param message
+ * @param condition
+ */
+ public static void assume(String message, boolean condition) {
+ if (!condition) {
+ LOG.warn(message);
+ }
+ Assume.assumeTrue(message, condition);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-5899. Debug log in
AbstractCSQueue#canAssignToThisQueue needs improvement. Contributed by Ying
Zhang.
Posted by st...@apache.org.
YARN-5899. Debug log in AbstractCSQueue#canAssignToThisQueue needs improvement. Contributed by Ying Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9594c35d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9594c35d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9594c35d
Branch: refs/heads/HADOOP-13345
Commit: 9594c35dcb655add1991d8fd15897b40c4ad6205
Parents: 6332a31
Author: Sunil G <su...@apache.org>
Authored: Tue Jan 10 14:08:01 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Jan 10 14:08:01 2017 +0530
----------------------------------------------------------------------
.../scheduler/capacity/AbstractCSQueue.java | 45 ++++++++++++--------
1 file changed, 28 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9594c35d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index d1fa410..cefa1e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -632,31 +632,42 @@ public abstract class AbstractCSQueue implements CSQueue {
if (Resources.lessThan(resourceCalculator, clusterResource,
newTotalWithoutReservedResource, currentLimitResource)) {
if (LOG.isDebugEnabled()) {
- LOG.debug(
- "try to use reserved: " + getQueueName() + " usedResources: "
- + queueUsage.getUsed() + ", clusterResources: "
- + clusterResource + ", reservedResources: "
- + resourceCouldBeUnreserved
- + ", capacity-without-reserved: "
- + newTotalWithoutReservedResource + ", maxLimitCapacity: "
- + currentLimitResource);
+ LOG.debug("try to use reserved: " + getQueueName()
+ + " usedResources: " + queueUsage.getUsed()
+ + ", clusterResources: " + clusterResource
+ + ", reservedResources: " + resourceCouldBeUnreserved
+ + ", capacity-without-reserved: "
+ + newTotalWithoutReservedResource
+ + ", maxLimitCapacity: " + currentLimitResource);
}
return true;
}
}
+
+ // Can not assign to this queue
if (LOG.isDebugEnabled()) {
- LOG.debug(getQueueName() + "Check assign to queue, nodePartition="
- + nodePartition + " usedResources: " + queueUsage
- .getUsed(nodePartition) + " clusterResources: " + clusterResource
- + " currentUsedCapacity " + Resources
- .divide(resourceCalculator, clusterResource,
- queueUsage.getUsed(nodePartition), labelManager
- .getResourceByLabel(nodePartition, clusterResource))
- + " max-capacity: " + queueCapacities
- .getAbsoluteMaximumCapacity(nodePartition) + ")");
+ LOG.debug("Failed to assign to queue: " + getQueueName()
+ + " nodePatrition: " + nodePartition
+ + ", usedResources: " + queueUsage.getUsed(nodePartition)
+ + ", clusterResources: " + clusterResource
+ + ", reservedResources: " + resourceCouldBeUnreserved
+ + ", maxLimitCapacity: " + currentLimitResource
+ + ", currTotalUsed:" + usedExceptKillable);
}
return false;
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Check assign to queue: " + getQueueName()
+ + " nodePartition: " + nodePartition
+ + ", usedResources: " + queueUsage.getUsed(nodePartition)
+ + ", clusterResources: " + clusterResource
+ + ", currentUsedCapacity: " + Resources
+ .divide(resourceCalculator, clusterResource,
+ queueUsage.getUsed(nodePartition), labelManager
+ .getResourceByLabel(nodePartition, clusterResource))
+ + ", max-capacity: " + queueCapacities
+ .getAbsoluteMaximumCapacity(nodePartition));
+ }
return true;
} finally {
readLock.unlock();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-4899. Queue metrics of SLS
capacity scheduler only activated after app submit to the queue. (Jonathan
Hung via wangda)
Posted by st...@apache.org.
YARN-4899. Queue metrics of SLS capacity scheduler only activated after app submit to the queue. (Jonathan Hung via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ed63e3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ed63e3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ed63e3e
Branch: refs/heads/HADOOP-13345
Commit: 5ed63e3e9d9937cf7441b7ceb5feafbf486f3387
Parents: a0a2761
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Jan 4 19:17:11 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Jan 4 19:17:11 2017 -0800
----------------------------------------------------------------------
.../sls/scheduler/SLSCapacityScheduler.java | 44 ++++++++++++++++++++
1 file changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ed63e3e/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index cf08309..89f9ad3 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -57,7 +57,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
@@ -84,6 +86,7 @@ import com.codahale.metrics.Timer;
public class SLSCapacityScheduler extends CapacityScheduler implements
SchedulerWrapper,Configurable {
private static final String EOL = System.getProperty("line.separator");
+ private static final String QUEUE_COUNTER_PREFIX = "counter.queue.";
private static final int SAMPLING_SIZE = 60;
private ScheduledExecutorService pool;
// counters for scheduler allocate/handle operations
@@ -747,6 +750,47 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
}
}
+ private void initQueueMetrics(CSQueue queue) {
+ if (queue instanceof LeafQueue) {
+ SortedMap<String, Counter> counterMap = metrics.getCounters();
+ String queueName = queue.getQueueName();
+ String[] names = new String[]{
+ QUEUE_COUNTER_PREFIX + queueName + ".pending.memory",
+ QUEUE_COUNTER_PREFIX + queueName + ".pending.cores",
+ QUEUE_COUNTER_PREFIX + queueName + ".allocated.memory",
+ QUEUE_COUNTER_PREFIX + queueName + ".allocated.cores" };
+
+ for (int i = names.length - 1; i >= 0; i--) {
+ if (!counterMap.containsKey(names[i])) {
+ metrics.counter(names[i]);
+ counterMap = metrics.getCounters();
+ }
+ }
+
+ queueLock.lock();
+ try {
+ if (!schedulerMetrics.isTracked(queueName)) {
+ schedulerMetrics.trackQueue(queueName);
+ }
+ } finally {
+ queueLock.unlock();
+ }
+
+ return;
+ }
+
+ for (CSQueue child : queue.getChildQueues()) {
+ initQueueMetrics(child);
+ }
+ }
+
+ @Override
+ public void serviceInit(Configuration configuration) throws Exception {
+ super.serviceInit(configuration);
+
+ initQueueMetrics(getRootQueue());
+ }
+
public void setQueueSet(Set<String> queues) {
this.queueSet = queues;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-5937. stop-yarn.sh is not
able to gracefully stop node managers. Contributed by Weiwei Yang.
Posted by st...@apache.org.
YARN-5937. stop-yarn.sh is not able to gracefully stop node managers. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41db07d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41db07d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41db07d5
Branch: refs/heads/HADOOP-13345
Commit: 41db07d532f41fd35b11935b2bb042973831951b
Parents: 9594c35
Author: Naganarasimha <na...@apache.org>
Authored: Tue Jan 10 15:35:01 2017 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Tue Jan 10 15:35:01 2017 +0530
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41db07d5/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
index 1ed52dd..f6c6b78 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
@@ -42,6 +42,14 @@ else
exit 1
fi
+# stop nodemanager
+echo "Stopping nodemanagers"
+"${HADOOP_YARN_HOME}/bin/yarn" \
+ --config "${HADOOP_CONF_DIR}" \
+ --workers \
+ --daemon stop \
+ nodemanager
+
# stop resourceManager
HARM=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.enabled 2>&-)
if [[ ${HARM} = "false" ]]; then
@@ -67,14 +75,6 @@ else
resourcemanager
fi
-# stop nodemanager
-echo "Stopping nodemanagers"
-"${HADOOP_YARN_HOME}/bin/yarn" \
- --config "${HADOOP_CONF_DIR}" \
- --workers \
- --daemon stop \
- nodemanager
-
# stop proxyserver
PROXYSERVER=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.web-proxy.address 2>&- | cut -f1 -d:)
if [[ -n ${PROXYSERVER} ]]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-6040. Introduce api
independent PendingAsk to replace usage of ResourceRequest within Scheduler
classes. (Wangda Tan via asuresh)
Posted by st...@apache.org.
YARN-6040. Introduce api independent PendingAsk to replace usage of ResourceRequest within Scheduler classes. (Wangda Tan via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2977bc6a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2977bc6a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2977bc6a
Branch: refs/heads/HADOOP-13345
Commit: 2977bc6a141041ef7579efc416e93fc55e0c2a1a
Parents: 4a659ff
Author: Arun Suresh <as...@apache.org>
Authored: Fri Jan 6 09:59:57 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Jan 6 09:59:57 2017 -0800
----------------------------------------------------------------------
.../scheduler/AppSchedulingInfo.java | 163 +++++++------
.../scheduler/SchedulerApplicationAttempt.java | 44 ++--
.../allocator/AbstractContainerAllocator.java | 4 +
.../allocator/RegularContainerAllocator.java | 132 +++++-----
.../scheduler/common/PendingAsk.java | 57 +++++
.../scheduler/common/fica/FiCaSchedulerApp.java | 55 +++--
.../scheduler/fair/FSAppAttempt.java | 160 +++++++------
.../scheduler/fair/FSPreemptionThread.java | 11 +-
.../scheduler/fifo/FifoAppAttempt.java | 10 +-
.../scheduler/fifo/FifoScheduler.java | 121 +++++-----
.../LocalitySchedulingPlacementSet.java | 129 ++++++++--
.../placement/SchedulingPlacementSet.java | 71 +++++-
.../scheduler/TestAbstractYarnScheduler.java | 15 +-
.../scheduler/TestAppSchedulingInfo.java | 4 +-
.../TestSchedulerApplicationAttempt.java | 3 +-
.../capacity/TestCapacityScheduler.java | 238 +++++++++----------
.../scheduler/capacity/TestLeafQueue.java | 67 +++---
.../TestNodeLabelContainerAllocation.java | 65 +++--
.../scheduler/capacity/TestReservations.java | 22 +-
19 files changed, 803 insertions(+), 568 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index d901d90..e2ff082 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -34,16 +34,18 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.LocalitySchedulingPlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.ResourceRequestUpdateResult;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -582,16 +584,10 @@ public class AppSchedulingInfo {
return schedulerKeys.keySet();
}
- @SuppressWarnings("unchecked")
- public Map<String, ResourceRequest> getResourceRequests(
- SchedulerRequestKey schedulerKey) {
- SchedulingPlacementSet ps = schedulerKeyToPlacementSets.get(schedulerKey);
- if (null != ps) {
- return ps.getResourceRequests();
- }
- return Collections.emptyMap();
- }
-
+ /**
+ * Used by REST API to fetch ResourceRequest
+ * @return All pending ResourceRequests.
+ */
public List<ResourceRequest> getAllResourceRequests() {
List<ResourceRequest> ret = new ArrayList<>();
try {
@@ -605,53 +601,51 @@ public class AppSchedulingInfo {
return ret;
}
- public ResourceRequest getResourceRequest(SchedulerRequestKey schedulerKey,
- String resourceName) {
+ public SchedulingPlacementSet getFirstSchedulingPlacementSet() {
try {
- this.readLock.lock();
- SchedulingPlacementSet ps =
- schedulerKeyToPlacementSets.get(schedulerKey);
- return (ps == null) ? null : ps.getResourceRequest(resourceName);
+ readLock.lock();
+ for (SchedulerRequestKey key : schedulerKeys.keySet()) {
+ SchedulingPlacementSet ps = schedulerKeyToPlacementSets.get(key);
+ if (null != ps) {
+ return ps;
+ }
+ }
+ return null;
} finally {
- this.readLock.unlock();
+ readLock.unlock();
}
+
}
- public Resource getResource(SchedulerRequestKey schedulerKey) {
+ public PendingAsk getNextPendingAsk() {
try {
- this.readLock.lock();
- ResourceRequest request =
- getResourceRequest(schedulerKey, ResourceRequest.ANY);
- return (request == null) ? null : request.getCapability();
+ readLock.lock();
+ SchedulerRequestKey firstRequestKey = schedulerKeys.firstKey();
+ return getPendingAsk(firstRequestKey, ResourceRequest.ANY);
} finally {
- this.readLock.unlock();
+ readLock.unlock();
}
+
}
- /**
- * Method to return the next resource request to be serviced.
- *
- * In the initial implementation, we just pick any {@link ResourceRequest}
- * corresponding to the highest priority.
- *
- * @return next {@link ResourceRequest} to allocate resources for.
- */
- @Unstable
- public synchronized ResourceRequest getNextResourceRequest() {
- SchedulingPlacementSet<SchedulerNode> ps = schedulerKeyToPlacementSets.get(
- schedulerKeys.firstKey());
- if (null != ps) {
- for (ResourceRequest rr : ps.getResourceRequests().values()) {
- return rr;
- }
- }
+ public PendingAsk getPendingAsk(SchedulerRequestKey schedulerKey) {
+ return getPendingAsk(schedulerKey, ResourceRequest.ANY);
+ }
- return null;
+ public PendingAsk getPendingAsk(SchedulerRequestKey schedulerKey,
+ String resourceName) {
+ try {
+ this.readLock.lock();
+ SchedulingPlacementSet ps = schedulerKeyToPlacementSets.get(schedulerKey);
+ return (ps == null) ? PendingAsk.ZERO : ps.getPendingAsk(resourceName);
+ } finally {
+ this.readLock.unlock();
+ }
}
/**
* Returns if the place (node/rack today) is either blacklisted by the
- * application (user) or the system
+ * application (user) or the system.
*
* @param resourceName
* the resourcename
@@ -724,7 +718,6 @@ public class AppSchedulingInfo {
public List<ResourceRequest> allocate(NodeType type,
SchedulerNode node, SchedulerRequestKey schedulerKey,
- ResourceRequest request,
Container containerAllocated) {
try {
writeLock.lock();
@@ -733,19 +726,13 @@ public class AppSchedulingInfo {
updateMetricsForAllocatedContainer(type, containerAllocated);
}
- return schedulerKeyToPlacementSets.get(schedulerKey)
- .allocate(schedulerKey, type, node, request);
+ return schedulerKeyToPlacementSets.get(schedulerKey).allocate(
+ schedulerKey, type, node);
} finally {
writeLock.unlock();
}
}
- public List<ResourceRequest> allocate(NodeType type,
- SchedulerNode node, SchedulerRequestKey schedulerKey,
- Container containerAllocated) {
- return allocate(type, node, schedulerKey, null, containerAllocated);
- }
-
public void checkForDeactivation() {
if (schedulerKeys.isEmpty()) {
activeUsersManager.deactivateApplication(user, applicationId);
@@ -758,18 +745,20 @@ public class AppSchedulingInfo {
QueueMetrics oldMetrics = queue.getMetrics();
QueueMetrics newMetrics = newQueue.getMetrics();
for (SchedulingPlacementSet ps : schedulerKeyToPlacementSets.values()) {
- ResourceRequest request = ps.getResourceRequest(ResourceRequest.ANY);
- if (request != null && request.getNumContainers() > 0) {
- oldMetrics.decrPendingResources(user, request.getNumContainers(),
- request.getCapability());
- newMetrics.incrPendingResources(user, request.getNumContainers(),
- request.getCapability());
-
- Resource delta = Resources.multiply(request.getCapability(),
- request.getNumContainers());
+ PendingAsk ask = ps.getPendingAsk(ResourceRequest.ANY);
+ if (ask.getCount() > 0) {
+ oldMetrics.decrPendingResources(user, ask.getCount(),
+ ask.getPerAllocationResource());
+ newMetrics.incrPendingResources(user, ask.getCount(),
+ ask.getPerAllocationResource());
+
+ Resource delta = Resources.multiply(ask.getPerAllocationResource(),
+ ask.getCount());
// Update Queue
- queue.decPendingResource(request.getNodeLabelExpression(), delta);
- newQueue.incPendingResource(request.getNodeLabelExpression(), delta);
+ queue.decPendingResource(
+ ps.getPrimaryRequestedNodePartition(), delta);
+ newQueue.incPendingResource(
+ ps.getPrimaryRequestedNodePartition(), delta);
}
}
oldMetrics.moveAppFrom(this);
@@ -789,16 +778,16 @@ public class AppSchedulingInfo {
this.writeLock.lock();
QueueMetrics metrics = queue.getMetrics();
for (SchedulingPlacementSet ps : schedulerKeyToPlacementSets.values()) {
- ResourceRequest request = ps.getResourceRequest(ResourceRequest.ANY);
- if (request != null && request.getNumContainers() > 0) {
- metrics.decrPendingResources(user, request.getNumContainers(),
- request.getCapability());
+ PendingAsk ask = ps.getPendingAsk(ResourceRequest.ANY);
+ if (ask.getCount() > 0) {
+ metrics.decrPendingResources(user, ask.getCount(),
+ ask.getPerAllocationResource());
// Update Queue
queue.decPendingResource(
- request.getNodeLabelExpression(),
- Resources.multiply(request.getCapability(),
- request.getNumContainers()));
+ ps.getPrimaryRequestedNodePartition(),
+ Resources.multiply(ask.getPerAllocationResource(),
+ ask.getCount()));
}
}
metrics.finishAppAttempt(applicationId, pending, user);
@@ -906,4 +895,38 @@ public class AppSchedulingInfo {
return (SchedulingPlacementSet<N>) schedulerKeyToPlacementSets.get(
schedulerkey);
}
+
+ /**
+ * Can delay to next?.
+ *
+ * @param schedulerKey schedulerKey
+ * @param resourceName resourceName
+ *
+ * @return If request exists, return {relaxLocality}
+ * Otherwise, return true.
+ */
+ public boolean canDelayTo(
+ SchedulerRequestKey schedulerKey, String resourceName) {
+ try {
+ this.readLock.lock();
+ SchedulingPlacementSet ps =
+ schedulerKeyToPlacementSets.get(schedulerKey);
+ return (ps == null) || ps.canDelayTo(resourceName);
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ public boolean acceptNodePartition(SchedulerRequestKey schedulerKey,
+ String nodePartition, SchedulingMode schedulingMode) {
+ try {
+ this.readLock.lock();
+ SchedulingPlacementSet ps =
+ schedulerKeyToPlacementSets.get(schedulerKey);
+ return (ps != null) && ps.acceptNodePartition(nodePartition,
+ schedulingMode);
+ } finally {
+ this.readLock.unlock();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 4a8b2da..1f6bd1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -78,6 +77,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Scheduli
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
@@ -283,11 +283,6 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return appSchedulingInfo.getUser();
}
- public Map<String, ResourceRequest> getResourceRequests(
- SchedulerRequestKey schedulerKey) {
- return appSchedulingInfo.getResourceRequests(schedulerKey);
- }
-
public Set<ContainerId> getPendingRelease() {
return this.pendingRelease;
}
@@ -299,34 +294,28 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
public Collection<SchedulerRequestKey> getSchedulerKeys() {
return appSchedulingInfo.getSchedulerKeys();
}
-
- public ResourceRequest getResourceRequest(
+
+ public PendingAsk getPendingAsk(
SchedulerRequestKey schedulerKey, String resourceName) {
try {
readLock.lock();
- return appSchedulingInfo.getResourceRequest(schedulerKey, resourceName);
+ return appSchedulingInfo.getPendingAsk(schedulerKey, resourceName);
} finally {
readLock.unlock();
}
-
}
- public int getTotalRequiredResources(
- SchedulerRequestKey schedulerKey) {
- try {
- readLock.lock();
- ResourceRequest request =
- getResourceRequest(schedulerKey, ResourceRequest.ANY);
- return request == null ? 0 : request.getNumContainers();
- } finally {
- readLock.unlock();
- }
+ public int getOutstandingAsksCount(SchedulerRequestKey schedulerKey) {
+ return getOutstandingAsksCount(schedulerKey, ResourceRequest.ANY);
}
- public Resource getResource(SchedulerRequestKey schedulerKey) {
+ public int getOutstandingAsksCount(SchedulerRequestKey schedulerKey,
+ String resourceName) {
try {
readLock.lock();
- return appSchedulingInfo.getResource(schedulerKey);
+ SchedulingPlacementSet ps = appSchedulingInfo.getSchedulingPlacementSet(
+ schedulerKey);
+ return ps == null ? 0 : ps.getOutstandingAsksCount(resourceName);
} finally {
readLock.unlock();
}
@@ -625,16 +614,13 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
try {
readLock.lock();
for (SchedulerRequestKey schedulerKey : getSchedulerKeys()) {
- Map<String, ResourceRequest> requests = getResourceRequests(
- schedulerKey);
- if (requests != null) {
+ SchedulingPlacementSet ps = getSchedulingPlacementSet(schedulerKey);
+ if (ps != null &&
+ ps.getOutstandingAsksCount(ResourceRequest.ANY) > 0) {
LOG.debug("showRequests:" + " application=" + getApplicationId()
+ " headRoom=" + getHeadroom() + " currentConsumption="
+ attemptResourceUsage.getUsed().getMemorySize());
- for (ResourceRequest request : requests.values()) {
- LOG.debug("showRequests:" + " application=" + getApplicationId()
- + " request=" + request);
- }
+ ps.showRequests();
}
}
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
index 5bb91e2..a411f17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
@@ -46,6 +47,7 @@ public abstract class AbstractContainerAllocator {
private static final Log LOG = LogFactory.getLog(AbstractContainerAllocator.class);
FiCaSchedulerApp application;
+ AppSchedulingInfo appInfo;
final ResourceCalculator rc;
final RMContext rmContext;
ActivitiesManager activitiesManager;
@@ -59,6 +61,8 @@ public abstract class AbstractContainerAllocator {
ResourceCalculator rc, RMContext rmContext,
ActivitiesManager activitiesManager) {
this.application = application;
+ this.appInfo =
+ application == null ? null : application.getAppSchedulingInfo();
this.rc = rc;
this.rmContext = rmContext;
this.activitiesManager = activitiesManager;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index eeb0815..8078bcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger;
@@ -54,6 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -64,8 +64,6 @@ import org.apache.hadoop.yarn.util.resource.Resources;
*/
public class RegularContainerAllocator extends AbstractContainerAllocator {
private static final Log LOG = LogFactory.getLog(RegularContainerAllocator.class);
-
- private ResourceRequest lastResourceRequest = null;
public RegularContainerAllocator(FiCaSchedulerApp application,
ResourceCalculator rc, RMContext rmContext,
@@ -103,9 +101,10 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
Priority priority = schedulerKey.getPriority();
FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
- ResourceRequest anyRequest =
- application.getResourceRequest(schedulerKey, ResourceRequest.ANY);
- if (null == anyRequest) {
+ PendingAsk offswitchPendingAsk = application.getPendingAsk(schedulerKey,
+ ResourceRequest.ANY);
+
+ if (offswitchPendingAsk.getCount() <= 0) {
ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
activitiesManager, node, application, priority,
ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST);
@@ -113,10 +112,10 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
// Required resource
- Resource required = anyRequest.getCapability();
+ Resource required = offswitchPendingAsk.getPerAllocationResource();
// Do we need containers at this 'priority'?
- if (application.getTotalRequiredResources(schedulerKey) <= 0) {
+ if (application.getOutstandingAsksCount(schedulerKey) <= 0) {
ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
activitiesManager, node, application, priority,
ActivityDiagnosticConstant.APPLICATION_PRIORITY_DO_NOT_NEED_RESOURCE);
@@ -141,11 +140,9 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
}
- // Is the node-label-expression of this offswitch resource request
- // matches the node's label?
+ // Is the nodePartition of pending request matches the node's partition
// If not match, jump to next priority.
- if (!SchedulerUtils.checkResourceRequestMatchingNodePartition(
- anyRequest.getNodeLabelExpression(), ps.getPartition(),
+ if (!appInfo.acceptNodePartition(schedulerKey, node.getPartition(),
schedulingMode)) {
ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
activitiesManager, node, application, priority,
@@ -182,8 +179,11 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
// This is to make sure non-partitioned-resource-request will prefer
// to be allocated to non-partitioned nodes
int missedNonPartitionedRequestSchedulingOpportunity = 0;
- if (anyRequest.getNodeLabelExpression()
- .equals(RMNodeLabelsManager.NO_LABEL)) {
+ // Only do this when request associated with given scheduler key accepts
+ // NO_LABEL under RESPECT_EXCLUSIVITY mode
+ if (StringUtils.equals(RMNodeLabelsManager.NO_LABEL,
+ appInfo.getSchedulingPlacementSet(schedulerKey)
+ .getPrimaryRequestedNodePartition())) {
missedNonPartitionedRequestSchedulingOpportunity =
application.addMissedNonPartitionedRequestSchedulingOpportunity(
schedulerKey);
@@ -264,8 +264,9 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
public float getLocalityWaitFactor(
SchedulerRequestKey schedulerKey, int clusterNodes) {
// Estimate: Required unique resources (i.e. hosts + racks)
- int requiredResources =
- Math.max(application.getResourceRequests(schedulerKey).size() - 1, 0);
+ int requiredResources = Math.max(
+ application.getSchedulingPlacementSet(schedulerKey)
+ .getUniqueLocationAsks() - 1, 0);
// waitFactor can't be more than '1'
// i.e. no point skipping more than clustersize opportunities
@@ -287,11 +288,10 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
// 'Delay' off-switch
- ResourceRequest offSwitchRequest =
- application.getResourceRequest(schedulerKey, ResourceRequest.ANY);
long missedOpportunities =
application.getSchedulingOpportunities(schedulerKey);
- long requiredContainers = offSwitchRequest.getNumContainers();
+ long requiredContainers = application.getOutstandingAsksCount(
+ schedulerKey);
float localityWaitFactor =
getLocalityWaitFactor(schedulerKey, rmContext.getScheduler()
@@ -304,9 +304,8 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
// Check if we need containers on this rack
- ResourceRequest rackLocalRequest =
- application.getResourceRequest(schedulerKey, node.getRackName());
- if (rackLocalRequest == null || rackLocalRequest.getNumContainers() <= 0) {
+ if (application.getOutstandingAsksCount(schedulerKey, node.getRackName())
+ <= 0) {
return false;
}
@@ -321,24 +320,21 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
// Check if we need containers on this host
if (type == NodeType.NODE_LOCAL) {
// Now check if we need containers on this host...
- ResourceRequest nodeLocalRequest =
- application.getResourceRequest(schedulerKey, node.getNodeName());
- if (nodeLocalRequest != null) {
- return nodeLocalRequest.getNumContainers() > 0;
- }
+ return application.getOutstandingAsksCount(schedulerKey,
+ node.getNodeName()) > 0;
}
return false;
}
private ContainerAllocation assignNodeLocalContainers(
- Resource clusterResource, ResourceRequest nodeLocalResourceRequest,
+ Resource clusterResource, PendingAsk nodeLocalAsk,
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
RMContainer reservedContainer, SchedulingMode schedulingMode,
ResourceLimits currentResoureLimits) {
if (canAssign(schedulerKey, node, NodeType.NODE_LOCAL, reservedContainer)) {
return assignContainer(clusterResource, node, schedulerKey,
- nodeLocalResourceRequest, NodeType.NODE_LOCAL, reservedContainer,
+ nodeLocalAsk, NodeType.NODE_LOCAL, reservedContainer,
schedulingMode, currentResoureLimits);
}
@@ -350,13 +346,13 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
private ContainerAllocation assignRackLocalContainers(
- Resource clusterResource, ResourceRequest rackLocalResourceRequest,
+ Resource clusterResource, PendingAsk rackLocalAsk,
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
RMContainer reservedContainer, SchedulingMode schedulingMode,
ResourceLimits currentResoureLimits) {
if (canAssign(schedulerKey, node, NodeType.RACK_LOCAL, reservedContainer)) {
return assignContainer(clusterResource, node, schedulerKey,
- rackLocalResourceRequest, NodeType.RACK_LOCAL, reservedContainer,
+ rackLocalAsk, NodeType.RACK_LOCAL, reservedContainer,
schedulingMode, currentResoureLimits);
}
@@ -368,13 +364,13 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
private ContainerAllocation assignOffSwitchContainers(
- Resource clusterResource, ResourceRequest offSwitchResourceRequest,
+ Resource clusterResource, PendingAsk offSwitchAsk,
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
RMContainer reservedContainer, SchedulingMode schedulingMode,
ResourceLimits currentResoureLimits) {
if (canAssign(schedulerKey, node, NodeType.OFF_SWITCH, reservedContainer)) {
return assignContainer(clusterResource, node, schedulerKey,
- offSwitchResourceRequest, NodeType.OFF_SWITCH, reservedContainer,
+ offSwitchAsk, NodeType.OFF_SWITCH, reservedContainer,
schedulingMode, currentResoureLimits);
}
@@ -396,12 +392,12 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
NodeType requestLocalityType = null;
// Data-local
- ResourceRequest nodeLocalResourceRequest =
- application.getResourceRequest(schedulerKey, node.getNodeName());
- if (nodeLocalResourceRequest != null) {
+ PendingAsk nodeLocalAsk =
+ application.getPendingAsk(schedulerKey, node.getNodeName());
+ if (nodeLocalAsk.getCount() > 0) {
requestLocalityType = NodeType.NODE_LOCAL;
allocation =
- assignNodeLocalContainers(clusterResource, nodeLocalResourceRequest,
+ assignNodeLocalContainers(clusterResource, nodeLocalAsk,
node, schedulerKey, reservedContainer, schedulingMode,
currentResoureLimits);
if (Resources.greaterThan(rc, clusterResource,
@@ -412,10 +408,10 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
// Rack-local
- ResourceRequest rackLocalResourceRequest =
- application.getResourceRequest(schedulerKey, node.getRackName());
- if (rackLocalResourceRequest != null) {
- if (!rackLocalResourceRequest.getRelaxLocality()) {
+ PendingAsk rackLocalAsk =
+ application.getPendingAsk(schedulerKey, node.getRackName());
+ if (rackLocalAsk.getCount() > 0) {
+ if (!appInfo.canDelayTo(schedulerKey, node.getRackName())) {
ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
activitiesManager, node, application, priority,
ActivityDiagnosticConstant.SKIP_PRIORITY_BECAUSE_OF_RELAX_LOCALITY);
@@ -427,7 +423,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
requestLocalityType;
allocation =
- assignRackLocalContainers(clusterResource, rackLocalResourceRequest,
+ assignRackLocalContainers(clusterResource, rackLocalAsk,
node, schedulerKey, reservedContainer, schedulingMode,
currentResoureLimits);
if (Resources.greaterThan(rc, clusterResource,
@@ -438,10 +434,10 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
// Off-switch
- ResourceRequest offSwitchResourceRequest =
- application.getResourceRequest(schedulerKey, ResourceRequest.ANY);
- if (offSwitchResourceRequest != null) {
- if (!offSwitchResourceRequest.getRelaxLocality()) {
+ PendingAsk offSwitchAsk =
+ application.getPendingAsk(schedulerKey, ResourceRequest.ANY);
+ if (offSwitchAsk.getCount() > 0) {
+ if (!appInfo.canDelayTo(schedulerKey, ResourceRequest.ANY)) {
ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
activitiesManager, node, application, priority,
ActivityDiagnosticConstant.SKIP_PRIORITY_BECAUSE_OF_RELAX_LOCALITY);
@@ -453,7 +449,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
requestLocalityType;
allocation =
- assignOffSwitchContainers(clusterResource, offSwitchResourceRequest,
+ assignOffSwitchContainers(clusterResource, offSwitchAsk,
node, schedulerKey, reservedContainer, schedulingMode,
currentResoureLimits);
@@ -474,41 +470,25 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
private ContainerAllocation assignContainer(Resource clusterResource,
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
- ResourceRequest request, NodeType type, RMContainer rmContainer,
+ PendingAsk pendingAsk, NodeType type, RMContainer rmContainer,
SchedulingMode schedulingMode, ResourceLimits currentResoureLimits) {
Priority priority = schedulerKey.getPriority();
- lastResourceRequest = request;
-
+
if (LOG.isDebugEnabled()) {
LOG.debug("assignContainers: node=" + node.getNodeName()
+ " application=" + application.getApplicationId()
+ " priority=" + schedulerKey.getPriority()
- + " request=" + request + " type=" + type);
- }
-
- // check if the resource request can access the label
- if (!SchedulerUtils.checkResourceRequestMatchingNodePartition(
- request.getNodeLabelExpression(), node.getPartition(),
- schedulingMode)) {
- // this is a reserved container, but we cannot allocate it now according
- // to label not match. This can be caused by node label changed
- // We should un-reserve this container.
- ActivitiesLogger.APP.recordAppActivityWithoutAllocation(activitiesManager,
- node, application, priority,
- ActivityDiagnosticConstant.REQUEST_CAN_NOT_ACCESS_NODE_LABEL,
- ActivityState.REJECTED);
- return new ContainerAllocation(rmContainer, null,
- AllocationState.LOCALITY_SKIPPED);
+ + " pendingAsk=" + pendingAsk + " type=" + type);
}
- Resource capability = request.getCapability();
+ Resource capability = pendingAsk.getPerAllocationResource();
Resource available = node.getUnallocatedResource();
Resource totalResource = node.getTotalResource();
if (!Resources.lessThanOrEqual(rc, clusterResource,
capability, totalResource)) {
LOG.warn("Node : " + node.getNodeID()
- + " does not have sufficient resource for request : " + request
+ + " does not have sufficient resource for ask : " + pendingAsk
+ " node total capability : " + node.getTotalResource());
// Skip this locality request
ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
@@ -600,9 +580,8 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
}
- ContainerAllocation result =
- new ContainerAllocation(unreservedContainer, request.getCapability(),
- AllocationState.ALLOCATED);
+ ContainerAllocation result = new ContainerAllocation(unreservedContainer,
+ pendingAsk.getPerAllocationResource(), AllocationState.ALLOCATED);
result.containerNodeType = type;
result.setToKillContainers(toKillContainers);
return result;
@@ -626,9 +605,8 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
}
- ContainerAllocation result =
- new ContainerAllocation(null, request.getCapability(),
- AllocationState.RESERVED);
+ ContainerAllocation result = new ContainerAllocation(null,
+ pendingAsk.getPerAllocationResource(), AllocationState.RESERVED);
result.containerNodeType = type;
result.setToKillContainers(null);
return result;
@@ -644,7 +622,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
boolean shouldAllocOrReserveNewContainer(
SchedulerRequestKey schedulerKey, Resource required) {
int requiredContainers =
- application.getTotalRequiredResources(schedulerKey);
+ application.getOutstandingAsksCount(schedulerKey);
int reservedContainers = application.getNumReservedContainers(schedulerKey);
int starvation = 0;
if (reservedContainers > 0) {
@@ -699,7 +677,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
SchedulerRequestKey schedulerKey, Container container) {
// Inform the application
RMContainer allocatedContainer = application.allocate(node, schedulerKey,
- lastResourceRequest, container);
+ container);
allocationResult.updatedContainer = allocatedContainer;
@@ -803,7 +781,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
} else {
// pre-check when allocating reserved container
- if (application.getTotalRequiredResources(schedulerKey) == 0) {
+ if (application.getOutstandingAsksCount(schedulerKey) == 0) {
// Release
return new ContainerAllocation(reservedContainer, null,
AllocationState.QUEUE_SKIPPED);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
new file mode 100644
index 0000000..85d8715
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.common;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * {@link PendingAsk} is the class to include minimal information of how much
+ * resource to ask under constraints (e.g. on one host / rack / node-attributes)
+ * , etc.
+ */
+public class PendingAsk {
+ private final Resource perAllocationResource;
+ private final int count;
+ public final static PendingAsk ZERO = new PendingAsk(Resources.none(), 0);
+
+ public PendingAsk(Resource res, int num) {
+ this.perAllocationResource = res;
+ this.count = num;
+ }
+
+ public Resource getPerAllocationResource() {
+ return perAllocationResource;
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<per-allocation-resource=");
+ sb.append(getPerAllocationResource());
+ sb.append(",repeat=");
+ sb.append(getCount());
+ sb.append(">");
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 809446f..4329335 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -70,6 +70,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -206,8 +208,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
}
public RMContainer allocate(FiCaSchedulerNode node,
- SchedulerRequestKey schedulerKey, ResourceRequest request,
- Container container) {
+ SchedulerRequestKey schedulerKey, Container container) {
try {
readLock.lock();
@@ -217,7 +218,16 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
// Required sanity check - AM can call 'allocate' to update resource
// request without locking the scheduler, hence we need to check
- if (getTotalRequiredResources(schedulerKey) <= 0) {
+ if (getOutstandingAsksCount(schedulerKey) <= 0) {
+ return null;
+ }
+
+ SchedulingPlacementSet<FiCaSchedulerNode> ps =
+ appSchedulingInfo.getSchedulingPlacementSet(schedulerKey);
+ if (null == ps) {
+ LOG.warn("Failed to get " + SchedulingPlacementSet.class.getName()
+ + " for application=" + getApplicationId() + " schedulerRequestKey="
+ + schedulerKey);
return null;
}
@@ -225,7 +235,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
RMContainer rmContainer = new RMContainerImpl(container, schedulerKey,
this.getApplicationAttemptId(), node.getNodeID(),
appSchedulingInfo.getUser(), this.rmContext,
- request.getNodeLabelExpression());
+ ps.getPrimaryRequestedNodePartition());
((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
// FIXME, should set when confirmed
@@ -694,21 +704,36 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
return false;
}
- public synchronized Map<String, Resource> getTotalPendingRequestsPerPartition() {
+ public Map<String, Resource> getTotalPendingRequestsPerPartition() {
+ try {
+ readLock.lock();
+
+ Map<String, Resource> ret = new HashMap<>();
+ for (SchedulerRequestKey schedulerKey : appSchedulingInfo
+ .getSchedulerKeys()) {
+ SchedulingPlacementSet<FiCaSchedulerNode> ps =
+ appSchedulingInfo.getSchedulingPlacementSet(schedulerKey);
+
+ String nodePartition = ps.getPrimaryRequestedNodePartition();
+ Resource res = ret.get(nodePartition);
+ if (null == res) {
+ res = Resources.createResource(0);
+ ret.put(nodePartition, res);
+ }
- Map<String, Resource> ret = new HashMap<String, Resource>();
- Resource res = null;
- for (SchedulerRequestKey key : appSchedulingInfo.getSchedulerKeys()) {
- ResourceRequest rr = appSchedulingInfo.getResourceRequest(key, "*");
- if ((res = ret.get(rr.getNodeLabelExpression())) == null) {
- res = Resources.createResource(0, 0);
- ret.put(rr.getNodeLabelExpression(), res);
+ PendingAsk ask = ps.getPendingAsk(ResourceRequest.ANY);
+ if (ask.getCount() > 0) {
+ Resources.addTo(res, Resources
+ .multiply(ask.getPerAllocationResource(),
+ ask.getCount()));
+ }
}
- Resources.addTo(res,
- Resources.multiply(rr.getCapability(), rr.getNumContainers()));
+ return ret;
+ } finally {
+ readLock.unlock();
}
- return ret;
+
}
public void markContainerForPreemption(ContainerId cont) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 0daa8a4..d7ed7d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -18,16 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import java.text.DecimalFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -56,11 +46,22 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
/**
* Represents an application attempt from the viewpoint of the Fair Scheduler.
*/
@@ -416,7 +417,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
public RMContainer allocate(NodeType type, FSSchedulerNode node,
- SchedulerRequestKey schedulerKey, ResourceRequest request,
+ SchedulerRequestKey schedulerKey, PendingAsk pendingAsk,
Container reservedContainer) {
RMContainer rmContainer;
Container container;
@@ -437,13 +438,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Required sanity check - AM can call 'allocate' to update resource
// request without locking the scheduler, hence we need to check
- if (getTotalRequiredResources(schedulerKey) <= 0) {
+ if (getOutstandingAsksCount(schedulerKey) <= 0) {
return null;
}
container = reservedContainer;
if (container == null) {
- container = createContainer(node, request.getCapability(),
+ container = createContainer(node, pendingAsk.getPerAllocationResource(),
schedulerKey);
}
@@ -459,7 +460,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Update consumption and track allocations
List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
- type, node, schedulerKey, request, container);
+ type, node, schedulerKey, container);
this.attemptResourceUsage.incUsed(container.getResource());
// Update resource requests related to "request" and store in RMContainer
@@ -632,7 +633,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* in {@link FSSchedulerNode}..
* return whether reservation was possible with the current threshold limits
*/
- private boolean reserve(ResourceRequest request, FSSchedulerNode node,
+ private boolean reserve(Resource perAllocationResource, FSSchedulerNode node,
Container reservedContainer, NodeType type,
SchedulerRequestKey schedulerKey) {
@@ -641,7 +642,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
" app_id=" + getApplicationId());
if (reservedContainer == null) {
reservedContainer =
- createContainer(node, request.getCapability(),
+ createContainer(node, perAllocationResource,
schedulerKey);
getMetrics().reserveResource(getUser(),
reservedContainer.getResource());
@@ -763,8 +764,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
*
* @param node
* The node to try placing the container on.
- * @param request
- * The ResourceRequest we're trying to satisfy.
+ * @param pendingAsk
+ * The {@link PendingAsk} we're trying to satisfy.
* @param type
* The locality of the assignment.
* @param reserved
@@ -776,11 +777,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* made, returns an empty resource.
*/
private Resource assignContainer(
- FSSchedulerNode node, ResourceRequest request, NodeType type,
+ FSSchedulerNode node, PendingAsk pendingAsk, NodeType type,
boolean reserved, SchedulerRequestKey schedulerKey) {
// How much does this request need?
- Resource capability = request.getCapability();
+ Resource capability = pendingAsk.getPerAllocationResource();
// How much does the node have?
Resource available = node.getUnallocatedResource();
@@ -794,7 +795,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
if (Resources.fitsIn(capability, available)) {
// Inform the application of the new container for this request
RMContainer allocatedContainer =
- allocate(type, node, schedulerKey, request,
+ allocate(type, node, schedulerKey, pendingAsk,
reservedContainer);
if (allocatedContainer == null) {
// Did the application need this resource?
@@ -825,8 +826,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
// The desired container won't fit here, so reserve
- if (isReservable(capability) &&
- reserve(request, node, reservedContainer, type, schedulerKey)) {
+ if (isReservable(capability) && reserve(
+ pendingAsk.getPerAllocationResource(), node, reservedContainer, type,
+ schedulerKey)) {
if (isWaitingForAMContainer()) {
updateAMDiagnosticMsg(capability,
" exceed the available resources of the node and the request is"
@@ -841,7 +843,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
if (LOG.isDebugEnabled()) {
LOG.debug("Couldn't creating reservation for " +
- getName() + ",at priority " + request.getPriority());
+ getName() + ",at priority " + schedulerKey.getPriority());
}
return Resources.none();
}
@@ -852,19 +854,16 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
getQueue().getPolicy().getResourceCalculator(), capacity);
}
- private boolean hasNodeOrRackLocalRequests(SchedulerRequestKey schedulerKey) {
- return getResourceRequests(schedulerKey).size() > 1;
- }
-
/**
* Whether the AM container for this app is over maxAMShare limit.
*/
private boolean isOverAMShareLimit() {
// Check the AM resource usage for the leaf queue
if (!isAmRunning() && !getUnmanagedAM()) {
- List<ResourceRequest> ask = appSchedulingInfo.getAllResourceRequests();
- if (ask.isEmpty() || !getQueue().canRunAppAM(
- ask.get(0).getCapability())) {
+ // Return true if we have not ask, or queue is not be able to run app's AM
+ PendingAsk ask = appSchedulingInfo.getNextPendingAsk();
+ if (ask.getCount() == 0 || !getQueue().canRunAppAM(
+ ask.getPerAllocationResource())) {
return true;
}
}
@@ -886,6 +885,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// (not scheduled) in order to promote better locality.
try {
writeLock.lock();
+
+ // TODO (wandga): All logics in this method should be added to
+ // SchedulerPlacement#canDelayTo which is independent from scheduler.
+ // Scheduler can choose to use various/pluggable delay-scheduling
+ // implementation.
for (SchedulerRequestKey schedulerKey : keysToTry) {
// Skip it for reserved container, since
// we already check it in isValidReservation.
@@ -895,14 +899,16 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
addSchedulingOpportunity(schedulerKey);
- ResourceRequest rackLocalRequest = getResourceRequest(schedulerKey,
+ PendingAsk rackLocalPendingAsk = getPendingAsk(schedulerKey,
node.getRackName());
- ResourceRequest localRequest = getResourceRequest(schedulerKey,
+ PendingAsk nodeLocalPendingAsk = getPendingAsk(schedulerKey,
node.getNodeName());
- if (localRequest != null && !localRequest.getRelaxLocality()) {
+ if (nodeLocalPendingAsk.getCount() > 0
+ && !appSchedulingInfo.canDelayTo(schedulerKey,
+ node.getNodeName())) {
LOG.warn("Relax locality off is not supported on local request: "
- + localRequest);
+ + nodeLocalPendingAsk);
}
NodeType allowedLocality;
@@ -918,23 +924,23 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
scheduler.getRackLocalityThreshold());
}
- if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0
- && localRequest != null && localRequest.getNumContainers() != 0) {
+ if (rackLocalPendingAsk.getCount() > 0
+ && nodeLocalPendingAsk.getCount() > 0) {
if (LOG.isTraceEnabled()) {
LOG.trace("Assign container on " + node.getNodeName()
+ " node, assignType: NODE_LOCAL" + ", allowedLocality: "
+ allowedLocality + ", priority: " + schedulerKey.getPriority()
+ ", app attempt id: " + this.attemptId);
}
- return assignContainer(node, localRequest, NodeType.NODE_LOCAL,
+ return assignContainer(node, nodeLocalPendingAsk, NodeType.NODE_LOCAL,
reserved, schedulerKey);
}
- if (rackLocalRequest != null && !rackLocalRequest.getRelaxLocality()) {
+ if (!appSchedulingInfo.canDelayTo(schedulerKey, node.getRackName())) {
continue;
}
- if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0
+ if (rackLocalPendingAsk.getCount() > 0
&& (allowedLocality.equals(NodeType.RACK_LOCAL) || allowedLocality
.equals(NodeType.OFF_SWITCH))) {
if (LOG.isTraceEnabled()) {
@@ -943,27 +949,26 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ allowedLocality + ", priority: " + schedulerKey.getPriority()
+ ", app attempt id: " + this.attemptId);
}
- return assignContainer(node, rackLocalRequest, NodeType.RACK_LOCAL,
+ return assignContainer(node, rackLocalPendingAsk, NodeType.RACK_LOCAL,
reserved, schedulerKey);
}
- ResourceRequest offSwitchRequest = getResourceRequest(schedulerKey,
+ PendingAsk offswitchAsk = getPendingAsk(schedulerKey,
ResourceRequest.ANY);
- if (offSwitchRequest != null && !offSwitchRequest.getRelaxLocality()) {
+ if (!appSchedulingInfo.canDelayTo(schedulerKey, ResourceRequest.ANY)) {
continue;
}
- if (offSwitchRequest != null
- && offSwitchRequest.getNumContainers() != 0) {
- if (!hasNodeOrRackLocalRequests(schedulerKey) || allowedLocality
- .equals(NodeType.OFF_SWITCH)) {
+ if (offswitchAsk.getCount() > 0) {
+ if (getSchedulingPlacementSet(schedulerKey).getUniqueLocationAsks()
+ <= 1 || allowedLocality.equals(NodeType.OFF_SWITCH)) {
if (LOG.isTraceEnabled()) {
LOG.trace("Assign container on " + node.getNodeName()
+ " node, assignType: OFF_SWITCH" + ", allowedLocality: "
+ allowedLocality + ", priority: " + schedulerKey.getPriority()
+ ", app attempt id: " + this.attemptId);
}
- return assignContainer(node, offSwitchRequest, NodeType.OFF_SWITCH,
+ return assignContainer(node, offswitchAsk, NodeType.OFF_SWITCH,
reserved, schedulerKey);
}
}
@@ -988,29 +993,35 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
*/
private boolean hasContainerForNode(SchedulerRequestKey key,
FSSchedulerNode node) {
- ResourceRequest anyRequest = getResourceRequest(key, ResourceRequest.ANY);
- ResourceRequest rackRequest = getResourceRequest(key, node.getRackName());
- ResourceRequest nodeRequest = getResourceRequest(key, node.getNodeName());
+ PendingAsk offswitchAsk = getPendingAsk(key, ResourceRequest.ANY);
+ Resource resource = offswitchAsk.getPerAllocationResource();
+ boolean hasRequestForOffswitch =
+ offswitchAsk.getCount() > 0;
+ boolean hasRequestForRack = getOutstandingAsksCount(key,
+ node.getRackName()) > 0;
+ boolean hasRequestForNode = getOutstandingAsksCount(key,
+ node.getNodeName()) > 0;
boolean ret = true;
if (!(// There must be outstanding requests at the given priority:
- anyRequest != null && anyRequest.getNumContainers() > 0 &&
- // If locality relaxation is turned off at *-level, there must be a
- // non-zero request for the node's rack:
- (anyRequest.getRelaxLocality() ||
- (rackRequest != null && rackRequest.getNumContainers() > 0)) &&
- // If locality relaxation is turned off at rack-level, there must be a
- // non-zero request at the node:
- (rackRequest == null || rackRequest.getRelaxLocality() ||
- (nodeRequest != null && nodeRequest.getNumContainers() > 0)) &&
- // The requested container must be able to fit on the node:
- Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null,
- anyRequest.getCapability(), node.getRMNode().getTotalCapability()))) {
+ hasRequestForOffswitch &&
+ // If locality relaxation is turned off at *-level, there must be a
+ // non-zero request for the node's rack:
+ (appSchedulingInfo.canDelayTo(key, ResourceRequest.ANY) ||
+ (hasRequestForRack)) &&
+ // If locality relaxation is turned off at rack-level,
+ // there must be a non-zero request at the node:
+ (!hasRequestForRack || appSchedulingInfo.canDelayTo(key,
+ node.getRackName()) || (hasRequestForNode)) &&
+ // The requested container must be able to fit on the node:
+ Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null,
+ resource,
+ node.getRMNode().getTotalCapability()))) {
ret = false;
- } else if (!getQueue().fitsInMaxShare(anyRequest.getCapability())) {
+ } else if (!getQueue().fitsInMaxShare(resource)) {
// The requested container must fit in queue maximum share
if (isWaitingForAMContainer()) {
- updateAMDiagnosticMsg(anyRequest.getCapability(),
+ updateAMDiagnosticMsg(resource,
" exceeds current queue or its parents maximum resource allowed).");
}
ret = false;
@@ -1091,10 +1102,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
return this.fairshareStarvation;
}
- ResourceRequest getNextResourceRequest() {
- return appSchedulingInfo.getNextResourceRequest();
- }
-
/**
* Helper method that captures if this app is identified to be starved.
* @return true if the app is starved for fairshare, false otherwise
@@ -1174,10 +1181,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
try {
writeLock.lock();
for (SchedulerRequestKey k : getSchedulerKeys()) {
- ResourceRequest r = getResourceRequest(k, ResourceRequest.ANY);
- if (r != null) {
- Resources.multiplyAndAddTo(demand, r.getCapability(),
- r.getNumContainers());
+ PendingAsk pendingAsk = getPendingAsk(k, ResourceRequest.ANY);
+ if (pendingAsk.getCount() > 0) {
+ Resources.multiplyAndAddTo(demand,
+ pendingAsk.getPerAllocationResource(),
+ pendingAsk.getCount());
}
}
} finally {
@@ -1189,9 +1197,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
public Resource assignContainer(FSSchedulerNode node) {
if (isOverAMShareLimit()) {
if (isWaitingForAMContainer()) {
- List<ResourceRequest> ask = appSchedulingInfo.getAllResourceRequests();
- updateAMDiagnosticMsg(ask.get(0).getCapability(), " exceeds maximum "
- + "AM resource allowed).");
+ PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk();
+ updateAMDiagnosticMsg(amAsk.getPerAllocationResource(),
+ " exceeds maximum AM resource allowed).");
}
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index 3579857..f432484 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.ArrayList;
@@ -90,14 +92,17 @@ class FSPreemptionThread extends Thread {
List<RMContainer> containers = new ArrayList<>(); // return value
// Find the nodes that match the next resource request
- ResourceRequest request = starvedApp.getNextResourceRequest();
+ SchedulingPlacementSet nextPs =
+ starvedApp.getAppSchedulingInfo().getFirstSchedulingPlacementSet();
+ PendingAsk firstPendingAsk = nextPs.getPendingAsk(ResourceRequest.ANY);
// TODO (KK): Should we check other resource requests if we can't match
// the first one?
- Resource requestCapability = request.getCapability();
+ Resource requestCapability = firstPendingAsk.getPerAllocationResource();
+
List<FSSchedulerNode> potentialNodes =
scheduler.getNodeTracker().getNodesByResourceName(
- request.getResourceName());
+ nextPs.getAcceptedResouceNames().next().toString());
// From the potential nodes, pick a node that has enough containers
// from apps over their fairshare
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
index fa61710..d932e0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
@@ -51,8 +51,7 @@ public class FifoAppAttempt extends FiCaSchedulerApp {
}
public RMContainer allocate(NodeType type, FiCaSchedulerNode node,
- SchedulerRequestKey schedulerKey, ResourceRequest request,
- Container container) {
+ SchedulerRequestKey schedulerKey, Container container) {
try {
writeLock.lock();
@@ -62,15 +61,14 @@ public class FifoAppAttempt extends FiCaSchedulerApp {
// Required sanity check - AM can call 'allocate' to update resource
// request without locking the scheduler, hence we need to check
- if (getTotalRequiredResources(schedulerKey) <= 0) {
+ if (getOutstandingAsksCount(schedulerKey) <= 0) {
return null;
}
// Create RMContainer
RMContainer rmContainer = new RMContainerImpl(container,
schedulerKey, this.getApplicationAttemptId(), node.getNodeID(),
- appSchedulingInfo.getUser(), this.rmContext,
- request.getNodeLabelExpression());
+ appSchedulingInfo.getUser(), this.rmContext, node.getPartition());
((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
updateAMContainerDiagnostics(AMState.ASSIGNED, null);
@@ -83,7 +81,7 @@ public class FifoAppAttempt extends FiCaSchedulerApp {
// Update consumption and track allocations
List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
- type, node, schedulerKey, request, container);
+ type, node, schedulerKey, container);
attemptResourceUsage.incUsed(node.getPartition(),
container.getResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index f4ab9c8..d52f538 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -18,16 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentSkipListMap;
-
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
@@ -90,6 +81,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemoved
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.utils.Lock;
@@ -97,7 +89,15 @@ import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
@LimitedPrivate("yarn")
@Evolving
@@ -545,35 +545,32 @@ public class FifoScheduler extends
private int getMaxAllocatableContainers(FifoAppAttempt application,
SchedulerRequestKey schedulerKey, FiCaSchedulerNode node, NodeType type) {
- int maxContainers = 0;
-
- ResourceRequest offSwitchRequest =
- application.getResourceRequest(schedulerKey, ResourceRequest.ANY);
- if (offSwitchRequest != null) {
- maxContainers = offSwitchRequest.getNumContainers();
- }
+ PendingAsk offswitchAsk = application.getPendingAsk(schedulerKey,
+ ResourceRequest.ANY);
+ int maxContainers = offswitchAsk.getCount();
if (type == NodeType.OFF_SWITCH) {
return maxContainers;
}
if (type == NodeType.RACK_LOCAL) {
- ResourceRequest rackLocalRequest =
- application.getResourceRequest(schedulerKey, node.getRMNode()
- .getRackName());
- if (rackLocalRequest == null) {
+ PendingAsk rackLocalAsk = application.getPendingAsk(schedulerKey,
+ node.getRackName());
+ if (rackLocalAsk.getCount() <= 0) {
return maxContainers;
}
- maxContainers = Math.min(maxContainers, rackLocalRequest.getNumContainers());
+ maxContainers = Math.min(maxContainers,
+ rackLocalAsk.getCount());
}
if (type == NodeType.NODE_LOCAL) {
- ResourceRequest nodeLocalRequest =
- application.getResourceRequest(schedulerKey, node.getRMNode()
- .getNodeAddress());
- if (nodeLocalRequest != null) {
- maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers());
+ PendingAsk nodeLocalAsk = application.getPendingAsk(schedulerKey,
+ node.getRMNode().getHostName());
+
+ if (nodeLocalAsk.getCount() > 0) {
+ maxContainers = Math.min(maxContainers,
+ nodeLocalAsk.getCount());
}
}
@@ -611,25 +608,21 @@ public class FifoScheduler extends
private int assignNodeLocalContainers(FiCaSchedulerNode node,
FifoAppAttempt application, SchedulerRequestKey schedulerKey) {
int assignedContainers = 0;
- ResourceRequest request =
- application.getResourceRequest(schedulerKey, node.getNodeName());
- if (request != null) {
+ PendingAsk nodeLocalAsk = application.getPendingAsk(schedulerKey,
+ node.getNodeName());
+ if (nodeLocalAsk.getCount() > 0) {
// Don't allocate on this node if we don't need containers on this rack
- ResourceRequest rackRequest =
- application.getResourceRequest(schedulerKey,
- node.getRMNode().getRackName());
- if (rackRequest == null || rackRequest.getNumContainers() <= 0) {
+ if (application.getOutstandingAsksCount(schedulerKey,
+ node.getRackName()) <= 0) {
return 0;
}
-
- int assignableContainers =
- Math.min(
- getMaxAllocatableContainers(application, schedulerKey, node,
- NodeType.NODE_LOCAL),
- request.getNumContainers());
+
+ int assignableContainers = Math.min(
+ getMaxAllocatableContainers(application, schedulerKey, node,
+ NodeType.NODE_LOCAL), nodeLocalAsk.getCount());
assignedContainers =
- assignContainer(node, application, schedulerKey,
- assignableContainers, request, NodeType.NODE_LOCAL);
+ assignContainer(node, application, schedulerKey, assignableContainers,
+ nodeLocalAsk.getPerAllocationResource(), NodeType.NODE_LOCAL);
}
return assignedContainers;
}
@@ -637,25 +630,21 @@ public class FifoScheduler extends
private int assignRackLocalContainers(FiCaSchedulerNode node,
FifoAppAttempt application, SchedulerRequestKey schedulerKey) {
int assignedContainers = 0;
- ResourceRequest request =
- application.getResourceRequest(schedulerKey, node.getRMNode()
- .getRackName());
- if (request != null) {
+ PendingAsk rackAsk = application.getPendingAsk(schedulerKey,
+ node.getRMNode().getRackName());
+ if (rackAsk.getCount() > 0) {
// Don't allocate on this rack if the application doens't need containers
- ResourceRequest offSwitchRequest =
- application.getResourceRequest(schedulerKey, ResourceRequest.ANY);
- if (offSwitchRequest.getNumContainers() <= 0) {
+ if (application.getOutstandingAsksCount(schedulerKey,
+ ResourceRequest.ANY) <= 0) {
return 0;
}
-
- int assignableContainers =
- Math.min(
- getMaxAllocatableContainers(application, schedulerKey, node,
- NodeType.RACK_LOCAL),
- request.getNumContainers());
+
+ int assignableContainers =
+ Math.min(getMaxAllocatableContainers(application, schedulerKey, node,
+ NodeType.RACK_LOCAL), rackAsk.getCount());
assignedContainers =
- assignContainer(node, application, schedulerKey,
- assignableContainers, request, NodeType.RACK_LOCAL);
+ assignContainer(node, application, schedulerKey, assignableContainers,
+ rackAsk.getPerAllocationResource(), NodeType.RACK_LOCAL);
}
return assignedContainers;
}
@@ -663,26 +652,26 @@ public class FifoScheduler extends
private int assignOffSwitchContainers(FiCaSchedulerNode node,
FifoAppAttempt application, SchedulerRequestKey schedulerKey) {
int assignedContainers = 0;
- ResourceRequest request =
- application.getResourceRequest(schedulerKey, ResourceRequest.ANY);
- if (request != null) {
+ PendingAsk offswitchAsk = application.getPendingAsk(schedulerKey,
+ ResourceRequest.ANY);
+ if (offswitchAsk.getCount() > 0) {
assignedContainers =
assignContainer(node, application, schedulerKey,
- request.getNumContainers(), request, NodeType.OFF_SWITCH);
+ offswitchAsk.getCount(),
+ offswitchAsk.getPerAllocationResource(), NodeType.OFF_SWITCH);
}
return assignedContainers;
}
private int assignContainer(FiCaSchedulerNode node, FifoAppAttempt application,
SchedulerRequestKey schedulerKey, int assignableContainers,
- ResourceRequest request, NodeType type) {
+ Resource capability, NodeType type) {
LOG.debug("assignContainers:" +
" node=" + node.getRMNode().getNodeAddress() +
" application=" + application.getApplicationId().getId() +
" priority=" + schedulerKey.getPriority().getPriority() +
" assignableContainers=" + assignableContainers +
- " request=" + request + " type=" + type);
- Resource capability = request.getCapability();
+ " capability=" + capability + " type=" + type);
// TODO: A buggy application with this zero would crash the scheduler.
int availableContainers =
@@ -708,7 +697,7 @@ public class FifoScheduler extends
// Inform the application
RMContainer rmContainer = application.allocate(type, node, schedulerKey,
- request, container);
+ container);
// Inform the node
node.allocateContainer(rmContainer);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
index 157518e..c32246d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
@@ -19,12 +19,16 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
import org.apache.commons.collections.IteratorUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import java.util.ArrayList;
@@ -37,9 +41,14 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
implements SchedulingPlacementSet<N> {
+ private static final Log LOG =
+ LogFactory.getLog(LocalitySchedulingPlacementSet.class);
+
private final Map<String, ResourceRequest> resourceRequestMap =
new ConcurrentHashMap<>();
private AppSchedulingInfo appSchedulingInfo;
+ private volatile String primaryRequestedPartition =
+ RMNodeLabelsManager.NO_LABEL;
private final ReentrantReadWriteLock.ReadLock readLock;
private final ReentrantReadWriteLock.WriteLock writeLock;
@@ -132,11 +141,14 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
resourceRequestMap.put(resourceName, request);
if (resourceName.equals(ResourceRequest.ANY)) {
+ String partition = request.getNodeLabelExpression() == null ?
+ RMNodeLabelsManager.NO_LABEL :
+ request.getNodeLabelExpression();
+
+ this.primaryRequestedPartition = partition;
+
//update the applications requested labels set
- appSchedulingInfo.addRequestedPartition(
- request.getNodeLabelExpression() == null ?
- RMNodeLabelsManager.NO_LABEL :
- request.getNodeLabelExpression());
+ appSchedulingInfo.addRequestedPartition(partition);
updateResult = new ResourceRequestUpdateResult(lastRequest, request);
}
@@ -152,11 +164,43 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
return resourceRequestMap;
}
- @Override
- public ResourceRequest getResourceRequest(String resourceName) {
+ private ResourceRequest getResourceRequest(String resourceName) {
return resourceRequestMap.get(resourceName);
}
+ @Override
+ public PendingAsk getPendingAsk(String resourceName) {
+ try {
+ readLock.lock();
+ ResourceRequest request = getResourceRequest(resourceName);
+ if (null == request) {
+ return PendingAsk.ZERO;
+ } else{
+ return new PendingAsk(request.getCapability(),
+ request.getNumContainers());
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ }
+
+ @Override
+ public int getOutstandingAsksCount(String resourceName) {
+ try {
+ readLock.lock();
+ ResourceRequest request = getResourceRequest(resourceName);
+ if (null == request) {
+ return 0;
+ } else{
+ return request.getNumContainers();
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ }
+
private void decrementOutstanding(SchedulerRequestKey schedulerRequestKey,
ResourceRequest offSwitchRequest) {
int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;
@@ -282,21 +326,66 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
}
@Override
+ public boolean canDelayTo(String resourceName) {
+ try {
+ readLock.lock();
+ ResourceRequest request = getResourceRequest(resourceName);
+ return request == null || request.getRelaxLocality();
+ } finally {
+ readLock.unlock();
+ }
+
+ }
+
+ @Override
+ public boolean acceptNodePartition(String nodePartition,
+ SchedulingMode schedulingMode) {
+ // We will only look at node label = nodeLabelToLookAt according to
+ // schedulingMode and partition of node.
+ String nodePartitionToLookAt;
+ if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY) {
+ nodePartitionToLookAt = nodePartition;
+ } else {
+ nodePartitionToLookAt = RMNodeLabelsManager.NO_LABEL;
+ }
+
+ return primaryRequestedPartition.equals(nodePartitionToLookAt);
+ }
+
+ @Override
+ public String getPrimaryRequestedNodePartition() {
+ return primaryRequestedPartition;
+ }
+
+ @Override
+ public int getUniqueLocationAsks() {
+ return resourceRequestMap.size();
+ }
+
+ @Override
+ public void showRequests() {
+ for (ResourceRequest request : resourceRequestMap.values()) {
+ if (request.getNumContainers() > 0) {
+ LOG.debug("\tRequest=" + request);
+ }
+ }
+ }
+
+ @Override
public List<ResourceRequest> allocate(SchedulerRequestKey schedulerKey,
- NodeType type, SchedulerNode node, ResourceRequest request) {
+ NodeType type, SchedulerNode node) {
try {
writeLock.lock();
List<ResourceRequest> resourceRequests = new ArrayList<>();
- if (null == request) {
- if (type == NodeType.NODE_LOCAL) {
- request = resourceRequestMap.get(node.getNodeName());
- } else if (type == NodeType.RACK_LOCAL) {
- request = resourceRequestMap.get(node.getRackName());
- } else{
- request = resourceRequestMap.get(ResourceRequest.ANY);
- }
+ ResourceRequest request;
+ if (type == NodeType.NODE_LOCAL) {
+ request = resourceRequestMap.get(node.getNodeName());
+ } else if (type == NodeType.RACK_LOCAL) {
+ request = resourceRequestMap.get(node.getRackName());
+ } else{
+ request = resourceRequestMap.get(ResourceRequest.ANY);
}
if (type == NodeType.NODE_LOCAL) {
@@ -312,4 +401,14 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
writeLock.unlock();
}
}
+
+ @Override
+ public Iterator<String> getAcceptedResouceNames() {
+ try {
+ readLock.lock();
+ return resourceRequestMap.keySet().iterator();
+ } finally {
+ readLock.unlock();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HADOOP-13597. Switch KMS from
Tomcat to Jetty. Contributed by John Zhuge.
Posted by st...@apache.org.
HADOOP-13597. Switch KMS from Tomcat to Jetty. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d182949
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d182949
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d182949
Branch: refs/heads/HADOOP-13345
Commit: 5d182949badb2eb80393de7ba3838102d006488b
Parents: 8850c05
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Jan 5 17:21:57 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Jan 5 17:21:57 2017 -0800
----------------------------------------------------------------------
.../resources/assemblies/hadoop-kms-dist.xml | 25 +-
.../server/AuthenticationFilter.java | 6 +-
.../src/main/bin/hadoop-functions.sh | 51 ++--
.../hadoop/conf/ConfigurationWithLogging.java | 113 +++++++++
.../org/apache/hadoop/http/HttpServer2.java | 163 ++++++++----
.../apache/hadoop/security/ssl/SSLFactory.java | 64 +++--
.../src/site/markdown/CommandsManual.md | 8 +-
.../src/test/scripts/hadoop_mkdir.bats | 42 ++++
.../src/test/scripts/hadoop_using_envvar.bats | 33 +++
.../dev-support/findbugsExcludeFile.xml | 2 +-
hadoop-common-project/hadoop-kms/pom.xml | 160 +-----------
.../hadoop-kms/src/main/conf/kms-env.sh | 48 ++--
.../src/main/conf/kms-log4j.properties | 3 +-
.../hadoop-kms/src/main/conf/kms-site.xml | 167 +------------
.../crypto/key/kms/server/KMSConfiguration.java | 16 ++
.../crypto/key/kms/server/KMSJMXServlet.java | 36 ---
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 23 +-
.../crypto/key/kms/server/KMSWebServer.java | 155 ++++++++++++
.../hadoop-kms/src/main/libexec/kms-config.sh | 72 ------
.../main/libexec/shellprofile.d/hadoop-kms.sh | 57 +++++
.../src/main/resources/kms-default.xml | 248 +++++++++++++++++++
.../main/resources/webapps/kms/WEB-INF/web.xml | 68 +++++
.../main/resources/webapps/static/index.html | 35 +++
.../hadoop-kms/src/main/sbin/kms.sh | 116 +++------
.../src/main/tomcat/ROOT/WEB-INF/web.xml | 16 --
.../hadoop-kms/src/main/tomcat/ROOT/index.html | 27 --
.../src/main/tomcat/logging.properties | 67 -----
.../hadoop-kms/src/main/tomcat/server.xml | 155 ------------
.../src/main/tomcat/ssl-server.xml.conf | 136 ----------
.../hadoop-kms/src/main/webapp/WEB-INF/web.xml | 78 ------
.../hadoop-kms/src/site/configuration.xsl | 49 ++++
.../hadoop-kms/src/site/markdown/index.md.vm | 122 ++++++---
.../hadoop/crypto/key/kms/server/MiniKMS.java | 118 ++-------
33 files changed, 1235 insertions(+), 1244 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml
index 5830bba..ff6f990 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml
@@ -21,6 +21,14 @@
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
+ <!-- Jar file -->
+ <fileSet>
+ <directory>target</directory>
+ <outputDirectory>/share/hadoop/common</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
<!-- Configuration files -->
<fileSet>
<directory>${basedir}/src/main/conf</directory>
@@ -41,7 +49,7 @@
<directory>${basedir}/src/main/libexec</directory>
<outputDirectory>/libexec</outputDirectory>
<includes>
- <include>*</include>
+ <include>**/*</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
@@ -51,4 +59,19 @@
<outputDirectory>/share/doc/hadoop/kms</outputDirectory>
</fileSet>
</fileSets>
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>/share/hadoop/common/lib</outputDirectory>
+ <!-- Exclude hadoop artifacts. They will be found via HADOOP* env -->
+ <excludes>
+ <exclude>org.apache.hadoop:hadoop-common</exclude>
+ <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
+ <!-- use slf4j from common to avoid multiple binding warnings -->
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>
+ <exclude>org.hsqldb:hsqldb</exclude>
+ </excludes>
+ </dependencySet>
+ </dependencySets>
</assembly>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 264d991..b10fc84 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -142,7 +142,7 @@ public class AuthenticationFilter implements Filter {
private String cookieDomain;
private String cookiePath;
private boolean isCookiePersistent;
- private boolean isInitializedByTomcat;
+ private boolean destroySecretProvider;
/**
* <p>Initializes the authentication filter and signer secret provider.</p>
@@ -209,7 +209,7 @@ public class AuthenticationFilter implements Filter {
secretProvider = constructSecretProvider(
filterConfig.getServletContext(),
config, false);
- isInitializedByTomcat = true;
+ destroySecretProvider = true;
} catch (Exception ex) {
throw new ServletException(ex);
}
@@ -356,7 +356,7 @@ public class AuthenticationFilter implements Filter {
authHandler.destroy();
authHandler = null;
}
- if (secretProvider != null && isInitializedByTomcat) {
+ if (secretProvider != null && destroySecretProvider) {
secretProvider.destroy();
secretProvider = null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index b6e2b59..3151023 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -262,6 +262,39 @@ function hadoop_deprecate_envvar
fi
}
+## @description Declare `var` being used and print its value.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param var
+function hadoop_using_envvar
+{
+ local var=$1
+ local val=${!var}
+
+ if [[ -n "${val}" ]]; then
+ hadoop_debug "${var} = ${val}"
+ fi
+}
+
+## @description Create the directory 'dir'.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param dir
+function hadoop_mkdir
+{
+ local dir=$1
+
+ if [[ ! -w "${dir}" ]] && [[ ! -d "${dir}" ]]; then
+ hadoop_error "WARNING: ${dir} does not exist. Creating."
+ if ! mkdir -p "${dir}"; then
+ hadoop_error "ERROR: Unable to create ${dir}. Aborting."
+ exit 1
+ fi
+ fi
+}
+
## @description Bootstraps the Hadoop shell environment
## @audience private
## @stability evolving
@@ -1396,14 +1429,7 @@ function hadoop_verify_piddir
hadoop_error "No pid directory defined."
exit 1
fi
- if [[ ! -w "${HADOOP_PID_DIR}" ]] && [[ ! -d "${HADOOP_PID_DIR}" ]]; then
- hadoop_error "WARNING: ${HADOOP_PID_DIR} does not exist. Creating."
- mkdir -p "${HADOOP_PID_DIR}" > /dev/null 2>&1
- if [[ $? -gt 0 ]]; then
- hadoop_error "ERROR: Unable to create ${HADOOP_PID_DIR}. Aborting."
- exit 1
- fi
- fi
+ hadoop_mkdir "${HADOOP_PID_DIR}"
touch "${HADOOP_PID_DIR}/$$" >/dev/null 2>&1
if [[ $? -gt 0 ]]; then
hadoop_error "ERROR: Unable to write in ${HADOOP_PID_DIR}. Aborting."
@@ -1421,14 +1447,7 @@ function hadoop_verify_logdir
hadoop_error "No log directory defined."
exit 1
fi
- if [[ ! -w "${HADOOP_LOG_DIR}" ]] && [[ ! -d "${HADOOP_LOG_DIR}" ]]; then
- hadoop_error "WARNING: ${HADOOP_LOG_DIR} does not exist. Creating."
- mkdir -p "${HADOOP_LOG_DIR}" > /dev/null 2>&1
- if [[ $? -gt 0 ]]; then
- hadoop_error "ERROR: Unable to create ${HADOOP_LOG_DIR}. Aborting."
- exit 1
- fi
- fi
+ hadoop_mkdir "${HADOOP_LOG_DIR}"
touch "${HADOOP_LOG_DIR}/$$" >/dev/null 2>&1
if [[ $? -gt 0 ]]; then
hadoop_error "ERROR: Unable to write in ${HADOOP_LOG_DIR}. Aborting."
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
new file mode 100644
index 0000000..8a5e054
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.conf;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Logs access to {@link Configuration}.
+ * Sensitive data will be redacted.
+ */
+@InterfaceAudience.Private
+public class ConfigurationWithLogging extends Configuration {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ConfigurationWithLogging.class);
+
+ private final Logger log;
+ private final ConfigRedactor redactor;
+
+ public ConfigurationWithLogging(Configuration conf) {
+ super(conf);
+ log = LOG;
+ redactor = new ConfigRedactor(conf);
+ }
+
+ /**
+ * @see Configuration#get(String).
+ */
+ @Override
+ public String get(String name) {
+ String value = super.get(name);
+ log.info("Got {} = '{}'", name, redactor.redact(name, value));
+ return value;
+ }
+
+ /**
+ * @see Configuration#get(String, String).
+ */
+ @Override
+ public String get(String name, String defaultValue) {
+ String value = super.get(name, defaultValue);
+ log.info("Got {} = '{}' (default '{}')", name,
+ redactor.redact(name, value), redactor.redact(name, defaultValue));
+ return value;
+ }
+
+ /**
+ * @see Configuration#getBoolean(String, boolean).
+ */
+ @Override
+ public boolean getBoolean(String name, boolean defaultValue) {
+ boolean value = super.getBoolean(name, defaultValue);
+ log.info("Got {} = '{}' (default '{}')", name, value, defaultValue);
+ return value;
+ }
+
+ /**
+ * @see Configuration#getFloat(String, float).
+ */
+ @Override
+ public float getFloat(String name, float defaultValue) {
+ float value = super.getFloat(name, defaultValue);
+ log.info("Got {} = '{}' (default '{}')", name, value, defaultValue);
+ return value;
+ }
+
+ /**
+ * @see Configuration#getInt(String, int).
+ */
+ @Override
+ public int getInt(String name, int defaultValue) {
+ int value = super.getInt(name, defaultValue);
+ log.info("Got {} = '{}' (default '{}')", name, value, defaultValue);
+ return value;
+ }
+
+ /**
+ * @see Configuration#getLong(String, long).
+ */
+ @Override
+ public long getLong(String name, long defaultValue) {
+ long value = super.getLong(name, defaultValue);
+ log.info("Got {} = '{}' (default '{}')", name, value, defaultValue);
+ return value;
+ }
+
+ /**
+ * @see Configuration#set(String, String, String).
+ */
+ @Override
+ public void set(String name, String value, String source) {
+ log.info("Set {} to '{}'{}", name, redactor.redact(name, value),
+ source == null ? "" : " from " + source);
+ super.set(name, value, source);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index b930f75..6e21592 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.http;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
+
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
@@ -45,7 +49,10 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.sun.jersey.spi.container.servlet.ServletContainer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -54,14 +61,15 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.ConfServlet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.security.AuthenticationFilterInitializer;
-import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import org.apache.hadoop.jmx.JMXJsonServlet;
import org.apache.hadoop.log.LogLevel;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Shell;
import org.eclipse.jetty.http.HttpVersion;
@@ -90,16 +98,9 @@ import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.servlet.ServletMapping;
import org.eclipse.jetty.util.ArrayUtil;
import org.eclipse.jetty.util.MultiException;
-import org.eclipse.jetty.webapp.WebAppContext;
-import org.eclipse.jetty.util.thread.QueuedThreadPool;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.sun.jersey.spi.container.servlet.ServletContainer;
import org.eclipse.jetty.util.ssl.SslContextFactory;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.eclipse.jetty.webapp.WebAppContext;
/**
* Create a Jetty embedded server to answer http requests. The primary goal is
@@ -116,9 +117,22 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_US
public final class HttpServer2 implements FilterContainer {
public static final Log LOG = LogFactory.getLog(HttpServer2.class);
+ public static final String HTTP_SCHEME = "http";
+ public static final String HTTPS_SCHEME = "https";
+
+ public static final String HTTP_MAX_REQUEST_HEADER_SIZE_KEY =
+ "hadoop.http.max.request.header.size";
+ public static final int HTTP_MAX_REQUEST_HEADER_SIZE_DEFAULT = 65536;
+ public static final String HTTP_MAX_RESPONSE_HEADER_SIZE_KEY =
+ "hadoop.http.max.response.header.size";
+ public static final int HTTP_MAX_RESPONSE_HEADER_SIZE_DEFAULT = 65536;
+ public static final String HTTP_MAX_THREADS_KEY = "hadoop.http.max.threads";
+ public static final String HTTP_TEMP_DIR_KEY = "hadoop.http.temp.dir";
+
static final String FILTER_INITIALIZER_PROPERTY
= "hadoop.http.filter.initializers";
- public static final String HTTP_MAX_THREADS = "hadoop.http.max.threads";
+ @Deprecated
+ public static final String HTTP_MAX_THREADS = HTTP_MAX_THREADS_KEY;
// The ServletContext attribute where the daemon Configuration
// gets stored.
@@ -158,6 +172,7 @@ public final class HttpServer2 implements FilterContainer {
private ArrayList<URI> endpoints = Lists.newArrayList();
private String name;
private Configuration conf;
+ private Configuration sslConf;
private String[] pathSpecs;
private AccessControlList adminsAcl;
private boolean securityEnabled = false;
@@ -253,6 +268,15 @@ public final class HttpServer2 implements FilterContainer {
return this;
}
+ /**
+ * Specify the SSL configuration to load. This API provides an alternative
+ * to keyStore/keyPassword/trustStore.
+ */
+ public Builder setSSLConf(Configuration sslCnf) {
+ this.sslConf = sslCnf;
+ return this;
+ }
+
public Builder setPathSpec(String[] pathSpec) {
this.pathSpecs = pathSpec;
return this;
@@ -315,7 +339,45 @@ public final class HttpServer2 implements FilterContainer {
return this;
}
+ /**
+ * A wrapper of {@link Configuration#getPassword(String)}. It returns
+ * <code>String</code> instead of <code>char[]</code> and throws
+ * {@link IOException} when the password not found.
+ *
+ * @param conf the configuration
+ * @param name the property name
+ * @return the password string
+ */
+ private static String getPassword(Configuration conf, String name)
+ throws IOException {
+ char[] passchars = conf.getPassword(name);
+ if (passchars == null) {
+ throw new IOException("Password " + name + " not found");
+ }
+ return new String(passchars);
+ }
+ /**
+ * Load SSL properties from the SSL configuration.
+ */
+ private void loadSSLConfiguration() throws IOException {
+ if (sslConf == null) {
+ return;
+ }
+ needsClientAuth(sslConf.getBoolean(
+ SSLFactory.SSL_SERVER_NEED_CLIENT_AUTH,
+ SSLFactory.SSL_SERVER_NEED_CLIENT_AUTH_DEFAULT));
+ keyStore(sslConf.get(SSLFactory.SSL_SERVER_KEYSTORE_LOCATION),
+ getPassword(sslConf, SSLFactory.SSL_SERVER_KEYSTORE_PASSWORD),
+ sslConf.get(SSLFactory.SSL_SERVER_KEYSTORE_TYPE,
+ SSLFactory.SSL_SERVER_KEYSTORE_TYPE_DEFAULT));
+ keyPassword(getPassword(sslConf,
+ SSLFactory.SSL_SERVER_KEYSTORE_KEYPASSWORD));
+ trustStore(sslConf.get(SSLFactory.SSL_SERVER_TRUSTSTORE_LOCATION),
+ getPassword(sslConf, SSLFactory.SSL_SERVER_TRUSTSTORE_PASSWORD),
+ sslConf.get(SSLFactory.SSL_SERVER_TRUSTSTORE_TYPE,
+ SSLFactory.SSL_SERVER_TRUSTSTORE_TYPE_DEFAULT));
+ }
public HttpServer2 build() throws IOException {
Preconditions.checkNotNull(name, "name is not set");
@@ -336,14 +398,32 @@ public final class HttpServer2 implements FilterContainer {
}
for (URI ep : endpoints) {
+ if (HTTPS_SCHEME.equals(ep.getScheme())) {
+ loadSSLConfiguration();
+ break;
+ }
+ }
+
+ int requestHeaderSize = conf.getInt(
+ HTTP_MAX_REQUEST_HEADER_SIZE_KEY,
+ HTTP_MAX_REQUEST_HEADER_SIZE_DEFAULT);
+ int responseHeaderSize = conf.getInt(
+ HTTP_MAX_RESPONSE_HEADER_SIZE_KEY,
+ HTTP_MAX_RESPONSE_HEADER_SIZE_DEFAULT);
+
+ HttpConfiguration httpConfig = new HttpConfiguration();
+ httpConfig.setRequestHeaderSize(requestHeaderSize);
+ httpConfig.setResponseHeaderSize(responseHeaderSize);
+
+ for (URI ep : endpoints) {
final ServerConnector connector;
String scheme = ep.getScheme();
- if ("http".equals(scheme)) {
- connector =
- HttpServer2.createDefaultChannelConnector(server.webServer);
- } else if ("https".equals(scheme)) {
- connector = createHttpsChannelConnector(server.webServer);
-
+ if (HTTP_SCHEME.equals(scheme)) {
+ connector = createHttpChannelConnector(server.webServer,
+ httpConfig);
+ } else if (HTTPS_SCHEME.equals(scheme)) {
+ connector = createHttpsChannelConnector(server.webServer,
+ httpConfig);
} else {
throw new HadoopIllegalArgumentException(
"unknown scheme for endpoint:" + ep);
@@ -356,16 +436,20 @@ public final class HttpServer2 implements FilterContainer {
return server;
}
- private ServerConnector createHttpsChannelConnector(Server server) {
+ private ServerConnector createHttpChannelConnector(
+ Server server, HttpConfiguration httpConfig) {
ServerConnector conn = new ServerConnector(server);
- HttpConfiguration httpConfig = new HttpConfiguration();
- httpConfig.setRequestHeaderSize(JettyUtils.HEADER_SIZE);
- httpConfig.setResponseHeaderSize(JettyUtils.HEADER_SIZE);
- httpConfig.setSecureScheme("https");
- httpConfig.addCustomizer(new SecureRequestCustomizer());
ConnectionFactory connFactory = new HttpConnectionFactory(httpConfig);
conn.addConnectionFactory(connFactory);
configureChannelConnector(conn);
+ return conn;
+ }
+
+ private ServerConnector createHttpsChannelConnector(
+ Server server, HttpConfiguration httpConfig) {
+ httpConfig.setSecureScheme(HTTPS_SCHEME);
+ httpConfig.addCustomizer(new SecureRequestCustomizer());
+ ServerConnector conn = createHttpChannelConnector(server, httpConfig);
SslContextFactory sslContextFactory = new SslContextFactory();
sslContextFactory.setNeedClientAuth(needsClientAuth);
@@ -397,7 +481,7 @@ public final class HttpServer2 implements FilterContainer {
this.webServer = new Server();
this.adminsAcl = b.adminsAcl;
this.handlers = new HandlerCollection();
- this.webAppContext = createWebAppContext(b.name, b.conf, adminsAcl, appDir);
+ this.webAppContext = createWebAppContext(b, adminsAcl, appDir);
this.xFrameOptionIsEnabled = b.xFrameEnabled;
this.xFrameOption = b.xFrameOption;
@@ -482,8 +566,8 @@ public final class HttpServer2 implements FilterContainer {
listeners.add(connector);
}
- private static WebAppContext createWebAppContext(String name,
- Configuration conf, AccessControlList adminsAcl, final String appDir) {
+ private static WebAppContext createWebAppContext(Builder b,
+ AccessControlList adminsAcl, final String appDir) {
WebAppContext ctx = new WebAppContext();
ctx.setDefaultsDescriptor(null);
ServletHolder holder = new ServletHolder(new DefaultServlet());
@@ -496,10 +580,15 @@ public final class HttpServer2 implements FilterContainer {
holder.setInitParameters(params);
ctx.setWelcomeFiles(new String[] {"index.html"});
ctx.addServlet(holder, "/");
- ctx.setDisplayName(name);
+ ctx.setDisplayName(b.name);
ctx.setContextPath("/");
- ctx.setWar(appDir + "/" + name);
- ctx.getServletContext().setAttribute(CONF_CONTEXT_ATTRIBUTE, conf);
+ ctx.setWar(appDir + "/" + b.name);
+ String tempDirectory = b.conf.get(HTTP_TEMP_DIR_KEY);
+ if (tempDirectory != null && !tempDirectory.isEmpty()) {
+ ctx.setTempDirectory(new File(tempDirectory));
+ ctx.setAttribute("javax.servlet.context.tempdir", tempDirectory);
+ }
+ ctx.getServletContext().setAttribute(CONF_CONTEXT_ATTRIBUTE, b.conf);
ctx.getServletContext().setAttribute(ADMINS_ACL, adminsAcl);
addNoCacheFilter(ctx);
return ctx;
@@ -541,18 +630,6 @@ public final class HttpServer2 implements FilterContainer {
}
}
- @InterfaceAudience.Private
- public static ServerConnector createDefaultChannelConnector(Server server) {
- ServerConnector conn = new ServerConnector(server);
- HttpConfiguration httpConfig = new HttpConfiguration();
- httpConfig.setRequestHeaderSize(JettyUtils.HEADER_SIZE);
- httpConfig.setResponseHeaderSize(JettyUtils.HEADER_SIZE);
- ConnectionFactory connFactory = new HttpConnectionFactory(httpConfig);
- conn.addConnectionFactory(connFactory);
- configureChannelConnector(conn);
- return conn;
- }
-
/** Get an array of FilterConfiguration specified in the conf */
private static FilterInitializer[] getFilterInitializers(Configuration conf) {
if (conf == null) {
@@ -1056,7 +1133,7 @@ public final class HttpServer2 implements FilterContainer {
}
try {
- // explicitly destroy the secrete provider
+ // explicitly destroy the secret provider
secretProvider.destroy();
// clear & stop webAppContext attributes to avoid memory leaks.
webAppContext.clearAttributes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
index 95cba80..cda26a5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
@@ -60,27 +60,61 @@ public class SSLFactory implements ConnectionConfigurator {
@InterfaceAudience.Private
public static enum Mode { CLIENT, SERVER }
+ public static final String SSL_CLIENT_CONF_KEY = "hadoop.ssl.client.conf";
+ public static final String SSL_CLIENT_CONF_DEFAULT = "ssl-client.xml";
+ public static final String SSL_SERVER_CONF_KEY = "hadoop.ssl.server.conf";
+ public static final String SSL_SERVER_CONF_DEFAULT = "ssl-server.xml";
+
public static final String SSL_REQUIRE_CLIENT_CERT_KEY =
- "hadoop.ssl.require.client.cert";
+ "hadoop.ssl.require.client.cert";
+ public static final boolean SSL_REQUIRE_CLIENT_CERT_DEFAULT = false;
public static final String SSL_HOSTNAME_VERIFIER_KEY =
- "hadoop.ssl.hostname.verifier";
- public static final String SSL_CLIENT_CONF_KEY =
- "hadoop.ssl.client.conf";
- public static final String SSL_SERVER_CONF_KEY =
- "hadoop.ssl.server.conf";
- public static final String SSLCERTIFICATE = IBM_JAVA?"ibmX509":"SunX509";
+ "hadoop.ssl.hostname.verifier";
+ public static final String SSL_ENABLED_PROTOCOLS_KEY =
+ "hadoop.ssl.enabled.protocols";
+ public static final String SSL_ENABLED_PROTOCOLS_DEFAULT =
+ "TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2";
+
+ public static final String SSL_SERVER_NEED_CLIENT_AUTH =
+ "ssl.server.need.client.auth";
+ public static final boolean SSL_SERVER_NEED_CLIENT_AUTH_DEFAULT = false;
+
+ public static final String SSL_SERVER_KEYSTORE_LOCATION =
+ "ssl.server.keystore.location";
+ public static final String SSL_SERVER_KEYSTORE_PASSWORD =
+ "ssl.server.keystore.password";
+ public static final String SSL_SERVER_KEYSTORE_TYPE =
+ "ssl.server.keystore.type";
+ public static final String SSL_SERVER_KEYSTORE_TYPE_DEFAULT = "jks";
+ public static final String SSL_SERVER_KEYSTORE_KEYPASSWORD =
+ "ssl.server.keystore.keypassword";
+
+ public static final String SSL_SERVER_TRUSTSTORE_LOCATION =
+ "ssl.server.truststore.location";
+ public static final String SSL_SERVER_TRUSTSTORE_PASSWORD =
+ "ssl.server.truststore.password";
+ public static final String SSL_SERVER_TRUSTSTORE_TYPE =
+ "ssl.server.truststore.type";
+ public static final String SSL_SERVER_TRUSTSTORE_TYPE_DEFAULT = "jks";
+
+ public static final String SSL_SERVER_EXCLUDE_CIPHER_LIST =
+ "ssl.server.exclude.cipher.list";
- public static final boolean DEFAULT_SSL_REQUIRE_CLIENT_CERT = false;
+ @Deprecated
+ public static final boolean DEFAULT_SSL_REQUIRE_CLIENT_CERT =
+ SSL_REQUIRE_CLIENT_CERT_DEFAULT;
+
+ public static final String SSLCERTIFICATE = IBM_JAVA?"ibmX509":"SunX509";
public static final String KEYSTORES_FACTORY_CLASS_KEY =
"hadoop.ssl.keystores.factory.class";
+ @Deprecated
public static final String SSL_ENABLED_PROTOCOLS =
- "hadoop.ssl.enabled.protocols";
+ SSL_ENABLED_PROTOCOLS_KEY;
+ @Deprecated
public static final String DEFAULT_SSL_ENABLED_PROTOCOLS =
- "TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2";
- public static final String SSL_SERVER_EXCLUDE_CIPHER_LIST =
- "ssl.server.exclude.cipher.list";
+ SSL_ENABLED_PROTOCOLS_DEFAULT;
private Configuration conf;
private Mode mode;
@@ -131,9 +165,11 @@ public class SSLFactory implements ConnectionConfigurator {
sslConf.setBoolean(SSL_REQUIRE_CLIENT_CERT_KEY, requireClientCert);
String sslConfResource;
if (mode == Mode.CLIENT) {
- sslConfResource = conf.get(SSL_CLIENT_CONF_KEY, "ssl-client.xml");
+ sslConfResource = conf.get(SSL_CLIENT_CONF_KEY,
+ SSL_CLIENT_CONF_DEFAULT);
} else {
- sslConfResource = conf.get(SSL_SERVER_CONF_KEY, "ssl-server.xml");
+ sslConfResource = conf.get(SSL_SERVER_CONF_KEY,
+ SSL_SERVER_CONF_DEFAULT);
}
sslConf.addResource(sslConfResource);
return sslConf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index 27a858a..8d98e91 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -207,6 +207,12 @@ NOTE: Some KeyProviders (e.g. org.apache.hadoop.crypto.key.JavaKeyStoreProvider)
NOTE: Some KeyProviders do not directly execute a key deletion (e.g. performs a soft-delete instead, or delay the actual deletion, to prevent mistake). In these cases, one may encounter errors when creating/deleting a key with the same name after deleting it. Please check the underlying KeyProvider for details.
+### `kms`
+
+Usage: `hadoop kms`
+
+Run KMS, the Key Management Server.
+
### `trace`
View and modify Hadoop tracing settings. See the [Tracing Guide](./Tracing.html).
@@ -267,8 +273,6 @@ This command works by sending a HTTP/HTTPS request to the daemon's internal Jett
* node manager
* Timeline server
-However, the command does not support KMS server, because its web interface is based on Tomcat, which does not support the servlet.
-
Files
-----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_mkdir.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_mkdir.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_mkdir.bats
new file mode 100644
index 0000000..90a4f1a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_mkdir.bats
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+@test "hadoop_mkdir (create)" {
+ DIR=${BATS_TMPDIR}/nodir
+ rm -fr ${DIR}
+ run hadoop_mkdir ${DIR}
+ [ "${status}" = 0 ]
+ [ "${output}" = "WARNING: ${DIR} does not exist. Creating." ]
+}
+
+
+@test "hadoop_mkdir (exists)" {
+ DIR=${BATS_TMPDIR}/exists
+ mkdir -p ${DIR}
+ run hadoop_mkdir ${DIR}
+ [ "${status}" = 0 ]
+ [ -z "${output}" ]
+}
+
+
+@test "hadoop_mkdir (failed)" {
+ DIR=${BATS_TMPDIR}/readonly_dir/dir
+ mkdir -p ${BATS_TMPDIR}/readonly_dir
+ chmod a-w ${BATS_TMPDIR}/readonly_dir
+ run hadoop_mkdir ${DIR}
+ [ "${status}" != 0 ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_using_envvar.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_using_envvar.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_using_envvar.bats
new file mode 100644
index 0000000..8f8e937
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_using_envvar.bats
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+@test "hadoop_using_envvar (has value)" {
+ HADOOP_SHELL_SCRIPT_DEBUG=true
+ VAR=value
+ run hadoop_using_envvar VAR
+ [ "${status}" = 0 ]
+ [ "${output}" = "DEBUG: VAR = value" ]
+}
+
+
+@test "hadoop_using_envvar (no value)" {
+ HADOOP_SHELL_SCRIPT_DEBUG=true
+ VAR=
+ run hadoop_using_envvar VAR
+ [ "${status}" = 0 ]
+ [ -z "${output}" ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml
index 929936d..f864c03 100644
--- a/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml
@@ -39,7 +39,7 @@
<Bug pattern="DM_EXIT"/>
</Match>
<!--
- KMS wants to log the exception before it's thrown to tomcat and disappear.
+ KMS wants to log the exception before it's thrown to Jetty and disappear.
-->
<Match>
<Class name="org.apache.hadoop.crypto.key.kms.server.KMS"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 73b8339..41c36e8 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -27,20 +27,11 @@
</parent>
<artifactId>hadoop-kms</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
- <packaging>war</packaging>
+ <packaging>jar</packaging>
<name>Apache Hadoop KMS</name>
<description>Apache Hadoop KMS</description>
- <properties>
- <kms.tomcat.dist.dir>
- ${project.build.directory}/${project.artifactId}-${project.version}/share/hadoop/kms/tomcat
- </kms.tomcat.dist.dir>
- <tomcat.download.url>
- http://archive.apache.org/dist/tomcat/tomcat-6/v${tomcat.version}/bin/apache-tomcat-${tomcat.version}.tar.gz
- </tomcat.download.url>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -80,12 +71,14 @@
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
- <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -101,14 +94,6 @@
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</exclusion>
@@ -229,66 +214,21 @@
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
- <id>create-web-xmls</id>
- <phase>generate-test-resources</phase>
+ <id>site</id>
+ <phase>site</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
- <mkdir dir="${project.build.directory}/test-classes/kms-webapp"/>
-
- <copy todir="${project.build.directory}/test-classes/kms-webapp">
- <fileset dir="${basedir}/src/main/webapp"/>
- </copy>
+ <xslt in="${basedir}/src/main/resources/kms-default.xml"
+ out="${project.build.directory}/site/kms-default.html"
+ style="${basedir}/src/site/configuration.xsl"/>
</target>
</configuration>
</execution>
</executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-war-plugin</artifactId>
- <executions>
- <execution>
- <id>default-war</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>war</goal>
- </goals>
- <configuration>
- <archiveClasses>true</archiveClasses>
- <warName>kms</warName>
- <webappDirectory>${project.build.directory}/kms
- </webappDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <id>prepare-jar</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>classes</classifier>
- </configuration>
- </execution>
- <execution>
- <id>prepare-test-jar</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
+ </plugin> <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
<configuration>
@@ -360,84 +300,6 @@
</execution>
</executions>
</plugin>
- <!-- Downloading Tomcat TAR.GZ, using downloads/ dir to avoid downloading over an over -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>dist</id>
- <goals>
- <goal>run</goal>
- </goals>
- <phase>package</phase>
- <configuration>
- <target>
- <mkdir dir="downloads"/>
- <get
- src="${tomcat.download.url}"
- dest="downloads/apache-tomcat-${tomcat.version}.tar.gz"
- verbose="true" skipexisting="true"/>
- <delete dir="${project.build.directory}/tomcat.exp"/>
- <mkdir dir="${project.build.directory}/tomcat.exp"/>
-
- <!-- Using Unix script to preserve file permissions -->
- <echo file="${project.build.directory}/tomcat-untar.sh">
- cd "${project.build.directory}/tomcat.exp"
- gzip -cd ../../downloads/apache-tomcat-${tomcat.version}.tar.gz | tar xf -
- </echo>
- <exec executable="${shell-executable}" dir="${project.build.directory}"
- failonerror="true">
- <arg line="./tomcat-untar.sh"/>
- </exec>
-
- <move
- file="${project.build.directory}/tomcat.exp/apache-tomcat-${tomcat.version}"
- tofile="${kms.tomcat.dist.dir}"/>
- <delete dir="${project.build.directory}/tomcat.exp"/>
- <delete dir="${kms.tomcat.dist.dir}/webapps"/>
- <mkdir dir="${kms.tomcat.dist.dir}/webapps"/>
- <delete file="${kms.tomcat.dist.dir}/conf/server.xml"/>
- <copy file="${basedir}/src/main/tomcat/server.xml"
- toDir="${kms.tomcat.dist.dir}/conf"/>
- <delete file="${kms.tomcat.dist.dir}/conf/ssl-server.xml.conf"/>
- <copy file="${basedir}/src/main/tomcat/ssl-server.xml.conf"
- toDir="${kms.tomcat.dist.dir}/conf"/>
- <delete
- file="${kms.tomcat.dist.dir}/conf/logging.properties"/>
- <copy file="${basedir}/src/main/tomcat/logging.properties"
- toDir="${kms.tomcat.dist.dir}/conf"/>
- <copy toDir="${kms.tomcat.dist.dir}/webapps/ROOT">
- <fileset dir="${basedir}/src/main/tomcat/ROOT"/>
- </copy>
- <copy toDir="${kms.tomcat.dist.dir}/webapps/kms">
- <fileset dir="${project.build.directory}/kms"/>
- </copy>
- </target>
- </configuration>
- </execution>
- <execution>
- <id>tar</id>
- <phase>package</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target if="tar">
- <!-- Using Unix script to preserve symlinks -->
- <echo file="${project.build.directory}/dist-maketar.sh">
- cd "${project.build.directory}"
- tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
- </echo>
- <exec executable="${shell-executable}" dir="${project.build.directory}"
- failonerror="true">
- <arg line="./dist-maketar.sh"/>
- </exec>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
</profile>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
index e42904d..0528932 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
@@ -18,6 +18,14 @@
# hadoop-env.sh is read prior to this file.
#
+# KMS config directory
+#
+# export KMS_CONFIG=${HADOOP_CONF_DIR}
+
+# KMS log directory
+#
+# export KMS_LOG=${HADOOP_LOG_DIR}
+
# KMS temporary directory
#
# export KMS_TEMP=${HADOOP_HOME}/temp
@@ -26,48 +34,22 @@
#
# export KMS_HTTP_PORT=9600
-# The Admin port used by KMS
-#
-# export KMS_ADMIN_PORT=$((KMS_HTTP_PORT + 1))
-
-# The maximum number of Tomcat handler threads
+# The maximum number of HTTP handler threads
#
# export KMS_MAX_THREADS=1000
-# The maximum size of Tomcat HTTP header
+# The maximum size of HTTP header
#
# export KMS_MAX_HTTP_HEADER_SIZE=65536
+# Whether SSL is enabled
+#
+# export KMS_SSL_ENABLED=false
+
# The location of the SSL keystore if using SSL
#
# export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore
-#
# The password of the SSL keystore if using SSL
#
-# export KMS_SSL_KEYSTORE_PASS=password
-
-
-##
-## Tomcat specific settings
-##
-#
-# Location of tomcat
-#
-# export KMS_CATALINA_HOME=${HADOOP_HOME}/share/hadoop/kms/tomcat
-
-# Java System properties for KMS should be specified in this variable.
-# The java.library.path and hadoop.home.dir properties are automatically
-# configured. In order to supplement java.library.path,
-# one should add to the JAVA_LIBRARY_PATH env var.
-#
-# export CATALINA_OPTS=
-
-# PID file
-#
-# export CATALINA_PID=${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms.pid
-
-# Output file
-#
-# export CATALINA_OUT=${KMS_LOG}/hadoop-${HADOOP_IDENT_STRING}-kms-${HOSTNAME}.out
-
+# export KMS_SSL_KEYSTORE_PASS=password
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties b/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties
index 8e6d909..15ff436 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties
@@ -32,7 +32,6 @@ log4j.appender.kms-audit.layout.ConversionPattern=%d{ISO8601} %m%n
log4j.logger.kms-audit=INFO, kms-audit
log4j.additivity.kms-audit=false
-log4j.rootLogger=ALL, kms
-log4j.logger.org.apache.hadoop.conf=ERROR
+log4j.rootLogger=INFO, kms
log4j.logger.org.apache.hadoop=INFO
log4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
index d188735..85e71c3 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
@@ -12,172 +12,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<configuration>
-
- <!-- KMS Backend KeyProvider -->
-
- <property>
- <name>hadoop.kms.key.provider.uri</name>
- <value>jceks://file@/${user.home}/kms.keystore</value>
- <description>
- URI of the backing KeyProvider for the KMS.
- </description>
- </property>
-
- <property>
- <name>hadoop.security.keystore.java-keystore-provider.password-file</name>
- <value>kms.keystore.password</value>
- <description>
- If using the JavaKeyStoreProvider, the file name for the keystore password.
- </description>
- </property>
-
- <!-- KMS Cache -->
-
- <property>
- <name>hadoop.kms.cache.enable</name>
- <value>true</value>
- <description>
- Whether the KMS will act as a cache for the backing KeyProvider.
- When the cache is enabled, operations like getKeyVersion, getMetadata,
- and getCurrentKey will sometimes return cached data without consulting
- the backing KeyProvider. Cached values are flushed when keys are deleted
- or modified.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.cache.timeout.ms</name>
- <value>600000</value>
- <description>
- Expiry time for the KMS key version and key metadata cache, in
- milliseconds. This affects getKeyVersion and getMetadata.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.current.key.cache.timeout.ms</name>
- <value>30000</value>
- <description>
- Expiry time for the KMS current key cache, in milliseconds. This
- affects getCurrentKey operations.
- </description>
- </property>
-
- <!-- KMS Audit -->
-
- <property>
- <name>hadoop.kms.audit.aggregation.window.ms</name>
- <value>10000</value>
- <description>
- Duplicate audit log events within the aggregation window (specified in
- ms) are quashed to reduce log traffic. A single message for aggregated
- events is printed at the end of the window, along with a count of the
- number of aggregated events.
- </description>
- </property>
-
- <!-- KMS Security -->
-
- <property>
- <name>hadoop.kms.authentication.type</name>
- <value>simple</value>
- <description>
- Authentication type for the KMS. Can be either "simple"
- or "kerberos".
- </description>
- </property>
- <property>
- <name>hadoop.kms.authentication.kerberos.keytab</name>
- <value>${user.home}/kms.keytab</value>
- <description>
- Path to the keytab with credentials for the configured Kerberos principal.
- </description>
- </property>
+<!-- Put site-specific property overrides in this file. -->
- <property>
- <name>hadoop.kms.authentication.kerberos.principal</name>
- <value>HTTP/localhost</value>
- <description>
- The Kerberos principal to use for the HTTP endpoint.
- The principal must start with 'HTTP/' as per the Kerberos HTTP SPNEGO specification.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.kerberos.name.rules</name>
- <value>DEFAULT</value>
- <description>
- Rules used to resolve Kerberos principal names.
- </description>
- </property>
-
- <!-- Authentication cookie signature source -->
-
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider</name>
- <value>random</value>
- <description>
- Indicates how the secret to sign the authentication cookies will be
- stored. Options are 'random' (default), 'string' and 'zookeeper'.
- If using a setup with multiple KMS instances, 'zookeeper' should be used.
- </description>
- </property>
-
- <!-- Configuration for 'zookeeper' authentication cookie signature source -->
-
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.path</name>
- <value>/hadoop-kms/hadoop-auth-signature-secret</value>
- <description>
- The Zookeeper ZNode path where the KMS instances will store and retrieve
- the secret from.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string</name>
- <value>#HOSTNAME#:#PORT#,...</value>
- <description>
- The Zookeeper connection string, a list of hostnames and port comma
- separated.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type</name>
- <value>none</value>
- <description>
- The Zookeeper authentication type, 'none' (default) or 'sasl' (Kerberos).
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab</name>
- <value>/etc/hadoop/conf/kms.keytab</value>
- <description>
- The absolute path for the Kerberos keytab with the credentials to
- connect to Zookeeper.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal</name>
- <value>kms/#HOSTNAME#</value>
- <description>
- The Kerberos service principal used to connect to Zookeeper.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.audit.logger</name>
- <value>org.apache.hadoop.crypto.key.kms.server.SimpleKMSAuditLogger</value>
- <description>
- The audit logger for KMS. It is a comma-separated list of KMSAuditLogger
- class names. Default is the text-format SimpleKMSAuditLogger only.
- If this is not configured, default will be used.
- </description>
- </property>
+<configuration>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index 600f1e9..1ef6c4e 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -32,6 +32,7 @@ import java.net.URL;
public class KMSConfiguration {
public static final String KMS_CONFIG_DIR = "kms.config.dir";
+ public static final String KMS_DEFAULT_XML = "kms-default.xml";
public static final String KMS_SITE_XML = "kms-site.xml";
public static final String KMS_ACLS_XML = "kms-acls.xml";
@@ -42,6 +43,16 @@ public class KMSConfiguration {
public static final String DEFAULT_KEY_ACL_PREFIX = "default.key.acl.";
public static final String WHITELIST_KEY_ACL_PREFIX = "whitelist.key.acl.";
+ // HTTP properties
+ public static final String HTTP_PORT_KEY = "hadoop.kms.http.port";
+ public static final int HTTP_PORT_DEFAULT = 9600;
+ public static final String HTTP_HOST_KEY = "hadoop.kms.http.host";
+ public static final String HTTP_HOST_DEFAULT = "0.0.0.0";
+
+ // SSL properties
+ public static final String SSL_ENABLED_KEY = "hadoop.kms.ssl.enabled";
+ public static final boolean SSL_ENABLED_DEFAULT = false;
+
// Property to set the backing KeyProvider
public static final String KEY_PROVIDER_URI = CONFIG_PREFIX +
"key.provider.uri";
@@ -77,6 +88,11 @@ public class KMSConfiguration {
public static final boolean KEY_AUTHORIZATION_ENABLE_DEFAULT = true;
+ static {
+ Configuration.addDefaultResource(KMS_DEFAULT_XML);
+ Configuration.addDefaultResource(KMS_SITE_XML);
+ }
+
static Configuration getConfiguration(boolean loadHadoopDefaults,
String ... resources) {
Configuration conf = new Configuration(loadHadoopDefaults);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
deleted file mode 100644
index 6918015..0000000
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.crypto.key.kms.server;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.jmx.JMXJsonServlet;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import java.io.IOException;
-
-@InterfaceAudience.Private
-public class KMSJMXServlet extends JMXJsonServlet {
-
- @Override
- protected boolean isInstrumentationAccessAllowed(HttpServletRequest request,
- HttpServletResponse response) throws IOException {
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
index 40ae19f..857139f 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -17,10 +17,17 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+
import com.codahale.metrics.JmxReporter;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.CachingKeyProvider;
@@ -34,15 +41,6 @@ import org.apache.hadoop.util.VersionInfo;
import org.apache.log4j.PropertyConfigurator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.slf4j.bridge.SLF4JBridgeHandler;
-
-import javax.servlet.ServletContextEvent;
-import javax.servlet.ServletContextListener;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URL;
@InterfaceAudience.Private
public class KMSWebApp implements ServletContextListener {
@@ -81,11 +79,6 @@ public class KMSWebApp implements ServletContextListener {
private static KMSAudit kmsAudit;
private static KeyProviderCryptoExtension keyProviderCryptoExtension;
- static {
- SLF4JBridgeHandler.removeHandlersForRootLogger();
- SLF4JBridgeHandler.install();
- }
-
private void initLogging(String confDir) {
if (System.getProperty("log4j.configuration") == null) {
System.setProperty("log4j.defaultInitOverride", "true");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
new file mode 100644
index 0000000..70945cb
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URL;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.ConfigurationWithLogging;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The KMS web server.
+ */
+@InterfaceAudience.Private
+public class KMSWebServer {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(KMSWebServer.class);
+
+ private static final String NAME = "kms";
+ private static final String SERVLET_PATH = "/kms";
+
+ private final HttpServer2 httpServer;
+ private final String scheme;
+
+ KMSWebServer(Configuration cnf) throws Exception {
+ ConfigurationWithLogging conf = new ConfigurationWithLogging(cnf);
+
+ // Add SSL configuration file
+ conf.addResource(conf.get(SSLFactory.SSL_SERVER_CONF_KEY,
+ SSLFactory.SSL_SERVER_CONF_DEFAULT));
+
+ // Override configuration with deprecated environment variables.
+ deprecateEnv("KMS_TEMP", conf, HttpServer2.HTTP_TEMP_DIR_KEY,
+ KMSConfiguration.KMS_SITE_XML);
+ deprecateEnv("KMS_HTTP_PORT", conf,
+ KMSConfiguration.HTTP_PORT_KEY, KMSConfiguration.KMS_SITE_XML);
+ deprecateEnv("KMS_MAX_THREADS", conf,
+ HttpServer2.HTTP_MAX_THREADS_KEY, KMSConfiguration.KMS_SITE_XML);
+ deprecateEnv("KMS_MAX_HTTP_HEADER_SIZE", conf,
+ HttpServer2.HTTP_MAX_REQUEST_HEADER_SIZE_KEY,
+ KMSConfiguration.KMS_SITE_XML);
+ deprecateEnv("KMS_MAX_HTTP_HEADER_SIZE", conf,
+ HttpServer2.HTTP_MAX_RESPONSE_HEADER_SIZE_KEY,
+ KMSConfiguration.KMS_SITE_XML);
+ deprecateEnv("KMS_SSL_ENABLED", conf,
+ KMSConfiguration.SSL_ENABLED_KEY, KMSConfiguration.KMS_SITE_XML);
+ deprecateEnv("KMS_SSL_KEYSTORE_FILE", conf,
+ SSLFactory.SSL_SERVER_KEYSTORE_LOCATION,
+ SSLFactory.SSL_SERVER_CONF_DEFAULT);
+ deprecateEnv("KMS_SSL_KEYSTORE_PASS", conf,
+ SSLFactory.SSL_SERVER_KEYSTORE_PASSWORD,
+ SSLFactory.SSL_SERVER_CONF_DEFAULT);
+
+ boolean sslEnabled = conf.getBoolean(KMSConfiguration.SSL_ENABLED_KEY,
+ KMSConfiguration.SSL_ENABLED_DEFAULT);
+ scheme = sslEnabled ? HttpServer2.HTTPS_SCHEME : HttpServer2.HTTP_SCHEME;
+
+ String host = conf.get(KMSConfiguration.HTTP_HOST_KEY,
+ KMSConfiguration.HTTP_HOST_DEFAULT);
+ int port = conf.getInt(KMSConfiguration.HTTP_PORT_KEY,
+ KMSConfiguration.HTTP_PORT_DEFAULT);
+ URI endpoint = new URI(scheme, null, host, port, null, null, null);
+
+ httpServer = new HttpServer2.Builder()
+ .setName(NAME)
+ .setConf(conf)
+ .setSSLConf(conf)
+ .authFilterConfigurationPrefix(KMSAuthenticationFilter.CONFIG_PREFIX)
+ .addEndpoint(endpoint)
+ .build();
+ }
+
+ /**
+ * Load the deprecated environment variable into the configuration.
+ *
+ * @param varName the environment variable name
+ * @param conf the configuration
+ * @param propName the configuration property name
+ * @param confFile the configuration file name
+ */
+ private static void deprecateEnv(String varName, Configuration conf,
+ String propName, String confFile) {
+ String value = System.getenv(varName);
+ if (value == null) {
+ return;
+ }
+ String propValue = conf.get(propName);
+ LOG.warn("Environment variable {} = '{}' is deprecated and overriding"
+ + " property {} = '{}', please set the property in {} instead.",
+ varName, value, propName, propValue, confFile);
+ conf.set(propName, value, "environment variable " + varName);
+ }
+
+ public void start() throws IOException {
+ httpServer.start();
+ }
+
+ public boolean isRunning() {
+ return httpServer.isAlive();
+ }
+
+ public void join() throws InterruptedException {
+ httpServer.join();
+ }
+
+ public void stop() throws Exception {
+ httpServer.stop();
+ }
+
+ public URL getKMSUrl() {
+ InetSocketAddress addr = httpServer.getConnectorAddress(0);
+ if (null == addr) {
+ return null;
+ }
+ try {
+ return new URL(scheme, addr.getHostName(), addr.getPort(),
+ SERVLET_PATH);
+ } catch (MalformedURLException ex) {
+ throw new RuntimeException("It should never happen: " + ex.getMessage(),
+ ex);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ StringUtils.startupShutdownMessage(KMSWebServer.class, args, LOG);
+ Configuration conf = KMSConfiguration.getKMSConf();
+ KMSWebServer kmsWebServer = new KMSWebServer(conf);
+ kmsWebServer.start();
+ kmsWebServer.join();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
deleted file mode 100644
index 52dba38..0000000
--- a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-function hadoop_subproject_init
-{
- local this
- local binparent
- local varlist
-
- if [[ -z "${HADOOP_KMS_ENV_PROCESSED}" ]]; then
- if [[ -e "${HADOOP_CONF_DIR}/kms-env.sh" ]]; then
- . "${HADOOP_CONF_DIR}/kms-env.sh"
- export HADOOP_KMS_ENV_PROCESSED=true
- fi
- fi
-
- export HADOOP_CATALINA_PREFIX=kms
-
- export HADOOP_CATALINA_TEMP="${KMS_TEMP:-${HADOOP_HOME}/temp}"
-
- hadoop_deprecate_envvar KMS_CONFIG HADOOP_CONF_DIR
-
- hadoop_deprecate_envvar KMS_LOG HADOOP_LOG_DIR
-
- export HADOOP_CATALINA_CONFIG="${HADOOP_CONF_DIR}"
- export HADOOP_CATALINA_LOG="${HADOOP_LOG_DIR}"
-
- export HADOOP_CATALINA_HTTP_PORT="${KMS_HTTP_PORT:-9600}"
- export HADOOP_CATALINA_ADMIN_PORT="${KMS_ADMIN_PORT:-$((HADOOP_CATALINA_HTTP_PORT+1))}"
- export HADOOP_CATALINA_MAX_THREADS="${KMS_MAX_THREADS:-1000}"
- export HADOOP_CATALINA_MAX_HTTP_HEADER_SIZE="${KMS_MAX_HTTP_HEADER_SIZE:-65536}"
-
- export HADOOP_CATALINA_SSL_KEYSTORE_FILE="${KMS_SSL_KEYSTORE_FILE:-${HOME}/.keystore}"
-
- export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_HOME}/share/hadoop/kms/tomcat}"
- export HADOOP_CATALINA_HOME="${KMS_CATALINA_HOME:-${CATALINA_BASE}}"
-
- export CATALINA_OUT="${CATALINA_OUT:-${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms-${HOSTNAME}.out}"
-
- export CATALINA_PID="${CATALINA_PID:-${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms.pid}"
-
- if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
- varlist=$(env | egrep '(^KMS|^CATALINA)' | cut -f1 -d= | grep -v _PASS)
- for i in ${varlist}; do
- hadoop_debug "Setting ${i} to ${!i}"
- done
- fi
-}
-
-if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
- [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
- . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
-elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
- . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then
- . "${HADOOP_HOME}/libexec/hadoop-config.sh"
-else
- echo "ERROR: Hadoop common not found." 2>&1
- exit 1
-fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/libexec/shellprofile.d/hadoop-kms.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/libexec/shellprofile.d/hadoop-kms.sh b/hadoop-common-project/hadoop-kms/src/main/libexec/shellprofile.d/hadoop-kms.sh
new file mode 100755
index 0000000..c530716
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/main/libexec/shellprofile.d/hadoop-kms.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+ hadoop_add_subcommand "kms" "run KMS, the Key Management Server"
+fi
+
+## @description Command handler for kms subcommand
+## @audience private
+## @stability stable
+## @replaceable no
+function hadoop_subcommand_kms
+{
+ if [[ -f "${HADOOP_CONF_DIR}/kms-env.sh" ]]; then
+ # shellcheck disable=SC1090
+ . "${HADOOP_CONF_DIR}/kms-env.sh"
+ fi
+
+ hadoop_deprecate_envvar KMS_CONFIG HADOOP_CONF_DIR
+ hadoop_deprecate_envvar KMS_LOG HADOOP_LOG_DIR
+
+ hadoop_using_envvar KMS_HTTP_PORT
+ hadoop_using_envvar KMS_MAX_HTTP_HEADER_SIZE
+ hadoop_using_envvar KMS_MAX_THREADS
+ hadoop_using_envvar KMS_SSL_ENABLED
+ hadoop_using_envvar KMS_SSL_KEYSTORE_FILE
+ hadoop_using_envvar KMS_TEMP
+
+ # shellcheck disable=SC2034
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION=true
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.crypto.key.kms.server.KMSWebServer
+
+ hadoop_add_param HADOOP_OPTS "-Dkms.config.dir=" \
+ "-Dkms.config.dir=${HADOOP_CONF_DIR}"
+ hadoop_add_param HADOOP_OPTS "-Dkms.log.dir=" \
+ "-Dkms.log.dir=${HADOOP_LOG_DIR}"
+
+ if [[ "${HADOOP_DAEMON_MODE}" == "default" ]] ||
+ [[ "${HADOOP_DAEMON_MODE}" == "start" ]]; then
+ hadoop_mkdir "${KMS_TEMP:-${HADOOP_HOME}/temp}"
+ fi
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml b/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml
new file mode 100644
index 0000000..2b178b8
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml
@@ -0,0 +1,248 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Do not modify this file directly. Instead, copy entries that you wish to
+ modify from this file into kms-site.xml and change them there. If
+ kms-site.xml does not already exist, create it.
+-->
+
+<configuration>
+
+ <property>
+ <name>hadoop.kms.http.port</name>
+ <value>9600</value>
+ <description>
+ The HTTP port for KMS REST API.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.http.host</name>
+ <value>0.0.0.0</value>
+ <description>
+ The bind host for KMS REST API.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.ssl.enabled</name>
+ <value>false</value>
+ <description>
+ Whether SSL is enabled. Default is false, i.e. disabled.
+ </description>
+ </property>
+
+ <!-- HTTP properties -->
+
+ <property>
+ <name>hadoop.http.max.threads</name>
+ <value>1000</value>
+ <description>
+ The maxmimum number of threads.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.max.request.header.size</name>
+ <value>65536</value>
+ <description>
+ The maxmimum HTTP request header size.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.max.response.header.size</name>
+ <value>65536</value>
+ <description>
+ The maxmimum HTTP response header size.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.temp.dir</name>
+ <value>${hadoop.tmp.dir}/kms</value>
+ <description>
+ KMS temp directory.
+ </description>
+ </property>
+
+ <!-- KMS Backend KeyProvider -->
+
+ <property>
+ <name>hadoop.kms.key.provider.uri</name>
+ <value>jceks://file@/${user.home}/kms.keystore</value>
+ <description>
+ URI of the backing KeyProvider for the KMS.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.keystore.java-keystore-provider.password-file</name>
+ <value></value>
+ <description>
+ If using the JavaKeyStoreProvider, the file name for the keystore password.
+ </description>
+ </property>
+
+ <!-- KMS Cache -->
+
+ <property>
+ <name>hadoop.kms.cache.enable</name>
+ <value>true</value>
+ <description>
+ Whether the KMS will act as a cache for the backing KeyProvider.
+ When the cache is enabled, operations like getKeyVersion, getMetadata,
+ and getCurrentKey will sometimes return cached data without consulting
+ the backing KeyProvider. Cached values are flushed when keys are deleted
+ or modified.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.cache.timeout.ms</name>
+ <value>600000</value>
+ <description>
+ Expiry time for the KMS key version and key metadata cache, in
+ milliseconds. This affects getKeyVersion and getMetadata.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.current.key.cache.timeout.ms</name>
+ <value>30000</value>
+ <description>
+ Expiry time for the KMS current key cache, in milliseconds. This
+ affects getCurrentKey operations.
+ </description>
+ </property>
+
+ <!-- KMS Audit -->
+
+ <property>
+ <name>hadoop.kms.audit.aggregation.window.ms</name>
+ <value>10000</value>
+ <description>
+ Duplicate audit log events within the aggregation window (specified in
+ ms) are quashed to reduce log traffic. A single message for aggregated
+ events is printed at the end of the window, along with a count of the
+ number of aggregated events.
+ </description>
+ </property>
+
+ <!-- KMS Security -->
+
+ <property>
+ <name>hadoop.kms.authentication.type</name>
+ <value>simple</value>
+ <description>
+ Authentication type for the KMS. Can be either 'simple' (default) or
+ 'kerberos'.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.kerberos.keytab</name>
+ <value>${user.home}/kms.keytab</value>
+ <description>
+ Path to the keytab with credentials for the configured Kerberos principal.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.kerberos.principal</name>
+ <value>HTTP/localhost</value>
+ <description>
+ The Kerberos principal to use for the HTTP endpoint.
+ The principal must start with 'HTTP/' as per the Kerberos HTTP SPNEGO specification.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.kerberos.name.rules</name>
+ <value>DEFAULT</value>
+ <description>
+ Rules used to resolve Kerberos principal names.
+ </description>
+ </property>
+
+ <!-- Authentication cookie signature source -->
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider</name>
+ <value>random</value>
+ <description>
+ Indicates how the secret to sign the authentication cookies will be
+ stored. Options are 'random' (default), 'string' and 'zookeeper'.
+ If using a setup with multiple KMS instances, 'zookeeper' should be used.
+ </description>
+ </property>
+
+ <!-- Configuration for 'zookeeper' authentication cookie signature source -->
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.path</name>
+ <value>/hadoop-kms/hadoop-auth-signature-secret</value>
+ <description>
+ The Zookeeper ZNode path where the KMS instances will store and retrieve
+ the secret from.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string</name>
+ <value>#HOSTNAME#:#PORT#,...</value>
+ <description>
+ The Zookeeper connection string, a list of hostnames and port comma
+ separated.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type</name>
+ <value>none</value>
+ <description>
+ The Zookeeper authentication type, 'none' (default) or 'sasl' (Kerberos).
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab</name>
+ <value>/etc/hadoop/conf/kms.keytab</value>
+ <description>
+ The absolute path for the Kerberos keytab with the credentials to
+ connect to Zookeeper.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal</name>
+ <value>kms/#HOSTNAME#</value>
+ <description>
+ The Kerberos service principal used to connect to Zookeeper.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.audit.logger</name>
+ <value>org.apache.hadoop.crypto.key.kms.server.SimpleKMSAuditLogger</value>
+ <description>
+ The audit logger for KMS. It is a comma-separated list of KMSAuditLogger
+ class names. Default is the text-format SimpleKMSAuditLogger only.
+ If this is not configured, default will be used.
+ </description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/resources/webapps/kms/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/resources/webapps/kms/WEB-INF/web.xml b/hadoop-common-project/hadoop-kms/src/main/resources/webapps/kms/WEB-INF/web.xml
new file mode 100644
index 0000000..1c14d28
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/main/resources/webapps/kms/WEB-INF/web.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
+
+ <listener>
+ <listener-class>org.apache.hadoop.crypto.key.kms.server.KMSWebApp</listener-class>
+ </listener>
+
+ <servlet>
+ <servlet-name>webservices-driver</servlet-name>
+ <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>org.apache.hadoop.crypto.key.kms.server</param-value>
+ </init-param>
+
+ <!-- Enables detailed Jersey request/response logging -->
+ <!--
+ <init-param>
+ <param-name>com.sun.jersey.spi.container.ContainerRequestFilters</param-name>
+ <param-value>com.sun.jersey.api.container.filter.LoggingFilter</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.spi.container.ContainerResponseFilters</param-name>
+ <param-value>com.sun.jersey.api.container.filter.LoggingFilter</param-value>
+ </init-param>
+ -->
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+
+ <servlet-mapping>
+ <servlet-name>webservices-driver</servlet-name>
+ <url-pattern>/kms/*</url-pattern>
+ </servlet-mapping>
+
+ <filter>
+ <filter-name>authFilter</filter-name>
+ <filter-class>org.apache.hadoop.crypto.key.kms.server.KMSAuthenticationFilter</filter-class>
+ </filter>
+
+ <filter>
+ <filter-name>MDCFilter</filter-name>
+ <filter-class>org.apache.hadoop.crypto.key.kms.server.KMSMDCFilter</filter-class>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>authFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
+ <filter-mapping>
+ <filter-name>MDCFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
+</web-app>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/resources/webapps/static/index.html
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/resources/webapps/static/index.html b/hadoop-common-project/hadoop-kms/src/main/resources/webapps/static/index.html
new file mode 100644
index 0000000..9925ad9
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/main/resources/webapps/static/index.html
@@ -0,0 +1,35 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
+-->
+<html>
+<head>
+ <title>Hadoop KMS</title>
+</head>
+<body>
+<h1>Hadoop KMS</h1>
+<ul>
+ <li>KMS REST API end-point <b>/kms/v1/</b></li>
+ <ul>
+ <li><a href="/kms/v1/keys/names">/kms/v1/keys/names</a>
+ to list all keys</li>
+ </ul>
+ <li><a href="/conf">KMS configuration properties</a></li>
+ <li><a href="/jmx">KMS JMX</a></li>
+ <li><a href="/logLevel">KMS log level</a></li>
+ <li><a href="/logs">KMS log files</a></li>
+ <li><a href="/stacks">KMS stacks</a></li>
+</ul>
+</body>
+</html>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HADOOP-13885. Implement
getLinkTarget for ViewFileSystem. Contributed by Manoj Govindassamy.
Posted by st...@apache.org.
HADOOP-13885. Implement getLinkTarget for ViewFileSystem. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/511d39e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/511d39e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/511d39e0
Branch: refs/heads/HADOOP-13345
Commit: 511d39e0740f36bf937e7bcf974e1050f0e7c1e0
Parents: 603cbcd
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Jan 9 18:01:10 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Jan 9 18:01:37 2017 -0800
----------------------------------------------------------------------
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 5 ++
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 11 +++
.../fs/viewfs/ViewFileSystemBaseTest.java | 73 ++++++++++++++++++++
3 files changed, 89 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/511d39e0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index 272433f..fe2f8a9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -234,6 +234,11 @@ class ChRootedFileSystem extends FilterFileSystem {
}
@Override
+ public Path getLinkTarget(Path f) throws IOException {
+ return super.getLinkTarget(fullPath(f));
+ }
+
+ @Override
public void access(Path path, FsAction mode) throws AccessControlException,
FileNotFoundException, IOException {
super.access(fullPath(path), mode);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/511d39e0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 8be666c..e16da64 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -876,6 +876,17 @@ public class ViewFileSystem extends FileSystem {
}
}
+ @Override
+ public Path getLinkTarget(Path path) throws IOException {
+ InodeTree.ResolveResult<FileSystem> res;
+ try {
+ res = fsState.resolve(getUriPath(path), true);
+ } catch (FileNotFoundException e) {
+ throw new NotInMountpointException(path, "getLinkTarget");
+ }
+ return res.targetFileSystem.getLinkTarget(res.remainingPath);
+ }
+
/**
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/511d39e0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index 9a0bf02..61295b4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.junit.Assume;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
@@ -1137,4 +1138,76 @@ abstract public class ViewFileSystemBaseTest {
"the mounted FileSystem!",
usedSpaceByPathViaTargetFs, usedSpaceByPathViaViewFs);
}
+
+ @Test
+ public void testLinkTarget() throws Exception {
+
+ Assume.assumeTrue(fsTarget.supportsSymlinks() &&
+ fsTarget.areSymlinksEnabled());
+
+ // Symbolic link
+ final String targetFileName = "debug.log";
+ final String linkFileName = "debug.link";
+ final Path targetFile = new Path(targetTestRoot, targetFileName);
+ final Path symLink = new Path(targetTestRoot, linkFileName);
+
+ FileSystemTestHelper.createFile(fsTarget, targetFile);
+ fsTarget.createSymlink(targetFile, symLink, false);
+
+ final Path mountTargetRootPath = new Path("/targetRoot");
+ final Path mountTargetSymLinkPath = new Path(mountTargetRootPath,
+ linkFileName);
+ final Path expectedMountLinkTarget = fsTarget.makeQualified(
+ new Path(targetTestRoot, targetFileName));
+ final Path actualMountLinkTarget = fsView.getLinkTarget(
+ mountTargetSymLinkPath);
+
+ assertEquals("Resolved link target path not matching!",
+ expectedMountLinkTarget, actualMountLinkTarget);
+
+ // Relative symbolic link
+ final String relativeFileName = "dir2/../" + targetFileName;
+ final String link2FileName = "dir2/rel.link";
+ final Path relTargetFile = new Path(targetTestRoot, relativeFileName);
+ final Path relativeSymLink = new Path(targetTestRoot, link2FileName);
+ fsTarget.createSymlink(relTargetFile, relativeSymLink, true);
+
+ final Path mountTargetRelativeSymLinkPath = new Path(mountTargetRootPath,
+ link2FileName);
+ final Path expectedMountRelLinkTarget = fsTarget.makeQualified(
+ new Path(targetTestRoot, relativeFileName));
+ final Path actualMountRelLinkTarget = fsView.getLinkTarget(
+ mountTargetRelativeSymLinkPath);
+
+ assertEquals("Resolved relative link target path not matching!",
+ expectedMountRelLinkTarget, actualMountRelLinkTarget);
+
+ try {
+ fsView.getLinkTarget(new Path("/linkToAFile"));
+ fail("Resolving link target for a ViewFs mount link should fail!");
+ } catch (Exception e) {
+ LOG.info("Expected exception: " + e);
+ assertThat(e.getMessage(),
+ containsString("not a symbolic link"));
+ }
+
+ try {
+ fsView.getLinkTarget(fsView.makeQualified(
+ new Path(mountTargetRootPath, targetFileName)));
+ fail("Resolving link target for a non sym link should fail!");
+ } catch (Exception e) {
+ LOG.info("Expected exception: " + e);
+ assertThat(e.getMessage(),
+ containsString("not a symbolic link"));
+ }
+
+ try {
+ fsView.getLinkTarget(new Path("/targetRoot/non-existing-file"));
+ fail("Resolving link target for a non existing link should fail!");
+ } catch (Exception e) {
+ LOG.info("Expected exception: " + e);
+ assertThat(e.getMessage(),
+ containsString("File does not exist:"));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HADOOP-13597. Switch KMS from
Tomcat to Jetty. Contributed by John Zhuge.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
index 7611f2a..b4955d5 100755
--- a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
@@ -13,92 +13,52 @@
# limitations under the License.
#
-MYNAME="${BASH_SOURCE-$0}"
+MYNAME="${0##*/}"
-function hadoop_usage
+## @description Print usage
+## @audience private
+## @stability stable
+## @replaceable no
+function print_usage
{
- hadoop_add_subcommand "run" "Start kms in the current window"
- hadoop_add_subcommand "run -security" "Start in the current window with security manager"
- hadoop_add_subcommand "start" "Start kms in a separate window"
- hadoop_add_subcommand "start -security" "Start in a separate window with security manager"
- hadoop_add_subcommand "status" "Return the LSB compliant status"
- hadoop_add_subcommand "stop" "Stop kms, waiting up to 5 seconds for the process to end"
- hadoop_add_subcommand "top n" "Stop kms, waiting up to n seconds for the process to end"
- hadoop_add_subcommand "stop -force" "Stop kms, wait up to 5 seconds and then use kill -KILL if still running"
- hadoop_add_subcommand "stop n -force" "Stop kms, wait up to n seconds and then use kill -KILL if still running"
- hadoop_generate_usage "${MYNAME}" false
+ cat <<EOF
+Usage: ${MYNAME} run|start|status|stop
+commands:
+ run Run KMS, the Key Management Server
+ start Start KMS as a daemon
+ status Return the status of the KMS daemon
+ stop Stop the KMS daemon
+EOF
}
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
- bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
- HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/kms-config.sh" ]]; then
- . "${HADOOP_LIBEXEC_DIR}/kms-config.sh"
-else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/kms-config.sh." 2>&1
- exit 1
-fi
-
-# The Java System property 'kms.http.port' it is not used by Kms,
-# it is used in Tomcat's server.xml configuration file
-#
-
-hadoop_debug "Using CATALINA_OPTS: ${CATALINA_OPTS}"
-
-# We're using hadoop-common, so set up some stuff it might need:
-hadoop_finalize
-
-hadoop_verify_logdir
+echo "WARNING: ${MYNAME} is deprecated," \
+ "please use 'hadoop [--daemon start|status|stop] kms'." >&2
if [[ $# = 0 ]]; then
- case "${HADOOP_DAEMON_MODE}" in
- status)
- hadoop_status_daemon "${CATALINA_PID}"
- exit
- ;;
- start)
- set -- "start"
- ;;
- stop)
- set -- "stop"
- ;;
- esac
+ print_usage
+ exit
fi
-hadoop_finalize_catalina_opts
-export CATALINA_OPTS
-
-# A bug in catalina.sh script does not use CATALINA_OPTS for stopping the server
-#
-if [[ "${1}" = "stop" ]]; then
- export JAVA_OPTS=${CATALINA_OPTS}
-fi
+case $1 in
+ run)
+ args=("kms")
+ ;;
+ start|stop|status)
+ args=("--daemon" "$1" "kms")
+ ;;
+ *)
+ echo "Unknown sub-command \"$1\"."
+ print_usage
+ exit 1
+ ;;
+esac
-# If ssl, the populate the passwords into ssl-server.xml before starting tomcat
-#
-# KMS_SSL_KEYSTORE_PASS is a bit odd.
-# if undefined, then the if test will not enable ssl on its own
-# if "", set it to "password".
-# if custom, use provided password
-#
-if [[ -f "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" ]]; then
- if [[ -n "${KMS_SSL_KEYSTORE_PASS+x}" ]]; then
- export KMS_SSL_KEYSTORE_PASS=${KMS_SSL_KEYSTORE_PASS:-password}
- KMS_SSL_KEYSTORE_PASS_ESCAPED=$(hadoop_xml_escape \
- "$(hadoop_sed_escape "$KMS_SSL_KEYSTORE_PASS")")
- sed -e 's/"_kms_ssl_keystore_pass_"/'"\"${KMS_SSL_KEYSTORE_PASS_ESCAPED}\""'/g' \
- "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" \
- > "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml"
- chmod 700 "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml" >/dev/null 2>&1
- fi
+# Locate bin
+if [[ -n "${HADOOP_HOME}" ]]; then
+ bin="${HADOOP_HOME}/bin"
+else
+ sbin=$(cd -P -- "$(dirname -- "$0")" >/dev/null && pwd -P)
+ bin=$(cd -P -- "${sbin}/../bin" >/dev/null && pwd -P)
fi
-exec "${HADOOP_CATALINA_HOME}/bin/catalina.sh" "$@"
+exec "${bin}/hadoop" "${args[@]}"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/WEB-INF/web.xml b/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/WEB-INF/web.xml
deleted file mode 100644
index 9d0ae0d..0000000
--- a/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/WEB-INF/web.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
-</web-app>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/index.html
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/index.html b/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/index.html
deleted file mode 100644
index e9e4512..0000000
--- a/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/index.html
+++ /dev/null
@@ -1,27 +0,0 @@
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
--->
-<html>
-<head>
- <title>Hadoop KMS</title>
-</head>
-<body>
-<h1>Hadoop KMS</h1>
-<ul>
- <li>KMS REST API end-point <b>/kms/v1/*</b></li>
- <li><a href="/kms/jmx">KMS JMX JSON end-point</a></li>
-</ul>
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/tomcat/logging.properties
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/tomcat/logging.properties b/hadoop-common-project/hadoop-kms/src/main/tomcat/logging.properties
deleted file mode 100644
index 7562719..0000000
--- a/hadoop-common-project/hadoop-kms/src/main/tomcat/logging.properties
+++ /dev/null
@@ -1,67 +0,0 @@
-#
-# All Rights Reserved.
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-handlers = 1catalina.org.apache.juli.FileHandler, 2localhost.org.apache.juli.FileHandler, 3manager.org.apache.juli.FileHandler, 4host-manager.org.apache.juli.FileHandler, java.util.logging.ConsoleHandler
-
-.handlers = 1catalina.org.apache.juli.FileHandler, java.util.logging.ConsoleHandler
-
-############################################################
-# Handler specific properties.
-# Describes specific configuration info for Handlers.
-############################################################
-
-1catalina.org.apache.juli.FileHandler.level = FINE
-1catalina.org.apache.juli.FileHandler.directory = ${kms.log.dir}
-1catalina.org.apache.juli.FileHandler.prefix = kms-catalina.
-
-2localhost.org.apache.juli.FileHandler.level = FINE
-2localhost.org.apache.juli.FileHandler.directory = ${kms.log.dir}
-2localhost.org.apache.juli.FileHandler.prefix = kms-localhost.
-
-3manager.org.apache.juli.FileHandler.level = FINE
-3manager.org.apache.juli.FileHandler.directory = ${kms.log.dir}
-3manager.org.apache.juli.FileHandler.prefix = kms-manager.
-
-4host-manager.org.apache.juli.FileHandler.level = FINE
-4host-manager.org.apache.juli.FileHandler.directory = ${kms.log.dir}
-4host-manager.org.apache.juli.FileHandler.prefix = kms-host-manager.
-
-java.util.logging.ConsoleHandler.level = FINE
-java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
-
-
-############################################################
-# Facility specific properties.
-# Provides extra control for each logger.
-############################################################
-
-org.apache.catalina.core.ContainerBase.[Catalina].[localhost].level = INFO
-org.apache.catalina.core.ContainerBase.[Catalina].[localhost].handlers = 2localhost.org.apache.juli.FileHandler
-
-org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/manager].level = INFO
-org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/manager].handlers = 3manager.org.apache.juli.FileHandler
-
-org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/host-manager].level = INFO
-org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/host-manager].handlers = 4host-manager.org.apache.juli.FileHandler
-
-# For example, set the com.xyz.foo logger to only log SEVERE
-# messages:
-#org.apache.catalina.startup.ContextConfig.level = FINE
-#org.apache.catalina.startup.HostConfig.level = FINE
-#org.apache.catalina.session.ManagerBase.level = FINE
-#org.apache.catalina.core.AprLifecycleListener.level=FINE
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml b/hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml
deleted file mode 100644
index d8fd161..0000000
--- a/hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml
+++ /dev/null
@@ -1,155 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<!--
-
- All Rights Reserved.
-
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Note: A "Server" is not itself a "Container", so you may not
- define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/server.html
- -->
-<Server port="${kms.admin.port}" shutdown="SHUTDOWN">
-
- <!--APR library loader. Documentation at /docs/apr.html -->
- <Listener className="org.apache.catalina.core.AprLifecycleListener"
- SSLEngine="on"/>
- <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
- <Listener className="org.apache.catalina.core.JasperListener"/>
- <!-- Prevent memory leaks due to use of particular java/javax APIs-->
- <Listener
- className="org.apache.catalina.core.JreMemoryLeakPreventionListener"/>
- <!-- JMX Support for the Tomcat server. Documentation at /docs/non-existent.html -->
- <Listener className="org.apache.catalina.mbeans.ServerLifecycleListener"/>
- <Listener
- className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener"/>
-
- <!-- Global JNDI resources
- Documentation at /docs/jndi-resources-howto.html
- -->
- <GlobalNamingResources>
- <!-- Editable user database that can also be used by
- UserDatabaseRealm to authenticate users
- -->
- <Resource name="UserDatabase" auth="Container"
- type="org.apache.catalina.UserDatabase"
- description="User database that can be updated and saved"
- factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
- pathname="conf/tomcat-users.xml"/>
- </GlobalNamingResources>
-
- <!-- A "Service" is a collection of one or more "Connectors" that share
- a single "Container" Note: A "Service" is not itself a "Container",
- so you may not define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/service.html
- -->
- <Service name="Catalina">
-
- <!--The connectors can use a shared executor, you can define one or more named thread pools-->
- <!--
- <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
- maxThreads="150" minSpareThreads="4"/>
- -->
-
-
- <!-- A "Connector" represents an endpoint by which requests are received
- and responses are returned. Documentation at :
- Java HTTP Connector: /docs/config/http.html (blocking & non-blocking)
- Java AJP Connector: /docs/config/ajp.html
- APR (HTTP/AJP) Connector: /docs/apr.html
- Define a non-SSL HTTP/1.1 Connector on port ${kms.http.port}
- -->
- <Connector port="${kms.http.port}" protocol="HTTP/1.1"
- maxThreads="${kms.max.threads}"
- connectionTimeout="20000"
- redirectPort="8443"
- maxHttpHeaderSize="${kms.max.http.header.size}"/>
- <!-- A "Connector" using the shared thread pool-->
- <!--
- <Connector executor="tomcatThreadPool"
- port="${kms.http.port}" protocol="HTTP/1.1"
- connectionTimeout="20000"
- redirectPort="8443" />
- -->
- <!-- Define a SSL HTTP/1.1 Connector on port 8443
- This connector uses the JSSE configuration, when using APR, the
- connector should be using the OpenSSL style configuration
- described in the APR documentation -->
- <!--
- <Connector port="8443" protocol="HTTP/1.1" SSLEnabled="true"
- maxThreads="150" scheme="https" secure="true"
- clientAuth="false" sslProtocol="TLS" />
- -->
-
- <!-- Define an AJP 1.3 Connector on port 8009 -->
-
-
- <!-- An Engine represents the entry point (within Catalina) that processes
- every request. The Engine implementation for Tomcat stand alone
- analyzes the HTTP headers included with the request, and passes them
- on to the appropriate Host (virtual host).
- Documentation at /docs/config/engine.html -->
-
- <!-- You should set jvmRoute to support load-balancing via AJP ie :
- <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
- -->
- <Engine name="Catalina" defaultHost="localhost">
-
- <!--For clustering, please take a look at documentation at:
- /docs/cluster-howto.html (simple how to)
- /docs/config/cluster.html (reference documentation) -->
- <!--
- <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
- -->
-
- <!-- The request dumper valve dumps useful debugging information about
- the request and response data received and sent by Tomcat.
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.valves.RequestDumperValve"/>
- -->
-
- <!-- This Realm uses the UserDatabase configured in the global JNDI
- resources under the key "UserDatabase". Any edits
- that are performed against this UserDatabase are immediately
- available for use by the Realm. -->
- <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
- resourceName="UserDatabase"/>
-
- <!-- Define the default virtual host
- Note: XML Schema validation will not work with Xerces 2.2.
- -->
- <Host name="localhost" appBase="webapps"
- unpackWARs="true" autoDeploy="true"
- xmlValidation="false" xmlNamespaceAware="false">
-
- <!-- SingleSignOn valve, share authentication between web applications
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
- -->
-
- <!-- Access log processes all example.
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
- prefix="localhost_access_log." suffix=".txt" pattern="common" resolveHosts="false"/>
- -->
-
- </Host>
- </Engine>
- </Service>
-</Server>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml.conf
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml.conf b/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml.conf
deleted file mode 100644
index 272542a..0000000
--- a/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml.conf
+++ /dev/null
@@ -1,136 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<!--
-
- All Rights Reserved.
-
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Note: A "Server" is not itself a "Container", so you may not
- define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/server.html
- -->
-<Server port="${kms.admin.port}" shutdown="SHUTDOWN">
-
- <!--APR library loader. Documentation at /docs/apr.html -->
- <Listener className="org.apache.catalina.core.AprLifecycleListener"
- SSLEngine="on"/>
- <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
- <Listener className="org.apache.catalina.core.JasperListener"/>
- <!-- Prevent memory leaks due to use of particular java/javax APIs-->
- <Listener
- className="org.apache.catalina.core.JreMemoryLeakPreventionListener"/>
- <!-- JMX Support for the Tomcat server. Documentation at /docs/non-existent.html -->
- <Listener className="org.apache.catalina.mbeans.ServerLifecycleListener"/>
- <Listener
- className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener"/>
-
- <!-- Global JNDI resources
- Documentation at /docs/jndi-resources-howto.html
- -->
- <GlobalNamingResources>
- <!-- Editable user database that can also be used by
- UserDatabaseRealm to authenticate users
- -->
- <Resource name="UserDatabase" auth="Container"
- type="org.apache.catalina.UserDatabase"
- description="User database that can be updated and saved"
- factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
- pathname="conf/tomcat-users.xml"/>
- </GlobalNamingResources>
-
- <!-- A "Service" is a collection of one or more "Connectors" that share
- a single "Container" Note: A "Service" is not itself a "Container",
- so you may not define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/service.html
- -->
- <Service name="Catalina">
-
- <!--The connectors can use a shared executor, you can define one or more named thread pools-->
- <!--
- <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
- maxThreads="150" minSpareThreads="4"/>
- -->
-
- <!-- Define a SSL HTTP/1.1 Connector on port 8443
- This connector uses the JSSE configuration, when using APR, the
- connector should be using the OpenSSL style configuration
- described in the APR documentation -->
- <Connector port="${kms.http.port}" protocol="HTTP/1.1" SSLEnabled="true"
- maxThreads="${kms.max.threads}" scheme="https" secure="true"
- maxHttpHeaderSize="${kms.max.http.header.size}"
- clientAuth="false" sslEnabledProtocols="TLSv1,TLSv1.1,TLSv1.2,SSLv2Hello"
- keystoreFile="${kms.ssl.keystore.file}"
- keystorePass="_kms_ssl_keystore_pass_"/>
-
- <!-- Define an AJP 1.3 Connector on port 8009 -->
-
-
- <!-- An Engine represents the entry point (within Catalina) that processes
- every request. The Engine implementation for Tomcat stand alone
- analyzes the HTTP headers included with the request, and passes them
- on to the appropriate Host (virtual host).
- Documentation at /docs/config/engine.html -->
-
- <!-- You should set jvmRoute to support load-balancing via AJP ie :
- <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
- -->
- <Engine name="Catalina" defaultHost="localhost">
-
- <!--For clustering, please take a look at documentation at:
- /docs/cluster-howto.html (simple how to)
- /docs/config/cluster.html (reference documentation) -->
- <!--
- <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
- -->
-
- <!-- The request dumper valve dumps useful debugging information about
- the request and response data received and sent by Tomcat.
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.valves.RequestDumperValve"/>
- -->
-
- <!-- This Realm uses the UserDatabase configured in the global JNDI
- resources under the key "UserDatabase". Any edits
- that are performed against this UserDatabase are immediately
- available for use by the Realm. -->
- <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
- resourceName="UserDatabase"/>
-
- <!-- Define the default virtual host
- Note: XML Schema validation will not work with Xerces 2.2.
- -->
- <Host name="localhost" appBase="webapps"
- unpackWARs="true" autoDeploy="true"
- xmlValidation="false" xmlNamespaceAware="false">
-
- <!-- SingleSignOn valve, share authentication between web applications
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
- -->
-
- <!-- Access log processes all example.
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
- prefix="localhost_access_log." suffix=".txt" pattern="common" resolveHosts="false"/>
- -->
-
- </Host>
- </Engine>
- </Service>
-</Server>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml b/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml
deleted file mode 100644
index d081764..0000000
--- a/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml
+++ /dev/null
@@ -1,78 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
-
- <listener>
- <listener-class>org.apache.hadoop.crypto.key.kms.server.KMSWebApp</listener-class>
- </listener>
-
- <servlet>
- <servlet-name>webservices-driver</servlet-name>
- <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
- <init-param>
- <param-name>com.sun.jersey.config.property.packages</param-name>
- <param-value>org.apache.hadoop.crypto.key.kms.server</param-value>
- </init-param>
-
- <!-- Enables detailed Jersey request/response logging -->
- <!--
- <init-param>
- <param-name>com.sun.jersey.spi.container.ContainerRequestFilters</param-name>
- <param-value>com.sun.jersey.api.container.filter.LoggingFilter</param-value>
- </init-param>
- <init-param>
- <param-name>com.sun.jersey.spi.container.ContainerResponseFilters</param-name>
- <param-value>com.sun.jersey.api.container.filter.LoggingFilter</param-value>
- </init-param>
- -->
- <load-on-startup>1</load-on-startup>
- </servlet>
-
- <servlet>
- <servlet-name>jmx-servlet</servlet-name>
- <servlet-class>org.apache.hadoop.crypto.key.kms.server.KMSJMXServlet</servlet-class>
- </servlet>
-
- <servlet-mapping>
- <servlet-name>webservices-driver</servlet-name>
- <url-pattern>/*</url-pattern>
- </servlet-mapping>
-
- <servlet-mapping>
- <servlet-name>jmx-servlet</servlet-name>
- <url-pattern>/jmx</url-pattern>
- </servlet-mapping>
-
- <filter>
- <filter-name>authFilter</filter-name>
- <filter-class>org.apache.hadoop.crypto.key.kms.server.KMSAuthenticationFilter</filter-class>
- </filter>
-
- <filter>
- <filter-name>MDCFilter</filter-name>
- <filter-class>org.apache.hadoop.crypto.key.kms.server.KMSMDCFilter</filter-class>
- </filter>
-
- <filter-mapping>
- <filter-name>authFilter</filter-name>
- <url-pattern>/*</url-pattern>
- </filter-mapping>
-
- <filter-mapping>
- <filter-name>MDCFilter</filter-name>
- <url-pattern>/*</url-pattern>
- </filter-mapping>
-
-</web-app>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/site/configuration.xsl
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/configuration.xsl b/hadoop-common-project/hadoop-kms/src/site/configuration.xsl
new file mode 100644
index 0000000..8f2ae9b
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/site/configuration.xsl
@@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="html"/>
+ <xsl:template match="configuration">
+ <html>
+ <body>
+ <h2>Configuration Properties</h2>
+ <table border="1">
+ <tr>
+ <th>name</th>
+ <th>value</th>
+ <th>description</th>
+ </tr>
+ <xsl:for-each select="property">
+ <tr>
+ <td>
+ <a name="{name}">
+ <xsl:value-of select="name"/>
+ </a>
+ </td>
+ <td>
+ <xsl:value-of select="value"/>
+ </td>
+ <td>
+ <xsl:value-of select="description"/>
+ </td>
+ </tr>
+ </xsl:for-each>
+ </table>
+ </body>
+ </html>
+ </xsl:template>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
index 58c1fbd..810d568 100644
--- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
@@ -29,7 +29,7 @@ The client is a KeyProvider implementation interacts with the KMS using the KMS
KMS and its client have built-in security and they support HTTP SPNEGO Kerberos authentication and HTTPS secure transport.
-KMS is a Java web-application and it runs using a pre-configured Tomcat bundled with the Hadoop distribution.
+KMS is a Java Jetty web-application.
KMS Client Configuration
------------------------
@@ -51,6 +51,15 @@ The following is an example to configure HDFS NameNode as a KMS client in
KMS
---
+$H3 Start/Stop the KMS
+
+To start/stop KMS, use `hadoop --daemon start|stop kms`. For example:
+
+ hadoop-${project.version} $ hadoop --daemon start kms
+
+NOTE: The script `kms.sh` is deprecated. It is now just a wrapper of
+`hadoop kms`.
+
$H3 KMS Configuration
Configure the KMS backing KeyProvider properties in the `etc/hadoop/kms-site.xml` configuration file:
@@ -71,6 +80,15 @@ The password file is looked up in the Hadoop's configuration directory via the c
NOTE: You need to restart the KMS for the configuration changes to take effect.
+$H3 KMS HTTP Configuration
+
+KMS pre-configures the HTTP port to 9600.
+
+KMS supports the following HTTP [configuration properties](./kms-default.html)
+in `etc/hadoop/kms-site.xml`.
+
+NOTE: You need to restart the KMS for the configuration changes to take effect.
+
$H3 KMS Cache
KMS has two kinds of caching: a CachingKeyProvider for caching the encryption keys, and a KeyProvider for caching the EEKs.
@@ -180,36 +198,6 @@ The Aggregation interval is configured via the property :
<value>10000</value>
</property>
-$H3 Start/Stop the KMS
-
-To start/stop KMS use KMS's sbin/kms.sh script. For example:
-
- hadoop-${project.version} $ sbin/kms.sh start
-
-NOTE: Invoking the script without any parameters list all possible parameters (start, stop, run, etc.). The `kms.sh` script is a wrapper for Tomcat's `catalina.sh` script that sets the environment variables and Java System properties required to run KMS.
-
-$H3 Embedded Tomcat Configuration
-
-To configure the embedded Tomcat go to the `share/hadoop/kms/tomcat/conf`.
-
-KMS pre-configures the HTTP and Admin ports in Tomcat's `server.xml` to 9600 and 9601.
-
-Tomcat logs are also preconfigured to go to Hadoop's `logs/` directory.
-
-The following environment variables (which can be set in KMS's `etc/hadoop/kms-env.sh` script) can be used to alter those values:
-
-* KMS_HTTP_PORT
-* KMS_ADMIN_PORT
-* KMS_MAX_THREADS
-* KMS_MAX_HTTP_HEADER_SIZE
-* KMS_LOGNOTE: You need to restart the KMS for the configuration changes to take effect.
-
-$H3 Loading native libraries
-
-The following environment variable (which can be set in KMS's `etc/hadoop/kms-env.sh` script) can be used to specify the location of any required native libraries. For eg. Tomact native Apache Portable Runtime (APR) libraries:
-
-* JAVA_LIBRARY_PATH
-
$H3 KMS Security Configuration
$H4 Enabling Kerberos HTTP SPNEGO Authentication
@@ -279,20 +267,52 @@ If `users`, `groups` or `hosts` has a `*`, it means there are no restrictions fo
$H4 KMS over HTTPS (SSL)
-To configure KMS to work over HTTPS the following 2 properties must be set in the `etc/hadoop/kms_env.sh` script (shown with default values):
+Enable SSL in `etc/hadoop/kms-site.xml`:
-* KMS_SSL_KEYSTORE_FILE=$HOME/.keystore
-* KMS_SSL_KEYSTORE_PASS=password
+```xml
+ <property>
+ <name>hadoop.kms.ssl.enabled</name>
+ <value>true</value>
+ <description>
+ Whether SSL is enabled. Default is false, i.e. disabled.
+ </description>
+ </property>
+
+```
-In the KMS `tomcat/conf` directory, replace the `server.xml` file with the provided `ssl-server.xml` file.
+Configure `etc/hadoop/ssl-server.xml` with proper values, for example:
+
+```xml
+<property>
+ <name>ssl.server.keystore.location</name>
+ <value>${user.home}/.keystore</value>
+ <description>Keystore to be used. Must be specified.</description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.password</name>
+ <value></value>
+ <description>Must be specified.</description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.keypassword</name>
+ <value></value>
+ <description>Must be specified.</description>
+</property>
+```
You need to create an SSL certificate for the KMS. As the `kms` Unix user, using the Java `keytool` command to create the SSL certificate:
- $ keytool -genkey -alias tomcat -keyalg RSA
+ $ keytool -genkey -alias jetty -keyalg RSA
-You will be asked a series of questions in an interactive prompt. It will create the keystore file, which will be named **.keystore** and located in the `kms` user home directory.
+You will be asked a series of questions in an interactive prompt. It will
+create the keystore file, which will be named **.keystore** and located in the
+user's home directory.
-The password you enter for "keystore password" must match the value of the `KMS_SSL_KEYSTORE_PASS` environment variable set in the `kms-env.sh` script in the configuration directory.
+The password you enter for "keystore password" must match the value of the
+property `ssl.server.keystore.password` set in the `ssl-server.xml` in the
+configuration directory.
The answer to "What is your first and last name?" (i.e. "CN") must be the hostname of the machine where the KMS will be running.
@@ -1032,3 +1052,29 @@ $H4 Get Keys Metadata
},
...
]
+
+$H3 Deprecated Environment Variables
+
+The following environment variables are deprecated. Set the corresponding
+configuration properties instead.
+
+Environment Variable | Configuration Property | Configuration File
+-------------------------|------------------------------|--------------------
+KMS_HTTP_PORT | hadoop.kms.http.port | kms-site.xml
+KMS_MAX_HTTP_HEADER_SIZE | hadoop.http.max.request.header.size and hadoop.http.max.response.header.size | kms-site.xml
+KMS_MAX_THREADS | hadoop.http.max.threads | kms-site.xml
+KMS_SSL_ENABLED | hadoop.kms.ssl.enabled | kms-site.xml
+KMS_SSL_KEYSTORE_FILE | ssl.server.keystore.location | ssl-server.xml
+KMS_SSL_KEYSTORE_PASS | ssl.server.keystore.password | ssl-server.xml
+KMS_TEMP | hadoop.http.temp.dir | kms-site.xml
+
+$H3 Default HTTP Services
+
+Name | Description
+-------------------|------------------------------------
+/conf | Display configuration properties
+/jmx | Java JMX management interface
+/logLevel | Get or set log level per class
+/logs | Display log files
+/stacks | Display JVM stacks
+/static/index.html | The static home page
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d182949/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
index 8b181ad..f911c5b 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -17,83 +17,23 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
-import com.google.common.base.Preconditions;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.http.JettyUtils;
-import org.apache.hadoop.util.ThreadUtil;
-import org.eclipse.jetty.http.HttpVersion;
-import org.eclipse.jetty.server.ConnectionFactory;
-import org.eclipse.jetty.server.HttpConfiguration;
-import org.eclipse.jetty.server.HttpConnectionFactory;
-import org.eclipse.jetty.server.SecureRequestCustomizer;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.server.SslConnectionFactory;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.eclipse.jetty.webapp.WebAppContext;
-
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
+import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Writer;
-import java.io.IOException;
-import java.net.MalformedURLException;
import java.net.URL;
-import java.util.UUID;
-
-public class MiniKMS {
- private static Server createJettyServer(String keyStore, String password, int inPort) {
- try {
- boolean ssl = keyStore != null;
- String host = "localhost";
- Server server = new Server();
- ServerConnector conn = new ServerConnector(server);
- HttpConfiguration httpConfig = new HttpConfiguration();
- httpConfig.setRequestHeaderSize(JettyUtils.HEADER_SIZE);
- httpConfig.setResponseHeaderSize(JettyUtils.HEADER_SIZE);
- httpConfig.setSecureScheme("https");
- httpConfig.addCustomizer(new SecureRequestCustomizer());
- ConnectionFactory connFactory = new HttpConnectionFactory(httpConfig);
- conn.addConnectionFactory(connFactory);
- conn.setHost(host);
- conn.setPort(inPort);
- if (ssl) {
- SslContextFactory sslContextFactory = new SslContextFactory();
- sslContextFactory.setNeedClientAuth(false);
- sslContextFactory.setKeyStorePath(keyStore);
- sslContextFactory.setKeyStoreType("jks");
- sslContextFactory.setKeyStorePassword(password);
- conn.addFirstConnectionFactory(
- new SslConnectionFactory(sslContextFactory,
- HttpVersion.HTTP_1_1.asString()));
- }
- server.addConnector(conn);
- return server;
- } catch (Exception ex) {
- throw new RuntimeException("Could not start embedded servlet container, "
- + ex.getMessage(), ex);
- }
- }
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.util.ThreadUtil;
- private static URL getJettyURL(Server server) {
- boolean ssl = server.getConnectors()[0]
- .getConnectionFactory(SslConnectionFactory.class) != null;
- try {
- String scheme = (ssl) ? "https" : "http";
- return new URL(scheme + "://" +
- ((ServerConnector)server.getConnectors()[0]).getHost() + ":"
- + ((ServerConnector)server.getConnectors()[0]).getLocalPort());
- } catch (MalformedURLException ex) {
- throw new RuntimeException("It should never happen, " + ex.getMessage(),
- ex);
- }
- }
+public class MiniKMS {
public static class Builder {
private File kmsConfDir;
@@ -150,7 +90,7 @@ public class MiniKMS {
private String log4jConfFile;
private String keyStore;
private String keyStorePassword;
- private Server jetty;
+ private KMSWebServer jetty;
private int inPort;
private URL kmsURL;
@@ -178,7 +118,6 @@ public class MiniKMS {
}
public void start() throws Exception {
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, kmsConfDir);
File aclsFile = new File(kmsConfDir, "kms-acls.xml");
if (!aclsFile.exists()) {
@@ -202,35 +141,20 @@ public class MiniKMS {
writer.close();
}
System.setProperty("log4j.configuration", log4jConfFile);
- jetty = createJettyServer(keyStore, keyStorePassword, inPort);
-
- // we need to do a special handling for MiniKMS to work when in a dir and
- // when in a JAR in the classpath thanks to Jetty way of handling of webapps
- // when they are in the a DIR, WAR or JAR.
- URL webXmlUrl = cl.getResource("kms-webapp/WEB-INF/web.xml");
- if (webXmlUrl == null) {
- throw new RuntimeException(
- "Could not find kms-webapp/ dir in test classpath");
- }
- boolean webXmlInJar = webXmlUrl.getPath().contains(".jar!/");
- String webappPath;
- if (webXmlInJar) {
- File webInf = new File("target/" + UUID.randomUUID().toString() +
- "/kms-webapp/WEB-INF");
- webInf.mkdirs();
- new File(webInf, "web.xml").delete();
- copyResource("kms-webapp/WEB-INF/web.xml", new File(webInf, "web.xml"));
- webappPath = webInf.getParentFile().getAbsolutePath();
- } else {
- webappPath = cl.getResource("kms-webapp").getPath();
- }
- WebAppContext context = new WebAppContext(webappPath, "/kms");
- if (webXmlInJar) {
- context.setClassLoader(cl);
+
+ final Configuration conf = KMSConfiguration.getKMSConf();
+ conf.set(KMSConfiguration.HTTP_HOST_KEY, "localhost");
+ conf.setInt(KMSConfiguration.HTTP_PORT_KEY, inPort);
+ if (keyStore != null) {
+ conf.setBoolean(KMSConfiguration.SSL_ENABLED_KEY, true);
+ conf.set(SSLFactory.SSL_SERVER_KEYSTORE_LOCATION, keyStore);
+ conf.set(SSLFactory.SSL_SERVER_KEYSTORE_PASSWORD, keyStorePassword);
+ conf.set(SSLFactory.SSL_SERVER_KEYSTORE_TYPE, "jks");
}
- jetty.setHandler(context);
+
+ jetty = new KMSWebServer(conf);
jetty.start();
- kmsURL = new URL(getJettyURL(jetty), "kms");
+ kmsURL = jetty.getKMSUrl();
}
public URL getKMSUrl() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-6054. TimelineServer fails to
start when some LevelDb state files are missing. Contributed by Ravi Prakash.
Posted by st...@apache.org.
YARN-6054. TimelineServer fails to start when some LevelDb state files are missing. Contributed by Ravi Prakash.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c431a69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c431a69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c431a69
Branch: refs/heads/HADOOP-13345
Commit: 4c431a694059e40e78365b02a1497a6c7e479a70
Parents: 41db07d
Author: Naganarasimha <na...@apache.org>
Authored: Tue Jan 10 15:54:16 2017 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Tue Jan 10 15:54:16 2017 +0530
----------------------------------------------------------------------
.../server/timeline/LeveldbTimelineStore.java | 33 ++++++++++++++++-
.../timeline/TestLeveldbTimelineStore.java | 39 ++++++++++++++++++++
2 files changed, 70 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c431a69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
index 8cfa0c7..ffe0413 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.timeline;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.collections.map.LRUMap;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -33,6 +34,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.timeline.*;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
@@ -123,6 +125,11 @@ public class LeveldbTimelineStore extends AbstractService
@VisibleForTesting
static final String FILENAME = "leveldb-timeline-store.ldb";
+ @VisibleForTesting
+ //Extension to FILENAME where backup will be stored in case we need to
+ //call LevelDb recovery
+ static final String BACKUP_EXT = ".backup-";
+
private static final byte[] START_TIME_LOOKUP_PREFIX = "k".getBytes(Charset.forName("UTF-8"));
private static final byte[] ENTITY_ENTRY_PREFIX = "e".getBytes(Charset.forName("UTF-8"));
private static final byte[] INDEXED_ENTRY_PREFIX = "i".getBytes(Charset.forName("UTF-8"));
@@ -175,6 +182,13 @@ public class LeveldbTimelineStore extends AbstractService
super(LeveldbTimelineStore.class.getName());
}
+ private JniDBFactory factory;
+
+ @VisibleForTesting
+ void setFactory(JniDBFactory fact) {
+ this.factory = fact;
+ }
+
@Override
@SuppressWarnings("unchecked")
protected void serviceInit(Configuration conf) throws Exception {
@@ -209,7 +223,10 @@ public class LeveldbTimelineStore extends AbstractService
options.cacheSize(conf.getLong(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE));
- JniDBFactory factory = new JniDBFactory();
+ if(factory == null) {
+ factory = new JniDBFactory();
+ }
+
Path dbPath = new Path(
conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH), FILENAME);
FileSystem localFS = null;
@@ -226,7 +243,19 @@ public class LeveldbTimelineStore extends AbstractService
IOUtils.cleanup(LOG, localFS);
}
LOG.info("Using leveldb path " + dbPath);
- db = factory.open(new File(dbPath.toString()), options);
+ try {
+ db = factory.open(new File(dbPath.toString()), options);
+ } catch (IOException ioe) {
+ File dbFile = new File(dbPath.toString());
+ File backupPath = new File(
+ dbPath.toString() + BACKUP_EXT + Time.monotonicNow());
+ LOG.warn("Incurred exception while loading LevelDb database. Backing " +
+ "up at "+ backupPath, ioe);
+ FileUtils.copyDirectory(dbFile, backupPath);
+ LOG.warn("Going to try repair");
+ factory.repair(dbFile, options);
+ db = factory.open(dbFile, options);
+ }
checkVersion();
startTimeWriteCache =
Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c431a69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
index 121e9f3..0c292d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
@@ -22,12 +22,14 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.io.File;
+import java.io.FileFilter;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -44,11 +46,14 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelineP
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
+import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.DBException;
+import org.iq80.leveldb.Options;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -464,4 +469,38 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
.iterator().next().size());
}
+ @Test
+ /**
+ * Test that LevelDb repair is attempted at least once during
+ * serviceInit for LeveldbTimelineStore in case open fails the
+ * first time.
+ */
+ public void testLevelDbRepair() throws IOException {
+ LeveldbTimelineStore store = new LeveldbTimelineStore();
+
+ JniDBFactory factory = Mockito.mock(JniDBFactory.class);
+ Mockito.when(
+ factory.open(Mockito.any(File.class), Mockito.any(Options.class)))
+ .thenThrow(new IOException()).thenCallRealMethod();
+ store.setFactory(factory);
+
+ //Create the LevelDb in a different location
+ File path = new File("target", this.getClass().getSimpleName() +
+ "-tmpDir1").getAbsoluteFile();
+ Configuration conf = new Configuration(this.config);
+ conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH,
+ path.getAbsolutePath());
+
+ try {
+ store.init(conf);
+ Mockito.verify(factory, Mockito.times(1))
+ .repair(Mockito.any(File.class), Mockito.any(Options.class));
+ FileFilter fileFilter = new WildcardFileFilter(
+ "*" + LeveldbTimelineStore.BACKUP_EXT +"*");
+ Assert.assertTrue(path.listFiles(fileFilter).length > 0);
+ } finally {
+ store.close();
+ fsContext.delete(new Path(path.getAbsolutePath()), true);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-6022. Revert changes of
AbstractResourceRequest (Contributed by Wangda Tan via Daniel Templeton)
Posted by st...@apache.org.
YARN-6022. Revert changes of AbstractResourceRequest (Contributed by Wangda Tan via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac16400e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac16400e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac16400e
Branch: refs/heads/HADOOP-13345
Commit: ac16400e1fb85a4186e5bf5bbc9cf204735ae74f
Parents: a59df15
Author: Daniel Templeton <te...@apache.org>
Authored: Sat Jan 7 08:23:06 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Sat Jan 7 08:24:30 2017 -0800
----------------------------------------------------------------------
.../api/records/AbstractResourceRequest.java | 55 --------------------
.../yarn/api/records/ResourceRequest.java | 19 ++++++-
.../api/records/UpdateContainerRequest.java | 18 ++++++-
.../server/scheduler/SchedulerRequestKey.java | 1 -
.../server/resourcemanager/RMAppManager.java | 2 +-
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../scheduler/AbstractYarnScheduler.java | 7 ++-
.../scheduler/SchedulerUtils.java | 18 ++++---
.../scheduler/YarnScheduler.java | 6 +--
.../scheduler/fair/FairScheduler.java | 5 +-
10 files changed, 55 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
deleted file mode 100644
index 819a607..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * {@code AbstractResourceRequest} represents a generic resource request made
- * by an application to the {@code ResourceManager}.
- * <p>
- * It includes:
- * <ul>
- * <li>{@link Resource} capability required for each request.</li>
- * </ul>
- *
- * @see Resource
- */
-@Public
-@Unstable
-public abstract class AbstractResourceRequest {
-
- /**
- * Set the <code>Resource</code> capability of the request
- * @param capability <code>Resource</code> capability of the request
- */
- @Public
- @Stable
- public abstract void setCapability(Resource capability);
-
- /**
- * Get the <code>Resource</code> capability of the request.
- * @return <code>Resource</code> capability of the request
- */
- @Public
- @Stable
- public abstract Resource getCapability();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index ce85b21..5bedc87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -58,8 +58,7 @@ import org.apache.hadoop.yarn.util.Records;
*/
@Public
@Stable
-public abstract class ResourceRequest extends AbstractResourceRequest
- implements Comparable<ResourceRequest>{
+public abstract class ResourceRequest implements Comparable<ResourceRequest> {
@Public
@Stable
@@ -506,6 +505,22 @@ public abstract class ResourceRequest extends AbstractResourceRequest
public void setAllocationRequestId(long allocationRequestID) {
throw new UnsupportedOperationException();
}
+
+ /**
+ * Set the <code>Resource</code> capability of the request.
+ * @param capability <code>Resource</code> capability of the request
+ */
+ @Public
+ @Stable
+ public abstract void setCapability(Resource capability);
+
+ /**
+ * Get the <code>Resource</code> capability of the request.
+ * @return <code>Resource</code> capability of the request
+ */
+ @Public
+ @Stable
+ public abstract Resource getCapability();
@Override
public int hashCode() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
index 925a7979..2060370 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
@@ -62,7 +62,7 @@ import org.apache.hadoop.yarn.util.Records;
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public abstract class UpdateContainerRequest extends AbstractResourceRequest {
+public abstract class UpdateContainerRequest {
@InterfaceAudience.Public
@InterfaceStability.Unstable
@@ -143,6 +143,22 @@ public abstract class UpdateContainerRequest extends AbstractResourceRequest {
@InterfaceStability.Unstable
public abstract void setExecutionType(ExecutionType executionType);
+ /**
+ * Set the <code>Resource</code> capability of the request.
+ * @param capability <code>Resource</code> capability of the request
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Unstable
+ public abstract void setCapability(Resource capability);
+
+ /**
+ * Get the <code>Resource</code> capability of the request.
+ * @return <code>Resource</code> capability of the request
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Unstable
+ public abstract Resource getCapability();
+
@Override
public int hashCode() {
final int prime = 2153;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
index 36a9149..02539ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.scheduler;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 9dd544e..8232b88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -459,7 +459,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
throw e;
}
- scheduler.normalizeRequest(amReq);
+ amReq.setCapability(scheduler.getNormalizedResource(amReq.getCapability()));
return amReq;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index ebbeb0f..94bfd58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -337,7 +337,7 @@ public class RMServerUtils {
return false;
}
ResourceScheduler scheduler = rmContext.getScheduler();
- scheduler.normalizeRequest(request);
+ request.setCapability(scheduler.getNormalizedResource(request.getCapability()));
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index acfcde8..89f9ffa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -1044,8 +1043,8 @@ public abstract class AbstractYarnScheduler
}
@Override
- public void normalizeRequest(AbstractResourceRequest ask) {
- SchedulerUtils.normalizeRequest(ask,
+ public Resource getNormalizedResource(Resource requestedResource) {
+ return SchedulerUtils.getNormalizedResource(requestedResource,
getResourceCalculator(),
getMinimumResourceCapability(),
getMaximumResourceCapability(),
@@ -1059,7 +1058,7 @@ public abstract class AbstractYarnScheduler
*/
protected void normalizeRequests(List<ResourceRequest> asks) {
for (ResourceRequest ask: asks) {
- normalizeRequest(ask);
+ ask.setCapability(getNormalizedResource(ask.getCapability()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 5360665..148f023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.io.IOException;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -33,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
@@ -134,29 +134,33 @@ public class SchedulerUtils {
* Utility method to normalize a resource request, by insuring that the
* requested memory is a multiple of minMemory and is not zero.
*/
+ @VisibleForTesting
public static void normalizeRequest(
ResourceRequest ask,
ResourceCalculator resourceCalculator,
Resource minimumResource,
Resource maximumResource) {
- normalizeRequest(ask, resourceCalculator,
- minimumResource, maximumResource, minimumResource);
+ ask.setCapability(
+ getNormalizedResource(ask.getCapability(), resourceCalculator,
+ minimumResource, maximumResource, minimumResource));
}
/**
* Utility method to normalize a resource request, by insuring that the
* requested memory is a multiple of increment resource and is not zero.
+ *
+ * @return normalized resource
*/
- public static void normalizeRequest(
- AbstractResourceRequest ask,
+ public static Resource getNormalizedResource(
+ Resource ask,
ResourceCalculator resourceCalculator,
Resource minimumResource,
Resource maximumResource,
Resource incrementResource) {
Resource normalized = Resources.normalize(
- resourceCalculator, ask.getCapability(), minimumResource,
+ resourceCalculator, ask, minimumResource,
maximumResource, incrementResource);
- ask.setCapability(normalized);
+ return normalized;
}
private static void normalizeNodeLabelExpressionInRequest(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 3185dc1..5579efd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -381,7 +380,8 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
/**
* Normalize a resource request.
*
- * @param request the resource request to be normalized
+ * @param requestedResource the resource to be normalized
+ * @return the normalized resource
*/
- void normalizeRequest(AbstractResourceRequest request);
+ Resource getNormalizedResource(Resource requestedResource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 78dfe61..6bbccc6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -799,8 +798,8 @@ public class FairScheduler extends
}
@Override
- public void normalizeRequest(AbstractResourceRequest ask) {
- SchedulerUtils.normalizeRequest(ask,
+ public Resource getNormalizedResource(Resource requestedResource) {
+ return SchedulerUtils.getNormalizedResource(requestedResource,
DOMINANT_RESOURCE_CALCULATOR,
minimumAllocation,
getMaximumResourceCapability(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDFS-9935. Remove LEASE_{SOFTLIMIT,
HARDLIMIT}_PERIOD and unused import from HdfsServerConstants. Contributed
Yiqun Lin.
Posted by st...@apache.org.
HDFS-9935. Remove LEASE_{SOFTLIMIT,HARDLIMIT}_PERIOD and unused import from HdfsServerConstants. Contributed Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5ec1e31
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5ec1e31
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5ec1e31
Branch: refs/heads/HADOOP-13345
Commit: a5ec1e31a3bed3701853073d3044ed2af8d9cedd
Parents: be529da
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Wed Jan 11 19:34:01 2017 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Wed Jan 11 19:34:01 2017 +0530
----------------------------------------------------------------------
.../hadoop/hdfs/server/common/HdfsServerConstants.java | 8 --------
1 file changed, 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5ec1e31/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index c9a46f0..c3098f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -25,7 +25,6 @@ import java.util.regex.Pattern;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -44,13 +43,6 @@ import org.apache.hadoop.util.StringUtils;
public interface HdfsServerConstants {
int MIN_BLOCKS_FOR_WRITE = 1;
- /**
- * Please see {@link HdfsConstants#LEASE_SOFTLIMIT_PERIOD} and
- * {@link HdfsConstants#LEASE_HARDLIMIT_PERIOD} for more information.
- */
- long LEASE_SOFTLIMIT_PERIOD = HdfsConstants.LEASE_SOFTLIMIT_PERIOD;
- long LEASE_HARDLIMIT_PERIOD = HdfsConstants.LEASE_HARDLIMIT_PERIOD;
-
long LEASE_RECOVER_PERIOD = 10 * 1000; // in ms
// We need to limit the length and depth of a path in the filesystem.
// HADOOP-438
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDFS-11072. Add ability to unset
and change directory EC policy. Contributed by Sammi Chen.
Posted by st...@apache.org.
HDFS-11072. Add ability to unset and change directory EC policy. Contributed by Sammi Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6923165
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6923165
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6923165
Branch: refs/heads/HADOOP-13345
Commit: e69231658dc4a79da936e6856017b5c4f6124ecb
Parents: 4046794
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Jan 10 11:32:45 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Jan 10 11:32:48 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 16 +-
.../hadoop/hdfs/DistributedFileSystem.java | 29 ++
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 10 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 7 +
.../ClientNamenodeProtocolTranslatorPB.java | 15 +
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +
.../src/main/proto/erasurecoding.proto | 7 +
...tNamenodeProtocolServerSideTranslatorPB.java | 14 +
.../server/namenode/FSDirErasureCodingOp.java | 115 +++++-
.../hdfs/server/namenode/FSNamesystem.java | 36 ++
.../hdfs/server/namenode/NameNodeRpcServer.java | 16 +
.../hdfs/tools/erasurecode/ECCommand.java | 34 ++
.../src/site/markdown/HDFSErasureCoding.md | 5 +
.../hadoop/hdfs/TestErasureCodingPolicies.java | 5 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 366 +++++++++++++++++++
.../test/resources/testErasureCodingConf.xml | 64 +++-
16 files changed, 725 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 0128b07..7b6a4e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2620,7 +2620,21 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
} catch (RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
SafeModeException.class,
- UnresolvedPathException.class);
+ UnresolvedPathException.class,
+ FileNotFoundException.class);
+ }
+ }
+
+ public void unsetErasureCodingPolicy(String src) throws IOException {
+ checkOpen();
+ try (TraceScope ignored =
+ newPathTraceScope("unsetErasureCodingPolicy", src)) {
+ namenode.unsetErasureCodingPolicy(src);
+ } catch (RemoteException re) {
+ throw re.unwrapRemoteException(AccessControlException.class,
+ SafeModeException.class,
+ UnresolvedPathException.class,
+ FileNotFoundException.class);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 4f97896..e9475d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2468,6 +2468,35 @@ public class DistributedFileSystem extends FileSystem {
}
/**
+ * Unset the erasure coding policy from the source path.
+ *
+ * @param path The directory to unset the policy
+ * @throws IOException
+ */
+ public void unsetErasureCodingPolicy(final Path path) throws IOException {
+ Path absF = fixRelativePart(path);
+ new FileSystemLinkResolver<Void>() {
+ @Override
+ public Void doCall(final Path p) throws IOException {
+ dfs.unsetErasureCodingPolicy(getPathName(p));
+ return null;
+ }
+
+ @Override
+ public Void next(final FileSystem fs, final Path p) throws IOException {
+ if (fs instanceof DistributedFileSystem) {
+ DistributedFileSystem myDfs = (DistributedFileSystem) fs;
+ myDfs.unsetErasureCodingPolicy(p);
+ return null;
+ }
+ throw new UnsupportedOperationException(
+ "Cannot unsetErasureCodingPolicy through a symlink to a "
+ + "non-DistributedFileSystem: " + path + " -> " + p);
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
* Get the root directory of Trash for a path in HDFS.
* 1. File in encryption zone returns /ez1/.Trash/username
* 2. File not in encryption zone, or encountered exception when checking
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index 5508064..a192fa8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -496,6 +496,16 @@ public class HdfsAdmin {
return dfs.getClient().getErasureCodingPolicies();
}
+ /**
+ * Unset erasure coding policy from the directory.
+ *
+ * @param path The source path referring to a directory.
+ * @throws IOException
+ */
+ public void unsetErasureCodingPolicy(final Path path) throws IOException {
+ dfs.unsetErasureCodingPolicy(path);
+ }
+
private void provisionEZTrash(Path path) throws IOException {
// make sure the path is an EZ
EncryptionZone ez = dfs.getEZForPath(path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 15bbe51..407621b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1535,6 +1535,13 @@ public interface ClientProtocol {
ErasureCodingPolicy getErasureCodingPolicy(String src) throws IOException;
/**
+ * Unset erasure coding policy from a specified path.
+ * @param src The path to unset policy.
+ */
+ @AtMostOnce
+ void unsetErasureCodingPolicy(String src) throws IOException;
+
+ /**
* Get {@link QuotaUsage} rooted at the specified directory.
* @param path The string representation of the path
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index f73abfd..de474b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -173,6 +173,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodin
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.UnsetErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto;
@@ -1475,6 +1476,20 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
+ public void unsetErasureCodingPolicy(String src)
+ throws IOException {
+ final UnsetErasureCodingPolicyRequestProto.Builder builder =
+ ErasureCodingProtos.UnsetErasureCodingPolicyRequestProto.newBuilder();
+ builder.setSrc(src);
+ UnsetErasureCodingPolicyRequestProto req = builder.build();
+ try {
+ rpcProxy.unsetErasureCodingPolicy(null, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
throws IOException {
SetXAttrRequestProto req = SetXAttrRequestProto.newBuilder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 54bed32..ff4db03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -898,6 +898,8 @@ service ClientNamenodeProtocol {
returns(GetEZForPathResponseProto);
rpc setErasureCodingPolicy(SetErasureCodingPolicyRequestProto)
returns(SetErasureCodingPolicyResponseProto);
+ rpc unsetErasureCodingPolicy(UnsetErasureCodingPolicyRequestProto)
+ returns(UnsetErasureCodingPolicyResponseProto);
rpc getCurrentEditLogTxid(GetCurrentEditLogTxidRequestProto)
returns(GetCurrentEditLogTxidResponseProto);
rpc getEditsFromTxid(GetEditsFromTxidRequestProto)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
index 4bb44fb..7ea8d32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
@@ -46,6 +46,13 @@ message GetErasureCodingPolicyResponseProto {
optional ErasureCodingPolicyProto ecPolicy = 1;
}
+message UnsetErasureCodingPolicyRequestProto {
+ required string src = 1;
+}
+
+message UnsetErasureCodingPolicyResponseProto {
+}
+
/**
* Block erasure coding reconstruction info
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 3974956..1aa15c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -215,6 +215,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodin
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.UnsetErasureCodingPolicyRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.UnsetErasureCodingPolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockStoragePolicyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto;
@@ -1450,6 +1452,18 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
@Override
+ public UnsetErasureCodingPolicyResponseProto unsetErasureCodingPolicy(
+ RpcController controller, UnsetErasureCodingPolicyRequestProto req)
+ throws ServiceException {
+ try {
+ server.unsetErasureCodingPolicy(req.getSrc());
+ return UnsetErasureCodingPolicyResponseProto.newBuilder().build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
public SetXAttrResponseProto setXAttr(RpcController controller,
SetXAttrRequestProto req) throws ServiceException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 1f3b135..0ab8c89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -80,7 +80,7 @@ final class FSDirErasureCodingOp {
try {
iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
src = iip.getPath();
- xAttrs = createErasureCodingPolicyXAttr(fsn, iip, ecPolicy);
+ xAttrs = setErasureCodingPolicyXAttr(fsn, iip, ecPolicy);
} finally {
fsd.writeUnlock();
}
@@ -88,21 +88,20 @@ final class FSDirErasureCodingOp {
return fsd.getAuditFileInfo(iip);
}
- static List<XAttr> createErasureCodingPolicyXAttr(final FSNamesystem fsn,
+ static List<XAttr> setErasureCodingPolicyXAttr(final FSNamesystem fsn,
final INodesInPath srcIIP, ErasureCodingPolicy ecPolicy) throws IOException {
FSDirectory fsd = fsn.getFSDirectory();
assert fsd.hasWriteLock();
Preconditions.checkNotNull(srcIIP, "INodes cannot be null");
String src = srcIIP.getPath();
- if (srcIIP.getLastINode() != null &&
- !srcIIP.getLastINode().isDirectory()) {
+ final INode inode = srcIIP.getLastINode();
+ if (inode == null) {
+ throw new FileNotFoundException("Path not found: " + srcIIP.getPath());
+ }
+ if (!inode.isDirectory()) {
throw new IOException("Attempt to set an erasure coding policy " +
"for a file " + src);
}
- if (getErasureCodingPolicyForPath(fsn, srcIIP) != null) {
- throw new IOException("Directory " + src + " already has an " +
- "erasure coding policy.");
- }
// System default erasure coding policy will be used since no specified.
if (ecPolicy == null) {
@@ -124,7 +123,7 @@ final class FSDirErasureCodingOp {
ecPolicyNames.add(activePolicy.getName());
}
throw new HadoopIllegalArgumentException("Policy [ " +
- ecPolicy.getName()+ " ] does not match any of the " +
+ ecPolicy.getName() + " ] does not match any of the " +
"supported policies. Please select any one of " + ecPolicyNames);
}
}
@@ -140,10 +139,76 @@ final class FSDirErasureCodingOp {
} finally {
IOUtils.closeStream(dOut);
}
+ // check whether the directory already has an erasure coding policy
+ // directly on itself.
+ final Boolean hasEcXAttr =
+ getErasureCodingPolicyXAttrForINode(fsn, inode) == null ? false : true;
final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
xattrs.add(ecXAttr);
- FSDirXAttrOp.unprotectedSetXAttrs(fsd, srcIIP, xattrs,
- EnumSet.of(XAttrSetFlag.CREATE));
+ final EnumSet<XAttrSetFlag> flag = hasEcXAttr ?
+ EnumSet.of(XAttrSetFlag.REPLACE) : EnumSet.of(XAttrSetFlag.CREATE);
+ FSDirXAttrOp.unprotectedSetXAttrs(fsd, srcIIP, xattrs, flag);
+ return xattrs;
+ }
+
+ /**
+ * Unset erasure coding policy from the given directory.
+ *
+ * @param fsn The namespace
+ * @param srcArg The path of the target directory.
+ * @param logRetryCache whether to record RPC ids in editlog for retry
+ * cache rebuilding
+ * @return {@link HdfsFileStatus}
+ * @throws IOException
+ */
+ static HdfsFileStatus unsetErasureCodingPolicy(final FSNamesystem fsn,
+ final String srcArg, final boolean logRetryCache) throws IOException {
+ assert fsn.hasWriteLock();
+
+ String src = srcArg;
+ FSPermissionChecker pc = fsn.getPermissionChecker();
+ FSDirectory fsd = fsn.getFSDirectory();
+ final INodesInPath iip;
+ List<XAttr> xAttrs;
+ fsd.writeLock();
+ try {
+ iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
+ src = iip.getPath();
+ xAttrs = removeErasureCodingPolicyXAttr(fsn, iip);
+ } finally {
+ fsd.writeUnlock();
+ }
+ if (xAttrs != null) {
+ fsn.getEditLog().logRemoveXAttrs(src, xAttrs, logRetryCache);
+ }
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ private static List<XAttr> removeErasureCodingPolicyXAttr(
+ final FSNamesystem fsn, final INodesInPath srcIIP) throws IOException {
+ FSDirectory fsd = fsn.getFSDirectory();
+ assert fsd.hasWriteLock();
+ Preconditions.checkNotNull(srcIIP, "INodes cannot be null");
+ String src = srcIIP.getPath();
+ final INode inode = srcIIP.getLastINode();
+ if (inode == null) {
+ throw new FileNotFoundException("Path not found: " + srcIIP.getPath());
+ }
+ if (!inode.isDirectory()) {
+ throw new IOException("Cannot unset an erasure coding policy " +
+ "on a file " + src);
+ }
+
+ // Check whether the directory has a specific erasure coding policy
+ // directly on itself.
+ final XAttr ecXAttr = getErasureCodingPolicyXAttrForINode(fsn, inode);
+ if (ecXAttr == null) {
+ return null;
+ }
+
+ final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
+ xattrs.add(ecXAttr);
+ FSDirXAttrOp.unprotectedRemoveXAttrs(fsd, srcIIP.getPath(), xattrs);
return xattrs;
}
@@ -271,4 +336,32 @@ final class FSDirErasureCodingOp {
}
return null;
}
+
+ private static XAttr getErasureCodingPolicyXAttrForINode(
+ FSNamesystem fsn, INode inode) throws IOException {
+ // INode can be null
+ if (inode == null) {
+ return null;
+ }
+ FSDirectory fsd = fsn.getFSDirectory();
+ fsd.readLock();
+ try {
+ // We don't allow setting EC policies on paths with a symlink. Thus
+ // if a symlink is encountered, the dir shouldn't have EC policy.
+ // TODO: properly support symlinks
+ if (inode.isSymlink()) {
+ return null;
+ }
+ final XAttrFeature xaf = inode.getXAttrFeature();
+ if (xaf != null) {
+ XAttr xattr = xaf.getXAttr(XATTR_ERASURECODING_POLICY);
+ if (xattr != null) {
+ return xattr;
+ }
+ }
+ } finally {
+ fsd.readUnlock();
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 90fb924..249324b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -6775,6 +6775,42 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
+ * Unset an erasure coding policy from the given path.
+ * @param srcArg The path of the target directory.
+ * @throws AccessControlException if the caller is not the superuser.
+ * @throws UnresolvedLinkException if the path can't be resolved.
+ * @throws SafeModeException if the Namenode is in safe mode.
+ */
+ void unsetErasureCodingPolicy(final String srcArg,
+ final boolean logRetryCache) throws IOException,
+ UnresolvedLinkException, SafeModeException, AccessControlException {
+ final String operationName = "unsetErasureCodingPolicy";
+ checkSuperuserPrivilege();
+ checkOperation(OperationCategory.WRITE);
+ HdfsFileStatus resultingStat = null;
+ boolean success = false;
+ writeLock();
+ try {
+ checkOperation(OperationCategory.WRITE);
+ checkNameNodeSafeMode("Cannot unset erasure coding policy on " + srcArg);
+ resultingStat = FSDirErasureCodingOp.unsetErasureCodingPolicy(this,
+ srcArg, logRetryCache);
+ success = true;
+ } catch (AccessControlException ace) {
+ logAuditEvent(success, operationName, srcArg, null,
+ resultingStat);
+ throw ace;
+ } finally {
+ writeUnlock(operationName);
+ if (success) {
+ getEditLog().logSync();
+ }
+ }
+ logAuditEvent(success, operationName, srcArg, null,
+ resultingStat);
+ }
+
+ /**
* Get the erasure coding policy information for specified path
*/
ErasureCodingPolicy getErasureCodingPolicy(String src)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 735b2c0..6a3f966 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -2231,6 +2231,22 @@ public class NameNodeRpcServer implements NamenodeProtocols {
return namesystem.getErasureCodingPolicy(src);
}
+ @Override // ClientProtocol
+ public void unsetErasureCodingPolicy(String src) throws IOException {
+ checkNNStartup();
+ final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+ if (cacheEntry != null && cacheEntry.isSuccess()) {
+ return;
+ }
+ boolean success = false;
+ try {
+ namesystem.unsetErasureCodingPolicy(src, cacheEntry != null);
+ success = true;
+ } finally {
+ RetryCache.setState(cacheEntry, success);
+ }
+ }
+
@Override // ReconfigurationProtocol
public void startReconfiguration() throws IOException {
checkNNStartup();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
index 978fe47..fc732e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
@@ -47,6 +47,8 @@ public abstract class ECCommand extends Command {
factory.addClass(SetECPolicyCommand.class, "-" + SetECPolicyCommand.NAME);
factory.addClass(GetECPolicyCommand.class, "-"
+ GetECPolicyCommand.NAME);
+ factory.addClass(UnsetECPolicyCommand.class, "-"
+ + UnsetECPolicyCommand.NAME);
factory.addClass(ListPolicies.class, "-" + ListPolicies.NAME);
}
@@ -211,4 +213,36 @@ public abstract class ECCommand extends Command {
out.println(sb.toString());
}
}
+
+ /**
+ * Unset the erasure coding policy from a directory.
+ */
+ static class UnsetECPolicyCommand extends ECCommand {
+ public static final String NAME = "unsetPolicy";
+ public static final String USAGE = "<path>";
+ public static final String DESCRIPTION =
+ "Unset erasure coding policy from a directory\n";
+
+ @Override
+ protected void processOptions(LinkedList<String> args) throws IOException {
+ if (args.isEmpty()) {
+ throw new HadoopIllegalArgumentException("<path> is missing");
+ }
+ if (args.size() > 1) {
+ throw new HadoopIllegalArgumentException("Too many arguments");
+ }
+ }
+
+ @Override
+ protected void processPath(PathData item) throws IOException {
+ super.processPath(item);
+ DistributedFileSystem dfs = (DistributedFileSystem) item.fs;
+ try {
+ dfs.unsetErasureCodingPolicy(item.path);
+ } catch (IOException e) {
+ throw new IOException("Unable to unset EC policy from directory "
+ + item.path + ". " + e.getMessage());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 517469d..d5dbd0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -129,6 +129,7 @@ Deployment
hdfs erasurecode [generic options]
[-setPolicy [-p <policyName>] <path>]
[-getPolicy <path>]
+ [-unsetPolicy <path>]
[-listPolicies]
[-usage [cmd ...]]
[-help [cmd ...]]
@@ -147,6 +148,10 @@ Below are the details about each command.
Get details of the ErasureCoding policy of a file or directory at the specified path.
+ * `[-unsetPolicy <path>]`
+
+ Unset an ErasureCoding policy from a directory at the specified path when previously user sets the ErasureCoding policy on this directory via "setPolicy" command. If the directory inherits the ErasureCoding policy from its parent group, "unsetPolicy" command on this directory will not have any effect. Unset ErasureCoding policy on a directory which doesn't have ErasureCoding policy will not return an error.
+
* `[-listPolicies]`
Lists all supported ErasureCoding policies. These names are suitable for use with the `setPolicy` command.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index b7c3ed8..27fbf18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -155,7 +155,7 @@ public class TestErasureCodingPolicies {
INode newInode = namesystem.getFSDirectory().getINode(newFile.toString());
assertTrue(newInode.asFile().isStriped());
- /* Verify that nested EC policies not supported */
+ /* Verify that nested EC policies are supported */
final Path dir1 = new Path("/dir1");
final Path dir2 = new Path(dir1, "dir2");
fs.mkdir(dir1, FsPermission.getDirDefault());
@@ -163,9 +163,8 @@ public class TestErasureCodingPolicies {
fs.mkdir(dir2, FsPermission.getDirDefault());
try {
fs.getClient().setErasureCodingPolicy(dir2.toString(), null);
- fail("Nested erasure coding policies");
} catch (IOException e) {
- assertExceptionContains("already has an erasure coding policy", e);
+ fail("Nested erasure coding policies are supported");
}
/* Verify that EC policy cannot be set on a file */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
new file mode 100644
index 0000000..1a4086e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -0,0 +1,366 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.junit.Assert;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
+import static org.junit.Assert.fail;
+
+/**
+ * Test unset and change directory's erasure coding policy.
+ */
+public class TestUnsetAndChangeDirectoryEcPolicy {
+
+ public static final Log LOG =
+ LogFactory.getLog(TestUnsetAndChangeDirectoryEcPolicy.class);
+
+ private MiniDFSCluster cluster;
+ private Configuration conf = new Configuration();
+ private DistributedFileSystem fs;
+ private ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager
+ .getSystemDefaultPolicy();
+ private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+ private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int stripsPerBlock = 2;
+ private final int blockSize = stripsPerBlock * cellSize;
+ private final int blockGroupSize = dataBlocks * blockSize;
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300000);
+
+ @Before
+ public void setup() throws IOException {
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
+ if (ErasureCodeNative.isNativeCodeLoaded()) {
+ conf.set(
+ CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ NativeRSRawErasureCoderFactory.class.getCanonicalName());
+ }
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
+ dataBlocks + parityBlocks).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ /*
+ * Test unset EC policy on directory.
+ */
+ @Test
+ public void testUnsetEcPolicy() throws Exception {
+ final int numBlocks = 1;
+ final int fileLen = blockGroupSize * numBlocks;
+ final Path dirPath = new Path("/striped");
+ final Path ecFilePath = new Path(dirPath, "ec_file");
+ final Path replicateFilePath = new Path(dirPath, "3x_file");
+
+ fs.mkdirs(dirPath);
+ // Test unset a directory which has no EC policy
+ fs.unsetErasureCodingPolicy(dirPath);
+ // Set EC policy on directory
+ fs.setErasureCodingPolicy(dirPath, ecPolicy);
+
+ DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
+ fs.unsetErasureCodingPolicy(dirPath);
+ DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 1, 0L);
+
+ // ec_file should has EC policy
+ ErasureCodingPolicy tempEcPolicy =
+ fs.getErasureCodingPolicy(ecFilePath);
+ Assert.assertTrue("Erasure coding policy mismatch!",
+ tempEcPolicy.getName().equals(ecPolicy.getName()));
+
+ // rep_file should not have EC policy
+ tempEcPolicy = fs.getErasureCodingPolicy(replicateFilePath);
+ Assert.assertNull("Replicate file should not have erasure coding policy!",
+ tempEcPolicy);
+
+ // Directory should not return erasure coding policy
+ tempEcPolicy = fs.getErasureCodingPolicy(dirPath);
+ Assert.assertNull("Directory should no have erasure coding policy set!",
+ tempEcPolicy);
+
+ fs.delete(dirPath, true);
+ }
+
+ /*
+ * Test nested directory with different EC policy.
+ */
+ @Test
+ public void testNestedEcPolicy() throws Exception {
+ final int numBlocks = 1;
+ final int fileLen = blockGroupSize * numBlocks;
+ final Path parentDir = new Path("/ec-6-3");
+ final Path childDir = new Path("/ec-6-3/ec-3-2");
+ final Path ec63FilePath = new Path(childDir, "ec_6_3_file");
+ final Path ec32FilePath = new Path(childDir, "ec_3_2_file");
+ final Path ec63FilePath2 = new Path(childDir, "ec_6_3_file_2");
+ final ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager
+ .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID);
+
+ fs.mkdirs(parentDir);
+ fs.setErasureCodingPolicy(parentDir, ecPolicy);
+ fs.mkdirs(childDir);
+ // Create RS(6,3) EC policy file
+ DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L);
+ // Set RS(3,2) EC policy on child directory
+ fs.setErasureCodingPolicy(childDir, ec32Policy);
+ // Create RS(3,2) EC policy file
+ DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L);
+
+ // Start to check
+ // ec_6_3_file should has RS-6-3 EC policy
+ ErasureCodingPolicy tempEcPolicy =
+ fs.getErasureCodingPolicy(ec63FilePath);
+ Assert.assertTrue("Erasure coding policy mismatch!",
+ tempEcPolicy.getName().equals(ecPolicy.getName()));
+
+ // ec_3_2_file should have RS-3-2 policy
+ tempEcPolicy = fs.getErasureCodingPolicy(ec32FilePath);
+ Assert.assertTrue("Erasure coding policy mismatch!",
+ tempEcPolicy.getName().equals(ec32Policy.getName()));
+
+ // Child directory should have RS-3-2 policy
+ tempEcPolicy = fs.getErasureCodingPolicy(childDir);
+ Assert.assertTrue(
+ "Directory should have erasure coding policy set!",
+ tempEcPolicy.getName().equals(ec32Policy.getName()));
+
+ // Unset EC policy on child directory
+ fs.unsetErasureCodingPolicy(childDir);
+ DFSTestUtil.createFile(fs, ec63FilePath2, fileLen, (short) 1, 0L);
+
+ // ec_6_3_file_2 should have RS-6-3 policy
+ tempEcPolicy = fs.getErasureCodingPolicy(ec63FilePath2);
+ Assert.assertTrue("Erasure coding policy mismatch!",
+ tempEcPolicy.getName().equals(ecPolicy.getName()));
+
+ // Child directory should have RS-6-3 policy now
+ tempEcPolicy = fs.getErasureCodingPolicy(childDir);
+ Assert.assertTrue(
+ "Directory should have erasure coding policy set!",
+ tempEcPolicy.getName().equals(ecPolicy.getName()));
+
+ fs.delete(parentDir, true);
+ }
+
+
+ /*
+ * Test unset EC policy on root directory.
+ */
+ @Test
+ public void testUnsetRootDirEcPolicy() throws Exception {
+ final int numBlocks = 1;
+ final int fileLen = blockGroupSize * numBlocks;
+ final Path rootPath = new Path("/");
+ final Path ecFilePath = new Path(rootPath, "ec_file");
+ final Path replicateFilePath = new Path(rootPath, "rep_file");
+
+ // Test unset root path which has no EC policy
+ fs.unsetErasureCodingPolicy(rootPath);
+ // Set EC policy on root path
+ fs.setErasureCodingPolicy(rootPath, ecPolicy);
+ DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
+ fs.unsetErasureCodingPolicy(rootPath);
+ DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 1, 0L);
+
+ // ec_file should has EC policy set
+ ErasureCodingPolicy tempEcPolicy =
+ fs.getErasureCodingPolicy(ecFilePath);
+ Assert.assertTrue("Erasure coding policy mismatch!",
+ tempEcPolicy.getName().equals(ecPolicy.getName()));
+
+ // rep_file should not have EC policy set
+ tempEcPolicy = fs.getErasureCodingPolicy(replicateFilePath);
+ Assert.assertNull("Replicate file should not have erasure coding policy!",
+ tempEcPolicy);
+
+ // Directory should not return erasure coding policy
+ tempEcPolicy = fs.getErasureCodingPolicy(rootPath);
+ Assert.assertNull("Directory should not have erasure coding policy set!",
+ tempEcPolicy);
+
+ fs.delete(rootPath, true);
+ }
+
+ /*
+ * Test change EC policy on root directory.
+ */
+ @Test
+ public void testChangeRootDirEcPolicy() throws Exception {
+ final int numBlocks = 1;
+ final int fileLen = blockGroupSize * numBlocks;
+ final Path rootPath = new Path("/");
+ final Path ec63FilePath = new Path(rootPath, "ec_6_3_file");
+ final Path ec32FilePath = new Path(rootPath, "ec_3_2_file");
+ final ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager
+ .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID);
+
+ fs.unsetErasureCodingPolicy(rootPath);
+ fs.setErasureCodingPolicy(rootPath, ecPolicy);
+ // Create RS(6,3) EC policy file
+ DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L);
+ // Change EC policy from RS(6,3) to RS(3,2)
+ fs.setErasureCodingPolicy(rootPath, ec32Policy);
+ DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L);
+
+ // start to check
+ // ec_6_3_file should has RS-6-3 ec policy set
+ ErasureCodingPolicy tempEcPolicy =
+ fs.getErasureCodingPolicy(ec63FilePath);
+ Assert.assertTrue("Erasure coding policy mismatch!",
+ tempEcPolicy.getName().equals(ecPolicy.getName()));
+
+ // ec_3_2_file should have RS-3-2 policy
+ tempEcPolicy = fs.getErasureCodingPolicy(ec32FilePath);
+ Assert.assertTrue("Erasure coding policy mismatch!",
+ tempEcPolicy.getName().equals(ec32Policy.getName()));
+
+ // Root directory should have RS-3-2 policy
+ tempEcPolicy = fs.getErasureCodingPolicy(rootPath);
+ Assert.assertTrue(
+ "Directory should have erasure coding policy!",
+ tempEcPolicy.getName().equals(ec32Policy.getName()));
+
+ fs.delete(rootPath, true);
+ }
+
+ /*
+ * Test different replica factor files.
+ */
+ @Test
+ public void testDifferentReplicaFactor() throws Exception {
+ final int numBlocks = 1;
+ final int fileLen = blockGroupSize * numBlocks;
+ final Path ecDirPath = new Path("/striped");
+ final Path ecFilePath = new Path(ecDirPath, "ec_file");
+ final Path replicateFilePath = new Path(ecDirPath, "rep_file");
+ final Path replicateFilePath2 = new Path(ecDirPath, "rep_file2");
+
+ fs.mkdirs(ecDirPath);
+ fs.setErasureCodingPolicy(ecDirPath, ecPolicy);
+ DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
+ fs.unsetErasureCodingPolicy(ecDirPath);
+ DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 3, 0L);
+ DFSTestUtil.createFile(fs, replicateFilePath2, fileLen, (short) 2, 0L);
+
+ // ec_file should has EC policy set
+ ErasureCodingPolicy tempEcPolicy =
+ fs.getErasureCodingPolicy(ecFilePath);
+ Assert.assertTrue("Erasure coding policy mismatch!",
+ tempEcPolicy.getName().equals(ecPolicy.getName()));
+
+ // rep_file should not have EC policy set
+ tempEcPolicy = fs.getErasureCodingPolicy(replicateFilePath);
+ Assert.assertNull("Replicate file should not have erasure coding policy!",
+ tempEcPolicy);
+ tempEcPolicy = fs.getErasureCodingPolicy(replicateFilePath2);
+ Assert.assertNull("Replicate file should not have erasure coding policy!",
+ tempEcPolicy);
+
+ // Directory should not return erasure coding policy
+ tempEcPolicy = fs.getErasureCodingPolicy(ecDirPath);
+ Assert.assertNull("Directory should not have erasure coding policy set!",
+ tempEcPolicy);
+
+ fs.delete(ecDirPath, true);
+ }
+
+
+ /*
+ * Test set and unset EC policy on directory doesn't exist.
+ */
+ @Test
+ public void testNonExistentDir() throws Exception {
+ final Path dirPath = new Path("/striped");
+
+ // Unset EC policy on non-existent directory
+ try {
+ fs.unsetErasureCodingPolicy(dirPath);
+ fail("FileNotFoundException should be thrown for a non-existent"
+ + " file path");
+ } catch (FileNotFoundException e) {
+ assertExceptionContains("Path not found: " + dirPath, e);
+ }
+
+ // Set EC policy on non-existent directory
+ try {
+ fs.setErasureCodingPolicy(dirPath, ecPolicy);
+ fail("FileNotFoundException should be thrown for a non-existent"
+ + " file path");
+ } catch (FileNotFoundException e) {
+ assertExceptionContains("Path not found: " + dirPath, e);
+ }
+ }
+
+ /*
+ * Test set and unset EC policy on file.
+ */
+ @Test
+ public void testEcPolicyOnFile() throws Exception {
+ final Path ecFilePath = new Path("/striped_file");
+ final int fileLen = blockGroupSize * 2;
+ DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
+
+ // Set EC policy on file
+ try {
+ fs.setErasureCodingPolicy(ecFilePath, ecPolicy);
+ fail("IOException should be thrown for setting EC policy on file");
+ } catch (IOException e) {
+ assertExceptionContains("Attempt to set an erasure coding policy " +
+ "for a file " + ecFilePath, e);
+ }
+
+ // Unset EC policy on file
+ try {
+ fs.unsetErasureCodingPolicy(ecFilePath);
+ fail("IOException should be thrown for unsetting EC policy on file");
+ } catch (IOException e) {
+ assertExceptionContains("Cannot unset an erasure coding policy on a file "
+ + ecFilePath, e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6923165/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index f8ee973..dd26b48 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -151,7 +151,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Directory /ecdir already has an erasure coding policy</expected-output>
+ <expected-output>EC policy set successfully at NAMENODE/ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -175,6 +175,68 @@
</test>
<test>
+ <description>unsetPolicy : unset policy and get</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /ecdir</command>
+ <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -unsetPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /ecdir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>is not erasure coded.</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>setPolicy : change different policy and get</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /ecdir</command>
+ <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-3-2-64k
+ /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /ecdir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-3-2-64k</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>unsetPolicy : unset inherited EC policy, has no effect</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /ecdir</command>
+ <command>-fs NAMENODE -mkdir /ecdir/child</command>
+ <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -unsetPolicy /ecdir/child</ec-admin-command>
+ <command>-fs NAMENODE -touchz /ecdir/child/ecfile</command>
+ <ec-admin-command>-fs NAMENODE -getPolicy /ecdir/child/ecfile</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /ecdir/child/ecfile</command>
+ <command>-fs NAMENODE -rmdir /ecdir/child</command>
+ <command>-fs NAMENODE -rmdir /ecdir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /noec</command>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-3955. Support for application
priority ACLs in queues of CapacityScheduler. (Sunil G via wangda)
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index a36db44..1348f51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerC
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -856,7 +857,10 @@ public class TestParentQueue {
TestUtils.spyHook);
YarnAuthorizationProvider authorizer =
YarnAuthorizationProvider.getInstance(conf);
- CapacitySchedulerQueueManager.setQueueAcls(authorizer, queues);
+ AppPriorityACLsManager appPriorityACLManager = new AppPriorityACLsManager(
+ conf);
+ CapacitySchedulerQueueManager.setQueueAcls(authorizer,
+ appPriorityACLManager, queues);
UserGroupInformation user = UserGroupInformation.getCurrentUser();
// Setup queue configs
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDFS-9483. Documentation does not
cover use of "swebhdfs" as URL scheme for SSL-secured WebHDFS. Contributed by
Surendra Singh Lilhore.
Posted by st...@apache.org.
HDFS-9483. Documentation does not cover use of "swebhdfs" as URL scheme for SSL-secured WebHDFS. Contributed by Surendra Singh Lilhore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c8f9e13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c8f9e13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c8f9e13
Branch: refs/heads/HADOOP-13345
Commit: 4c8f9e130230457fc897ed7a2a09e14d078be90a
Parents: 02766b6
Author: Chris Nauroth <cn...@apache.org>
Authored: Thu Jan 5 15:04:47 2017 -0800
Committer: Chris Nauroth <cn...@apache.org>
Committed: Thu Jan 5 15:04:47 2017 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md | 5 +++++
hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm | 1 +
2 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8f9e13/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 27fd13a..50b4360 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -211,6 +211,11 @@ In the REST API, the prefix "`/webhdfs/v1`" is inserted in the path and a query
http://<HOST>:<HTTP_PORT>/webhdfs/v1/<PATH>?op=...
+**Note** that if WebHDFS is secured with SSL, then the scheme should be "`swebhdfs://`".
+
+ swebhdfs://<HOST>:<HTTP_PORT>/<PATH>
+
+
### HDFS Configuration Options
Below are the HDFS configuration options for WebHDFS.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8f9e13/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
index fc26321..dbf0e8d 100644
--- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
+++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
@@ -417,6 +417,7 @@ $H3 Copying Between Versions of HDFS
HftpFileSystem, as webhdfs is available for both read and write operations,
DistCp can be run on both source and destination cluster.
Remote cluster is specified as `webhdfs://<namenode_hostname>:<http_port>`.
+ (Use the "`swebhdfs://`" scheme when webhdfs is secured with SSL).
When copying between same major versions of Hadoop cluster (e.g. between 2.X
and 2.X), use hdfs protocol for better performance.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-6009. Skip validating app
timeout value on recovery. Contributed by Rohith Sharma K S
Posted by st...@apache.org.
YARN-6009. Skip validating app timeout value on recovery. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02031645
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02031645
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02031645
Branch: refs/heads/HADOOP-13345
Commit: 020316458dfe6059b700f8d93a9791e4cb817b3f
Parents: 71a4acf
Author: Jian He <ji...@apache.org>
Authored: Fri Jan 6 18:19:09 2017 -0800
Committer: Jian He <ji...@apache.org>
Committed: Fri Jan 6 18:19:09 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/server/resourcemanager/RMAppManager.java | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02031645/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index bc21952..9dd544e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -342,13 +342,16 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private RMAppImpl createAndPopulateNewRMApp(
ApplicationSubmissionContext submissionContext, long submitTime,
String user, boolean isRecovery, long startTime) throws YarnException {
- // Do queue mapping
if (!isRecovery) {
+ // Do queue mapping
if (rmContext.getQueuePlacementManager() != null) {
// We only do queue mapping when it's a new application
rmContext.getQueuePlacementManager().placeApplication(
submissionContext, user);
}
+ // fail the submission if configured application timeout value is invalid
+ RMServerUtils.validateApplicationTimeouts(
+ submissionContext.getApplicationTimeouts());
}
ApplicationId applicationId = submissionContext.getApplicationId();
@@ -390,10 +393,6 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
}
}
- // fail the submission if configured application timeout value is invalid
- RMServerUtils.validateApplicationTimeouts(
- submissionContext.getApplicationTimeouts());
-
// Create RMApp
RMAppImpl application =
new RMAppImpl(applicationId, rmContext, this.conf,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-13578. Add Codec for
ZStandard Compression. Contributed by churro morales
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
new file mode 100644
index 0000000..04def24
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
@@ -0,0 +1,485 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.compress.zstd;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.compress.CompressionInputStream;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.CompressorStream;
+import org.apache.hadoop.io.compress.Decompressor;
+import org.apache.hadoop.io.compress.DecompressorStream;
+import org.apache.hadoop.io.compress.ZStandardCodec;
+import org.apache.hadoop.test.MultithreadedTestUtil;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+public class TestZStandardCompressorDecompressor {
+ private final static char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray();
+ private static final Random RANDOM = new Random(12345L);
+ private static final Configuration CONFIGURATION = new Configuration();
+ private static File compressedFile;
+ private static File uncompressedFile;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ CONFIGURATION.setInt(IO_FILE_BUFFER_SIZE_KEY, 1024 * 64);
+ uncompressedFile = new File(TestZStandardCompressorDecompressor.class
+ .getResource("/zstd/test_file.txt").toURI());
+ compressedFile = new File(TestZStandardCompressorDecompressor.class
+ .getResource("/zstd/test_file.txt.zst").toURI());
+ }
+
+ @Before
+ public void before() throws Exception {
+ assumeTrue(ZStandardCodec.isNativeCodeLoaded());
+ }
+
+ @Test
+ public void testCompressionCompressesCorrectly() throws Exception {
+ int uncompressedSize = (int) FileUtils.sizeOf(uncompressedFile);
+ byte[] bytes = FileUtils.readFileToByteArray(uncompressedFile);
+ assertEquals(uncompressedSize, bytes.length);
+
+ Configuration conf = new Configuration();
+ ZStandardCodec codec = new ZStandardCodec();
+ codec.setConf(conf);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ Compressor compressor = codec.createCompressor();
+ CompressionOutputStream outputStream =
+ codec.createOutputStream(baos, compressor);
+
+ for (byte aByte : bytes) {
+ outputStream.write(aByte);
+ }
+
+ outputStream.finish();
+ outputStream.close();
+ assertEquals(uncompressedSize, compressor.getBytesRead());
+ assertTrue(compressor.finished());
+
+ // just make sure we can decompress the file
+
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ Decompressor decompressor = codec.createDecompressor();
+ CompressionInputStream inputStream =
+ codec.createInputStream(bais, decompressor);
+ byte[] buffer = new byte[100];
+ int n = buffer.length;
+ while ((n = inputStream.read(buffer, 0, n)) != -1) {
+ byteArrayOutputStream.write(buffer, 0, n);
+ }
+ assertArrayEquals(bytes, byteArrayOutputStream.toByteArray());
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testCompressorSetInputNullPointerException() {
+ ZStandardCompressor compressor = new ZStandardCompressor();
+ compressor.setInput(null, 0, 10);
+ }
+
+ //test on NullPointerException in {@code decompressor.setInput()}
+ @Test(expected = NullPointerException.class)
+ public void testDecompressorSetInputNullPointerException() {
+ ZStandardDecompressor decompressor =
+ new ZStandardDecompressor(IO_FILE_BUFFER_SIZE_DEFAULT);
+ decompressor.setInput(null, 0, 10);
+ }
+
+ //test on ArrayIndexOutOfBoundsException in {@code compressor.setInput()}
+ @Test(expected = ArrayIndexOutOfBoundsException.class)
+ public void testCompressorSetInputAIOBException() {
+ ZStandardCompressor compressor = new ZStandardCompressor();
+ compressor.setInput(new byte[] {}, -5, 10);
+ }
+
+ //test on ArrayIndexOutOfBoundsException in {@code decompressor.setInput()}
+ @Test(expected = ArrayIndexOutOfBoundsException.class)
+ public void testDecompressorSetInputAIOUBException() {
+ ZStandardDecompressor decompressor =
+ new ZStandardDecompressor(IO_FILE_BUFFER_SIZE_DEFAULT);
+ decompressor.setInput(new byte[] {}, -5, 10);
+ }
+
+ //test on NullPointerException in {@code compressor.compress()}
+ @Test(expected = NullPointerException.class)
+ public void testCompressorCompressNullPointerException() throws Exception {
+ ZStandardCompressor compressor = new ZStandardCompressor();
+ byte[] bytes = generate(1024 * 6);
+ compressor.setInput(bytes, 0, bytes.length);
+ compressor.compress(null, 0, 0);
+ }
+
+ //test on NullPointerException in {@code decompressor.decompress()}
+ @Test(expected = NullPointerException.class)
+ public void testDecompressorCompressNullPointerException() throws Exception {
+ ZStandardDecompressor decompressor =
+ new ZStandardDecompressor(IO_FILE_BUFFER_SIZE_DEFAULT);
+ byte[] bytes = generate(1024 * 6);
+ decompressor.setInput(bytes, 0, bytes.length);
+ decompressor.decompress(null, 0, 0);
+ }
+
+ //test on ArrayIndexOutOfBoundsException in {@code compressor.compress()}
+ @Test(expected = ArrayIndexOutOfBoundsException.class)
+ public void testCompressorCompressAIOBException() throws Exception {
+ ZStandardCompressor compressor = new ZStandardCompressor();
+ byte[] bytes = generate(1024 * 6);
+ compressor.setInput(bytes, 0, bytes.length);
+ compressor.compress(new byte[] {}, 0, -1);
+ }
+
+ //test on ArrayIndexOutOfBoundsException in decompressor.decompress()
+ @Test(expected = ArrayIndexOutOfBoundsException.class)
+ public void testDecompressorCompressAIOBException() throws Exception {
+ ZStandardDecompressor decompressor =
+ new ZStandardDecompressor(IO_FILE_BUFFER_SIZE_DEFAULT);
+ byte[] bytes = generate(1024 * 6);
+ decompressor.setInput(bytes, 0, bytes.length);
+ decompressor.decompress(new byte[] {}, 0, -1);
+ }
+
+ // test ZStandardCompressor compressor.compress()
+ @Test
+ public void testSetInputWithBytesSizeMoreThenDefaultZStandardBufferSize()
+ throws Exception {
+ int bytesSize = 1024 * 2056 + 1;
+ ZStandardCompressor compressor = new ZStandardCompressor();
+ byte[] bytes = generate(bytesSize);
+ assertTrue("needsInput error !!!", compressor.needsInput());
+ compressor.setInput(bytes, 0, bytes.length);
+ byte[] emptyBytes = new byte[bytesSize];
+ int cSize = compressor.compress(emptyBytes, 0, bytes.length);
+ assertTrue(cSize > 0);
+ }
+
+ // test compress/decompress process through
+ // CompressionOutputStream/CompressionInputStream api
+ @Test
+ public void testCompressorDecompressorLogicWithCompressionStreams()
+ throws Exception {
+ DataOutputStream deflateOut = null;
+ DataInputStream inflateIn = null;
+ int byteSize = 1024 * 100;
+ byte[] bytes = generate(byteSize);
+ int bufferSize = IO_FILE_BUFFER_SIZE_DEFAULT;
+ try {
+ DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
+ CompressionOutputStream deflateFilter =
+ new CompressorStream(compressedDataBuffer, new ZStandardCompressor(),
+ bufferSize);
+ deflateOut =
+ new DataOutputStream(new BufferedOutputStream(deflateFilter));
+ deflateOut.write(bytes, 0, bytes.length);
+ deflateOut.flush();
+ deflateFilter.finish();
+
+ DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
+ deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0,
+ compressedDataBuffer.getLength());
+
+ CompressionInputStream inflateFilter =
+ new DecompressorStream(deCompressedDataBuffer,
+ new ZStandardDecompressor(bufferSize), bufferSize);
+
+ inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter));
+
+ byte[] result = new byte[byteSize];
+ inflateIn.read(result);
+ assertArrayEquals("original array not equals compress/decompressed array",
+ result, bytes);
+ } finally {
+ IOUtils.closeQuietly(deflateOut);
+ IOUtils.closeQuietly(inflateIn);
+ }
+ }
+
+ @Test
+ public void testZStandardCompressDecompressInMultiThreads() throws Exception {
+ MultithreadedTestUtil.TestContext ctx =
+ new MultithreadedTestUtil.TestContext();
+ for (int i = 0; i < 10; i++) {
+ ctx.addThread(new MultithreadedTestUtil.TestingThread(ctx) {
+ @Override
+ public void doWork() throws Exception {
+ testCompressDecompress();
+ }
+ });
+ }
+ ctx.startThreads();
+
+ ctx.waitFor(60000);
+ }
+
+ @Test
+ public void testCompressDecompress() throws Exception {
+ byte[] rawData;
+ int rawDataSize;
+ rawDataSize = IO_FILE_BUFFER_SIZE_DEFAULT;
+ rawData = generate(rawDataSize);
+ ZStandardCompressor compressor = new ZStandardCompressor();
+ ZStandardDecompressor decompressor =
+ new ZStandardDecompressor(IO_FILE_BUFFER_SIZE_DEFAULT);
+ assertFalse(compressor.finished());
+ compressor.setInput(rawData, 0, rawData.length);
+ assertEquals(0, compressor.getBytesRead());
+ compressor.finish();
+
+ byte[] compressedResult = new byte[rawDataSize];
+ int cSize = compressor.compress(compressedResult, 0, rawDataSize);
+ assertEquals(rawDataSize, compressor.getBytesRead());
+ assertTrue(cSize < rawDataSize);
+ decompressor.setInput(compressedResult, 0, cSize);
+ byte[] decompressedBytes = new byte[rawDataSize];
+ decompressor.decompress(decompressedBytes, 0, decompressedBytes.length);
+ assertEquals(bytesToHex(rawData), bytesToHex(decompressedBytes));
+ compressor.reset();
+ decompressor.reset();
+ }
+
+ @Test
+ public void testCompressingWithOneByteOutputBuffer() throws Exception {
+ int uncompressedSize = (int) FileUtils.sizeOf(uncompressedFile);
+ byte[] bytes = FileUtils.readFileToByteArray(uncompressedFile);
+ assertEquals(uncompressedSize, bytes.length);
+
+ Configuration conf = new Configuration();
+ ZStandardCodec codec = new ZStandardCodec();
+ codec.setConf(conf);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ Compressor compressor =
+ new ZStandardCompressor(3, IO_FILE_BUFFER_SIZE_DEFAULT, 1);
+ CompressionOutputStream outputStream =
+ codec.createOutputStream(baos, compressor);
+
+ for (byte aByte : bytes) {
+ outputStream.write(aByte);
+ }
+
+ outputStream.finish();
+ outputStream.close();
+ assertEquals(uncompressedSize, compressor.getBytesRead());
+ assertTrue(compressor.finished());
+
+ // just make sure we can decompress the file
+
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ Decompressor decompressor = codec.createDecompressor();
+ CompressionInputStream inputStream =
+ codec.createInputStream(bais, decompressor);
+ byte[] buffer = new byte[100];
+ int n = buffer.length;
+ while ((n = inputStream.read(buffer, 0, n)) != -1) {
+ byteArrayOutputStream.write(buffer, 0, n);
+ }
+ assertArrayEquals(bytes, byteArrayOutputStream.toByteArray());
+
+ }
+
+ @Test
+ public void testZStandardCompressDecompress() throws Exception {
+ byte[] rawData = null;
+ int rawDataSize = 0;
+ rawDataSize = IO_FILE_BUFFER_SIZE_DEFAULT;
+ rawData = generate(rawDataSize);
+ ZStandardCompressor compressor = new ZStandardCompressor();
+ ZStandardDecompressor decompressor = new ZStandardDecompressor(rawDataSize);
+ assertTrue(compressor.needsInput());
+ assertFalse("testZStandardCompressDecompress finished error",
+ compressor.finished());
+ compressor.setInput(rawData, 0, rawData.length);
+ compressor.finish();
+
+ byte[] compressedResult = new byte[rawDataSize];
+ int cSize = compressor.compress(compressedResult, 0, rawDataSize);
+ assertEquals(rawDataSize, compressor.getBytesRead());
+ assertTrue("compressed size no less then original size",
+ cSize < rawDataSize);
+ decompressor.setInput(compressedResult, 0, cSize);
+ byte[] decompressedBytes = new byte[rawDataSize];
+ decompressor.decompress(decompressedBytes, 0, decompressedBytes.length);
+ String decompressed = bytesToHex(decompressedBytes);
+ String original = bytesToHex(rawData);
+ assertEquals(original, decompressed);
+ compressor.reset();
+ decompressor.reset();
+ }
+
+ @Test
+ public void testDecompressingOutput() throws Exception {
+ byte[] expectedDecompressedResult =
+ FileUtils.readFileToByteArray(uncompressedFile);
+ ZStandardCodec codec = new ZStandardCodec();
+ codec.setConf(CONFIGURATION);
+ CompressionInputStream inputStream = codec
+ .createInputStream(FileUtils.openInputStream(compressedFile),
+ codec.createDecompressor());
+
+ byte[] toDecompress = new byte[100];
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ byte[] decompressedResult;
+ int totalFileSize = 0;
+ int result = toDecompress.length;
+ try {
+ while ((result = inputStream.read(toDecompress, 0, result)) != -1) {
+ baos.write(toDecompress, 0, result);
+ totalFileSize += result;
+ }
+ decompressedResult = baos.toByteArray();
+ } finally {
+ IOUtils.closeQuietly(baos);
+ }
+
+ assertEquals(decompressedResult.length, totalFileSize);
+ assertEquals(bytesToHex(expectedDecompressedResult),
+ bytesToHex(decompressedResult));
+ }
+
+ @Test
+ public void testZStandardDirectCompressDecompress() throws Exception {
+ int[] size = {1, 4, 16, 4 * 1024, 64 * 1024, 128 * 1024, 1024 * 1024 };
+ for (int aSize : size) {
+ System.out.println("aSize = " + aSize);
+ compressDecompressLoop(aSize);
+ }
+ }
+
+ private void compressDecompressLoop(int rawDataSize) throws IOException {
+ byte[] rawData = null;
+ rawData = generate(rawDataSize);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(rawDataSize + 12);
+ CompressionOutputStream deflateFilter =
+ new CompressorStream(baos, new ZStandardCompressor(), 4096);
+ DataOutputStream deflateOut =
+ new DataOutputStream(new BufferedOutputStream(deflateFilter));
+
+ deflateOut.write(rawData, 0, rawData.length);
+ deflateOut.flush();
+ deflateFilter.finish();
+ byte[] compressedResult = baos.toByteArray();
+ int compressedSize = compressedResult.length;
+ ZStandardDecompressor.ZStandardDirectDecompressor decompressor =
+ new ZStandardDecompressor.ZStandardDirectDecompressor(4096);
+
+ ByteBuffer inBuf = ByteBuffer.allocateDirect(compressedSize);
+ ByteBuffer outBuf = ByteBuffer.allocateDirect(8096);
+
+ inBuf.put(compressedResult, 0, compressedSize);
+ inBuf.flip();
+
+ ByteBuffer expected = ByteBuffer.wrap(rawData);
+
+ outBuf.clear();
+ while (!decompressor.finished()) {
+ decompressor.decompress(inBuf, outBuf);
+ if (outBuf.remaining() == 0) {
+ outBuf.flip();
+ while (outBuf.remaining() > 0) {
+ assertEquals(expected.get(), outBuf.get());
+ }
+ outBuf.clear();
+ }
+ }
+ outBuf.flip();
+ while (outBuf.remaining() > 0) {
+ assertEquals(expected.get(), outBuf.get());
+ }
+ outBuf.clear();
+
+ assertEquals(0, expected.remaining());
+ }
+
+ @Test
+ public void testReadingWithAStream() throws Exception {
+ FileInputStream inputStream = FileUtils.openInputStream(compressedFile);
+ ZStandardCodec codec = new ZStandardCodec();
+ codec.setConf(CONFIGURATION);
+ Decompressor decompressor = codec.createDecompressor();
+ CompressionInputStream cis =
+ codec.createInputStream(inputStream, decompressor);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ byte[] resultOfDecompression;
+ try {
+ byte[] buffer = new byte[100];
+ int n;
+ while ((n = cis.read(buffer, 0, buffer.length)) != -1) {
+ baos.write(buffer, 0, n);
+ }
+ resultOfDecompression = baos.toByteArray();
+ } finally {
+ IOUtils.closeQuietly(baos);
+ IOUtils.closeQuietly(cis);
+ }
+
+ byte[] expected = FileUtils.readFileToByteArray(uncompressedFile);
+ assertEquals(bytesToHex(expected), bytesToHex(resultOfDecompression));
+ }
+
+ @Test
+ public void testDecompressReturnsWhenNothingToDecompress() throws Exception {
+ ZStandardDecompressor decompressor =
+ new ZStandardDecompressor(IO_FILE_BUFFER_SIZE_DEFAULT);
+ int result = decompressor.decompress(new byte[10], 0, 10);
+ assertEquals(0, result);
+ }
+
+ public static byte[] generate(int size) {
+ byte[] data = new byte[size];
+ for (int i = 0; i < size; i++) {
+ data[i] = (byte) RANDOM.nextInt(16);
+ }
+ return data;
+ }
+
+ private static String bytesToHex(byte[] bytes) {
+ char[] hexChars = new char[bytes.length * 2];
+ for (int j = 0; j < bytes.length; j++) {
+ int v = bytes[j] & 0xFF;
+ hexChars[j * 2] = HEX_ARRAY[v >>> 4];
+ hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F];
+ }
+ return new String(hexChars);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt b/hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt
new file mode 100644
index 0000000..60432f7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+Apache License
+
+Version 2.0, January 2004
+
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of this License; and
+You must cause any modified files to carry prominent notices stating that You changed the files; and
+You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
+If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
+
+You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
+5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt.zst
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt.zst b/hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt.zst
new file mode 100644
index 0000000..0384b4f
Binary files /dev/null and b/hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt.zst differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 9f12986..2c8264d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -109,6 +109,10 @@
<snappy.lib></snappy.lib>
<snappy.include></snappy.include>
<require.snappy>false</require.snappy>
+ <zstd.prefix></zstd.prefix>
+ <zstd.lib></zstd.lib>
+ <zstd.include></zstd.include>
+ <require.zstd>false</require.zstd>
</properties>
<build>
<plugins>
@@ -197,6 +201,10 @@
<CUSTOM_SNAPPY_PREFIX>${snappy.prefix}</CUSTOM_SNAPPY_PREFIX>
<CUSTOM_SNAPPY_LIB>${snappy.lib}</CUSTOM_SNAPPY_LIB>
<CUSTOM_SNAPPY_INCLUDE>${snappy.include}</CUSTOM_SNAPPY_INCLUDE>
+ <REQUIRE_ZSTD>${require.zstd}</REQUIRE_ZSTD>
+ <CUSTOM_ZSTD_PREFIX>${zstd.prefix}</CUSTOM_ZSTD_PREFIX>
+ <CUSTOM_ZSTD_LIB>${zstd.lib}</CUSTOM_ZSTD_LIB>
+ <CUSTOM_ZSTD_INCLUDE>${zstd.include}</CUSTOM_ZSTD_INCLUDE>
</vars>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index e0d2be9..730ad05 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -41,6 +41,9 @@
<snappy.lib></snappy.lib>
<bundle.snappy>false</bundle.snappy>
<bundle.snappy.in.bin>false</bundle.snappy.in.bin>
+ <zstd.lib></zstd.lib>
+ <bundle.zstd>false</bundle.zstd>
+ <bundle.zstd.in.bin>false</bundle.zstd.in.bin>
<isal.lib></isal.lib>
<bundle.isal>false</bundle.isal>
<openssl.lib></openssl.lib>
@@ -363,6 +366,9 @@
<argument>--snappybinbundle=${bundle.snappy.in.bin}</argument>
<argument>--snappylib=${snappy.lib}</argument>
<argument>--snappylibbundle=${bundle.snappy}</argument>
+ <argument>--zstdbinbundle=${bundle.zstd.in.bin}</argument>
+ <argument>--zstdlib=${zstd.lib}</argument>
+ <argument>--zstdlibbundle=${bundle.zstd}</argument>
</arguments>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a935292..4751fc3 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1596,6 +1596,7 @@
<!-- attempt to open a file at this path. -->
<java.security.egd>file:/dev/urandom</java.security.egd>
<bundle.snappy.in.bin>true</bundle.snappy.in.bin>
+ <bundle.zstd.in.bin>true</bundle.zstd.in.bin>
<bundle.openssl.in.bin>true</bundle.openssl.in.bin>
</properties>
<build>
@@ -1607,6 +1608,7 @@
<environmentVariables>
<!-- Specify where to look for the native DLL on Windows -->
<PATH>${env.PATH};${hadoop.common.build.dir}/bin;${snappy.lib}</PATH>
+ <PATH>${env.PATH};${hadoop.common.build.dir}/bin;${zstd.lib}</PATH>
<PATH>${env.PATH};${hadoop.common.build.dir}/bin;${openssl.lib}</PATH>
<PATH>${env.PATH};${hadoop.common.build.dir}/bin;${isal.lib}</PATH>
</environmentVariables>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HADOOP-13780. LICENSE/NOTICE are
out of date for source artifacts.
Posted by st...@apache.org.
HADOOP-13780. LICENSE/NOTICE are out of date for source artifacts.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8850c056
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8850c056
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8850c056
Branch: refs/heads/HADOOP-13345
Commit: 8850c056a5af774e3c614b57135709baed718c7a
Parents: 4c8f9e1
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Jan 5 16:04:58 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Jan 5 16:04:58 2017 -0800
----------------------------------------------------------------------
LICENSE.txt | 1314 ++++++++++++++++++++++++++++++++----------------------
NOTICE.txt | 129 +++++-
2 files changed, 909 insertions(+), 534 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8850c056/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index fd07edf..d334204 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -324,6 +324,7 @@ The binary distribution of this product bundles these dependencies under the
following license:
re2j 1.0
---------------------------------------------------------------------
+(GO license)
This is a work derived from Russ Cox's RE2 in Go, whose license
http://golang.org/LICENSE is as follows:
@@ -577,17 +578,62 @@ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
-For:
+The binary distribution of this product bundles these dependencies under the
+following license:
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2
hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js
hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css
hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css
-And the binary distribution of this product bundles these dependencies under the
-following license:
-Mockito 1.8.5
-SLF4J 1.7.10
+bootstrap v3.3.6
+broccoli-asset-rev v2.4.2
+broccoli-funnel v1.0.1
+datatables v1.10.8
+em-helpers v0.5.13
+em-table v0.1.6
+ember v2.2.0
+ember-array-contains-helper v1.0.2
+ember-bootstrap v0.5.1
+ember-cli v1.13.13
+ember-cli-app-version v1.0.0
+ember-cli-babel v5.1.6
+ember-cli-content-security-policy v0.4.0
+ember-cli-dependency-checker v1.2.0
+ember-cli-htmlbars v1.0.2
+ember-cli-htmlbars-inline-precompile v0.3.1
+ember-cli-ic-ajax v0.2.1
+ember-cli-inject-live-reload v1.4.0
+ember-cli-jquery-ui v0.0.20
+ember-cli-qunit v1.2.1
+ember-cli-release v0.2.8
+ember-cli-shims v0.0.6
+ember-cli-sri v1.2.1
+ember-cli-test-loader v0.2.1
+ember-cli-uglify v1.2.0
+ember-d3 v0.1.0
+ember-data v2.1.0
+ember-disable-proxy-controllers v1.0.1
+ember-export-application-global v1.0.5
+ember-load-initializers v0.1.7
+ember-qunit v0.4.16
+ember-qunit-notifications v0.1.0
+ember-resolver v2.0.3
+ember-spin-spinner v0.2.3
+ember-truth-helpers v1.2.0
+jquery v2.1.4
+jquery-ui v1.11.4
+loader.js v3.3.0
+momentjs v2.10.6
+qunit v1.19.0
+select2 v4.0.0
+snippet-ss v1.11.0
+spin.js v2.3.2
+Azure Data Lake Store - Java client SDK 2.0.11
JCodings 1.0.8
Joni 2.1.2
+Mockito 1.8.5
+JUL to SLF4J bridge 1.7.10
+SLF4J API Module 1.7.10
+SLF4J LOG4J-12 Binding 1.7.10
--------------------------------------------------------------------------------
The MIT License (MIT)
@@ -614,9 +660,10 @@ For:
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js
hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery
+Apache HBase - Server which contains JQuery minified javascript library version 1.8.3
--------------------------------------------------------------------------------
-Copyright jQuery Foundation and other contributors, https://jquery.org/
+Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, https://jquery.org/
This software consists of voluntary contributions made by many
individuals. For exact contribution history, see the revision history
@@ -654,7 +701,7 @@ own licenses; we recommend you read them, as their terms may differ from
the terms above.
For:
-hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js.gz
+hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js
--------------------------------------------------------------------------------
Copyright (c) 2014 Ivan Bozhanov
@@ -691,6 +738,7 @@ The binary distribution of this product bundles these dependencies under the
following license:
HSQLDB Database 2.0.0
--------------------------------------------------------------------------------
+(HSQL License)
"COPYRIGHTS AND LICENSES (based on BSD License)
For work developed by the HSQL Development Group:
@@ -760,10 +808,14 @@ Hypersonic SQL Group."
The binary distribution of this product bundles these dependencies under the
following license:
+Java Servlet API 3.1.0
servlet-api 2.5
jsp-api 2.1
-Streaming API for XML 1.0
+jsr311-api 1.1.1
+Glassfish Jasper 6.1.14
+Servlet Specification 2.5 API 6.1.14
--------------------------------------------------------------------------------
+(CDDL 1.0)
COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0
1.�Definitions.�
@@ -1075,10 +1127,24 @@ of liability.�
The binary distribution of this product bundles these dependencies under the
following license:
-Jersey 1.9
-JAXB API bundle for GlassFish V3 2.2.2
+jersey-client 1.19
+jersey-core 1.19
+jersey-grizzly2 1.19
+jersey-grizzly2-servlet 1.19
+jersey-json 1.19
+jersey-server 1.19
+jersey-servlet 1.19
+jersey-guice 1.19
+Jersey Test Framework - Grizzly 2 Module 1.19
JAXB RI 2.2.3
+Java Architecture for XML Binding 2.2.11
+grizzly-framework 2.2.21
+grizzly-http 2.2.21
+grizzly-http-server 2.2.21
+grizzly-http-servlet 2.2.21
+grizzly-rcm 2.2.21
--------------------------------------------------------------------------------
+(CDDL 1.1)
COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1
1. Definitions.
@@ -1438,8 +1504,9 @@ derived.)
The binary distribution of this product bundles these dependencies under the
following license:
JUnit 4.11
-ecj-4.3.1.jar
+Eclipse JDT Core 3.1.1
--------------------------------------------------------------------------------
+(EPL v1.0)
Eclipse Public License - v 1.0
THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC
@@ -1636,14 +1703,17 @@ any resulting litigation.
The binary distribution of this product bundles these dependencies under the
following license:
-ASM Core 3.2
JSch 0.1.51
ParaNamer Core 2.3
JLine 0.9.94
leveldbjni-all 1.8
Hamcrest Core 1.3
+ASM Core 5.0.4
+ASM Commons 5.0.2
+ASM Tree 5.0.2
xmlenc Library 0.52
--------------------------------------------------------------------------------
+(3-clause BSD)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
@@ -1670,6 +1740,7 @@ The binary distribution of this product bundles these dependencies under the
following license:
FindBugs-jsr305 3.0.0
--------------------------------------------------------------------------------
+(2-clause BSD)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
@@ -1696,8 +1767,9 @@ either expressed or implied, of the FreeBSD Project.
The binary distribution of this product bundles these dependencies under the
following license:
-Java Concurrency in Practice book annotations 1.0
+"Java Concurrency in Practice" book annotations 1.0
--------------------------------------------------------------------------------
+(CCAL v2.5)
THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS
PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR
OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS
@@ -1875,536 +1947,716 @@ The binary distribution of this product bundles these dependencies under the
following license:
jamon-runtime 2.4.1
--------------------------------------------------------------------------------
- MOZILLA PUBLIC LICENSE
- Version 1.1
+(MPL 2.0)
+ Mozilla Public License
+ Version 2.0
- ---------------
+1. Definitions
-1. Definitions.
+1.1. \u201cContributor\u201d
+means each individual or legal entity that creates, contributes to the creation
+of, or owns Covered Software.
- 1.0.1. "Commercial Use" means distribution or otherwise making the
- Covered Code available to a third party.
-
- 1.1. "Contributor" means each entity that creates or contributes to
- the creation of Modifications.
-
- 1.2. "Contributor Version" means the combination of the Original
- Code, prior Modifications used by a Contributor, and the Modifications
- made by that particular Contributor.
-
- 1.3. "Covered Code" means the Original Code or Modifications or the
- combination of the Original Code and Modifications, in each case
- including portions thereof.
-
- 1.4. "Electronic Distribution Mechanism" means a mechanism generally
- accepted in the software development community for the electronic
- transfer of data.
-
- 1.5. "Executable" means Covered Code in any form other than Source
- Code.
-
- 1.6. "Initial Developer" means the individual or entity identified
- as the Initial Developer in the Source Code notice required by Exhibit
- A.
-
- 1.7. "Larger Work" means a work which combines Covered Code or
- portions thereof with code not governed by the terms of this License.
-
- 1.8. "License" means this document.
-
- 1.8.1. "Licensable" means having the right to grant, to the maximum
- extent possible, whether at the time of the initial grant or
- subsequently acquired, any and all of the rights conveyed herein.
-
- 1.9. "Modifications" means any addition to or deletion from the
- substance or structure of either the Original Code or any previous
- Modifications. When Covered Code is released as a series of files, a
- Modification is:
- A. Any addition to or deletion from the contents of a file
- containing Original Code or previous Modifications.
-
- B. Any new file that contains any part of the Original Code or
- previous Modifications.
-
- 1.10. "Original Code" means Source Code of computer software code
- which is described in the Source Code notice required by Exhibit A as
- Original Code, and which, at the time of its release under this
- License is not already Covered Code governed by this License.
-
- 1.10.1. "Patent Claims" means any patent claim(s), now owned or
- hereafter acquired, including without limitation, method, process,
- and apparatus claims, in any patent Licensable by grantor.
-
- 1.11. "Source Code" means the preferred form of the Covered Code for
- making modifications to it, including all modules it contains, plus
- any associated interface definition files, scripts used to control
- compilation and installation of an Executable, or source code
- differential comparisons against either the Original Code or another
- well known, available Covered Code of the Contributor's choice. The
- Source Code can be in a compressed or archival form, provided the
- appropriate decompression or de-archiving software is widely available
- for no charge.
-
- 1.12. "You" (or "Your") means an individual or a legal entity
- exercising rights under, and complying with all of the terms of, this
- License or a future version of this License issued under Section 6.1.
- For legal entities, "You" includes any entity which controls, is
- controlled by, or is under common control with You. For purposes of
- this definition, "control" means (a) the power, direct or indirect,
- to cause the direction or management of such entity, whether by
- contract or otherwise, or (b) ownership of more than fifty percent
- (50%) of the outstanding shares or beneficial ownership of such
- entity.
-
-2. Source Code License.
-
- 2.1. The Initial Developer Grant.
- The Initial Developer hereby grants You a world-wide, royalty-free,
- non-exclusive license, subject to third party intellectual property
- claims:
- (a) under intellectual property rights (other than patent or
- trademark) Licensable by Initial Developer to use, reproduce,
- modify, display, perform, sublicense and distribute the Original
- Code (or portions thereof) with or without Modifications, and/or
- as part of a Larger Work; and
-
- (b) under Patents Claims infringed by the making, using or
- selling of Original Code, to make, have made, use, practice,
- sell, and offer for sale, and/or otherwise dispose of the
- Original Code (or portions thereof).
-
- (c) the licenses granted in this Section 2.1(a) and (b) are
- effective on the date Initial Developer first distributes
- Original Code under the terms of this License.
-
- (d) Notwithstanding Section 2.1(b) above, no patent license is
- granted: 1) for code that You delete from the Original Code; 2)
- separate from the Original Code; or 3) for infringements caused
- by: i) the modification of the Original Code or ii) the
- combination of the Original Code with other software or devices.
-
- 2.2. Contributor Grant.
- Subject to third party intellectual property claims, each Contributor
- hereby grants You a world-wide, royalty-free, non-exclusive license
-
- (a) under intellectual property rights (other than patent or
- trademark) Licensable by Contributor, to use, reproduce, modify,
- display, perform, sublicense and distribute the Modifications
- created by such Contributor (or portions thereof) either on an
- unmodified basis, with other Modifications, as Covered Code
- and/or as part of a Larger Work; and
-
- (b) under Patent Claims infringed by the making, using, or
- selling of Modifications made by that Contributor either alone
- and/or in combination with its Contributor Version (or portions
- of such combination), to make, use, sell, offer for sale, have
- made, and/or otherwise dispose of: 1) Modifications made by that
- Contributor (or portions thereof); and 2) the combination of
- Modifications made by that Contributor with its Contributor
- Version (or portions of such combination).
-
- (c) the licenses granted in Sections 2.2(a) and 2.2(b) are
- effective on the date Contributor first makes Commercial Use of
- the Covered Code.
-
- (d) Notwithstanding Section 2.2(b) above, no patent license is
- granted: 1) for any code that Contributor has deleted from the
- Contributor Version; 2) separate from the Contributor Version;
- 3) for infringements caused by: i) third party modifications of
- Contributor Version or ii) the combination of Modifications made
- by that Contributor with other software (except as part of the
- Contributor Version) or other devices; or 4) under Patent Claims
- infringed by Covered Code in the absence of Modifications made by
- that Contributor.
+1.2. \u201cContributor Version\u201d
+means the combination of the Contributions of others (if any) used by a
+Contributor and that particular Contributor\u2019s Contribution.
-3. Distribution Obligations.
+1.3. \u201cContribution\u201d
+means Covered Software of a particular Contributor.
- 3.1. Application of License.
- The Modifications which You create or to which You contribute are
- governed by the terms of this License, including without limitation
- Section 2.2. The Source Code version of Covered Code may be
- distributed only under the terms of this License or a future version
- of this License released under Section 6.1, and You must include a
- copy of this License with every copy of the Source Code You
- distribute. You may not offer or impose any terms on any Source Code
- version that alters or restricts the applicable version of this
- License or the recipients' rights hereunder. However, You may include
- an additional document offering the additional rights described in
- Section 3.5.
-
- 3.2. Availability of Source Code.
- Any Modification which You create or to which You contribute must be
- made available in Source Code form under the terms of this License
- either on the same media as an Executable version or via an accepted
- Electronic Distribution Mechanism to anyone to whom you made an
- Executable version available; and if made available via Electronic
- Distribution Mechanism, must remain available for at least twelve (12)
- months after the date it initially became available, or at least six
- (6) months after a subsequent version of that particular Modification
- has been made available to such recipients. You are responsible for
- ensuring that the Source Code version remains available even if the
- Electronic Distribution Mechanism is maintained by a third party.
-
- 3.3. Description of Modifications.
- You must cause all Covered Code to which You contribute to contain a
- file documenting the changes You made to create that Covered Code and
- the date of any change. You must include a prominent statement that
- the Modification is derived, directly or indirectly, from Original
- Code provided by the Initial Developer and including the name of the
- Initial Developer in (a) the Source Code, and (b) in any notice in an
- Executable version or related documentation in which You describe the
- origin or ownership of the Covered Code.
-
- 3.4. Intellectual Property Matters
- (a) Third Party Claims.
- If Contributor has knowledge that a license under a third party's
- intellectual property rights is required to exercise the rights
- granted by such Contributor under Sections 2.1 or 2.2,
- Contributor must include a text file with the Source Code
- distribution titled "LEGAL" which describes the claim and the
- party making the claim in sufficient detail that a recipient will
- know whom to contact. If Contributor obtains such knowledge after
- the Modification is made available as described in Section 3.2,
- Contributor shall promptly modify the LEGAL file in all copies
- Contributor makes available thereafter and shall take other steps
- (such as notifying appropriate mailing lists or newsgroups)
- reasonably calculated to inform those who received the Covered
- Code that new knowledge has been obtained.
-
- (b) Contributor APIs.
- If Contributor's Modifications include an application programming
- interface and Contributor has knowledge of patent licenses which
- are reasonably necessary to implement that API, Contributor must
- also include this information in the LEGAL file.
-
- (c) Representations.
- Contributor represents that, except as disclosed pursuant to
- Section 3.4(a) above, Contributor believes that Contributor's
- Modifications are Contributor's original creation(s) and/or
- Contributor has sufficient rights to grant the rights conveyed by
- this License.
-
- 3.5. Required Notices.
- You must duplicate the notice in Exhibit A in each file of the Source
- Code. If it is not possible to put such notice in a particular Source
- Code file due to its structure, then You must include such notice in a
- location (such as a relevant directory) where a user would be likely
- to look for such a notice. If You created one or more Modification(s)
- You may add your name as a Contributor to the notice described in
- Exhibit A. You must also duplicate this License in any documentation
- for the Source Code where You describe recipients' rights or ownership
- rights relating to Covered Code. You may choose to offer, and to
- charge a fee for, warranty, support, indemnity or liability
- obligations to one or more recipients of Covered Code. However, You
- may do so only on Your own behalf, and not on behalf of the Initial
- Developer or any Contributor. You must make it absolutely clear than
- any such warranty, support, indemnity or liability obligation is
- offered by You alone, and You hereby agree to indemnify the Initial
- Developer and every Contributor for any liability incurred by the
- Initial Developer or such Contributor as a result of warranty,
- support, indemnity or liability terms You offer.
-
- 3.6. Distribution of Executable Versions.
- You may distribute Covered Code in Executable form only if the
- requirements of Section 3.1-3.5 have been met for that Covered Code,
- and if You include a notice stating that the Source Code version of
- the Covered Code is available under the terms of this License,
- including a description of how and where You have fulfilled the
- obligations of Section 3.2. The notice must be conspicuously included
- in any notice in an Executable version, related documentation or
- collateral in which You describe recipients' rights relating to the
- Covered Code. You may distribute the Executable version of Covered
- Code or ownership rights under a license of Your choice, which may
- contain terms different from this License, provided that You are in
- compliance with the terms of this License and that the license for the
- Executable version does not attempt to limit or alter the recipient's
- rights in the Source Code version from the rights set forth in this
- License. If You distribute the Executable version under a different
- license You must make it absolutely clear that any terms which differ
- from this License are offered by You alone, not by the Initial
- Developer or any Contributor. You hereby agree to indemnify the
- Initial Developer and every Contributor for any liability incurred by
- the Initial Developer or such Contributor as a result of any such
- terms You offer.
-
- 3.7. Larger Works.
- You may create a Larger Work by combining Covered Code with other code
- not governed by the terms of this License and distribute the Larger
- Work as a single product. In such a case, You must make sure the
- requirements of this License are fulfilled for the Covered Code.
-
-4. Inability to Comply Due to Statute or Regulation.
-
- If it is impossible for You to comply with any of the terms of this
- License with respect to some or all of the Covered Code due to
- statute, judicial order, or regulation then You must: (a) comply with
- the terms of this License to the maximum extent possible; and (b)
- describe the limitations and the code they affect. Such description
- must be included in the LEGAL file described in Section 3.4 and must
- be included with all distributions of the Source Code. Except to the
- extent prohibited by statute or regulation, such description must be
- sufficiently detailed for a recipient of ordinary skill to be able to
- understand it.
-
-5. Application of this License.
-
- This License applies to code to which the Initial Developer has
- attached the notice in Exhibit A and to related Covered Code.
-
-6. Versions of the License.
-
- 6.1. New Versions.
- Netscape Communications Corporation ("Netscape") may publish revised
- and/or new versions of the License from time to time. Each version
- will be given a distinguishing version number.
-
- 6.2. Effect of New Versions.
- Once Covered Code has been published under a particular version of the
- License, You may always continue to use it under the terms of that
- version. You may also choose to use such Covered Code under the terms
- of any subsequent version of the License published by Netscape. No one
- other than Netscape has the right to modify the terms applicable to
- Covered Code created under this License.
-
- 6.3. Derivative Works.
- If You create or use a modified version of this License (which you may
- only do in order to apply it to code which is not already Covered Code
- governed by this License), You must (a) rename Your license so that
- the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape",
- "MPL", "NPL" or any confusingly similar phrase do not appear in your
- license (except to note that your license differs from this License)
- and (b) otherwise make it clear that Your version of the license
- contains terms which differ from the Mozilla Public License and
- Netscape Public License. (Filling in the name of the Initial
- Developer, Original Code or Contributor in the notice described in
- Exhibit A shall not of themselves be deemed to be modifications of
- this License.)
-
-7. DISCLAIMER OF WARRANTY.
-
- COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS,
- WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
- WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF
- DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING.
- THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE
- IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT,
- YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE
- COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER
- OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF
- ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
-
-8. TERMINATION.
-
- 8.1. This License and the rights granted hereunder will terminate
- automatically if You fail to comply with terms herein and fail to cure
- such breach within 30 days of becoming aware of the breach. All
- sublicenses to the Covered Code which are properly granted shall
- survive any termination of this License. Provisions which, by their
- nature, must remain in effect beyond the termination of this License
- shall survive.
-
- 8.2. If You initiate litigation by asserting a patent infringement
- claim (excluding declatory judgment actions) against Initial Developer
- or a Contributor (the Initial Developer or Contributor against whom
- You file such action is referred to as "Participant") alleging that:
-
- (a) such Participant's Contributor Version directly or indirectly
- infringes any patent, then any and all rights granted by such
- Participant to You under Sections 2.1 and/or 2.2 of this License
- shall, upon 60 days notice from Participant terminate prospectively,
- unless if within 60 days after receipt of notice You either: (i)
- agree in writing to pay Participant a mutually agreeable reasonable
- royalty for Your past and future use of Modifications made by such
- Participant, or (ii) withdraw Your litigation claim with respect to
- the Contributor Version against such Participant. If within 60 days
- of notice, a reasonable royalty and payment arrangement are not
- mutually agreed upon in writing by the parties or the litigation claim
- is not withdrawn, the rights granted by Participant to You under
- Sections 2.1 and/or 2.2 automatically terminate at the expiration of
- the 60 day notice period specified above.
-
- (b) any software, hardware, or device, other than such Participant's
- Contributor Version, directly or indirectly infringes any patent, then
- any rights granted to You by such Participant under Sections 2.1(b)
- and 2.2(b) are revoked effective as of the date You first made, used,
- sold, distributed, or had made, Modifications made by that
- Participant.
-
- 8.3. If You assert a patent infringement claim against Participant
- alleging that such Participant's Contributor Version directly or
- indirectly infringes any patent where such claim is resolved (such as
- by license or settlement) prior to the initiation of patent
- infringement litigation, then the reasonable value of the licenses
- granted by such Participant under Sections 2.1 or 2.2 shall be taken
- into account in determining the amount or value of any payment or
- license.
-
- 8.4. In the event of termination under Sections 8.1 or 8.2 above,
- all end user license agreements (excluding distributors and resellers)
- which have been validly granted by You or any distributor hereunder
- prior to termination shall survive termination.
-
-9. LIMITATION OF LIABILITY.
-
- UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT
- (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL
- DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE,
- OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR
- ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY
- CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL,
- WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER
- COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN
- INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF
- LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY
- RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW
- PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE
- EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO
- THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
-
-10. U.S. GOVERNMENT END USERS.
-
- The Covered Code is a "commercial item," as that term is defined in
- 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer
- software" and "commercial computer software documentation," as such
- terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48
- C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995),
- all U.S. Government End Users acquire Covered Code with only those
- rights set forth herein.
-
-11. MISCELLANEOUS.
-
- This License represents the complete agreement concerning subject
- matter hereof. If any provision of this License is held to be
- unenforceable, such provision shall be reformed only to the extent
- necessary to make it enforceable. This License shall be governed by
- California law provisions (except to the extent applicable law, if
- any, provides otherwise), excluding its conflict-of-law provisions.
- With respect to disputes in which at least one party is a citizen of,
- or an entity chartered or registered to do business in the United
- States of America, any litigation relating to this License shall be
- subject to the jurisdiction of the Federal Courts of the Northern
- District of California, with venue lying in Santa Clara County,
- California, with the losing party responsible for costs, including
- without limitation, court costs and reasonable attorneys' fees and
- expenses. The application of the United Nations Convention on
- Contracts for the International Sale of Goods is expressly excluded.
- Any law or regulation which provides that the language of a contract
- shall be construed against the drafter shall not apply to this
- License.
-
-12. RESPONSIBILITY FOR CLAIMS.
-
- As between Initial Developer and the Contributors, each party is
- responsible for claims and damages arising, directly or indirectly,
- out of its utilization of rights under this License and You agree to
- work with Initial Developer and Contributors to distribute such
- responsibility on an equitable basis. Nothing herein is intended or
- shall be deemed to constitute any admission of liability.
-
-13. MULTIPLE-LICENSED CODE.
-
- Initial Developer may designate portions of the Covered Code as
- "Multiple-Licensed". "Multiple-Licensed" means that the Initial
- Developer permits you to utilize portions of the Covered Code under
- Your choice of the MPL or the alternative licenses, if any, specified
- by the Initial Developer in the file described in Exhibit A.
+1.4. \u201cCovered Software\u201d
+means Source Code Form to which the initial Contributor has attached the notice
+in Exhibit A, the Executable Form of such Source Code Form, and Modifications of
+such Source Code Form, in each case including portions thereof.
-The binary distribution of this product bundles these dependencies under the
-following license:
-bootstrap v3.3.6
-broccoli-asset-rev v2.4.2
-broccoli-funnel v1.0.1
-datatables v1.10.8
-em-helpers v0.5.13
-em-table v0.1.6
-ember v2.2.0
-ember-array-contains-helper v1.0.2
-ember-bootstrap v0.5.1
-ember-cli v1.13.13
-ember-cli-app-version v1.0.0
-ember-cli-babel v5.1.6
-ember-cli-content-security-policy v0.4.0
-ember-cli-dependency-checker v1.2.0
-ember-cli-htmlbars v1.0.2
-ember-cli-htmlbars-inline-precompile v0.3.1
-ember-cli-ic-ajax v0.2.1
-ember-cli-inject-live-reload v1.4.0
-ember-cli-jquery-ui v0.0.20
-ember-cli-qunit v1.2.1
-ember-cli-release v0.2.8
-ember-cli-shims v0.0.6
-ember-cli-sri v1.2.1
-ember-cli-test-loader v0.2.1
-ember-cli-uglify v1.2.0
-ember-d3 v0.1.0
-ember-data v2.1.0
-ember-disable-proxy-controllers v1.0.1
-ember-export-application-global v1.0.5
-ember-load-initializers v0.1.7
-ember-qunit v0.4.16
-ember-qunit-notifications v0.1.0
-ember-resolver v2.0.3
-ember-spin-spinner v0.2.3
-ember-truth-helpers v1.2.0
-jquery v2.1.4
-jquery-ui v1.11.4
-loader.js v3.3.0
-momentjs v2.10.6
-qunit v1.19.0
-select2 v4.0.0
-snippet-ss v1.11.0
-spin.js v2.3.2
--------------------------------------------------------------------------------
-The MIT License (MIT)
+1.5. \u201cIncompatible With Secondary Licenses\u201d
+means
-All rights reserved.
+that the initial Contributor has attached the notice described in Exhibit B to
+the Covered Software; or
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and assocated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+that the Covered Software was made available under the terms of version 1.1 or
+earlier of the License, but not also under the terms of a Secondary License.
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+1.6. \u201cExecutable Form\u201d
+means any form of the work other than Source Code Form.
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+1.7. \u201cLarger Work\u201d
+means a work that combines Covered Software with other material, in a separate
+file or files, that is not Covered Software.
+
+1.8. \u201cLicense\u201d
+means this document.
+
+1.9. \u201cLicensable\u201d
+means having the right to grant, to the maximum extent possible, whether at the
+time of the initial grant or subsequently, any and all of the rights conveyed by
+this License.
+
+1.10. \u201cModifications\u201d
+means any of the following:
+
+any file in Source Code Form that results from an addition to, deletion from, or
+modification of the contents of Covered Software; or
+
+any new file in Source Code Form that contains any Covered Software.
+
+1.11. \u201cPatent Claims\u201d of a Contributor
+means any patent claim(s), including without limitation, method, process, and
+apparatus claims, in any patent Licensable by such Contributor that would be
+infringed, but for the grant of the License, by the making, using, selling,
+offering for sale, having made, import, or transfer of either its Contributions
+or its Contributor Version.
+
+1.12. \u201cSecondary License\u201d
+means either the GNU General Public License, Version 2.0, the GNU Lesser General
+Public License, Version 2.1, the GNU Affero General Public License, Version 3.0,
+or any later versions of those licenses.
+
+1.13. \u201cSource Code Form\u201d
+means the form of the work preferred for making modifications.
+
+1.14. \u201cYou\u201d (or \u201cYour\u201d)
+means an individual or a legal entity exercising rights under this License. For
+legal entities, \u201cYou\u201d includes any entity that controls, is controlled by,
+or is under common control with You. For purposes of this definition,
+\u201ccontrol\u201d means (a) the power, direct or indirect, to cause the direction or
+management of such entity, whether by contract or otherwise, or (b) ownership of
+more than fifty percent (50%) of the outstanding shares or beneficial ownership
+of such entity.
+
+2. License Grants and Conditions
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive
+license:
+
+under intellectual property rights (other than patent or trademark) Licensable
+by such Contributor to use, reproduce, make available, modify, display, perform,
+distribute, and otherwise exploit its Contributions, either on an unmodified
+basis, with Modifications, or as part of a Larger Work; and
+
+under Patent Claims of such Contributor to make, use, sell, offer for sale, have
+made, import, and otherwise transfer either its Contributions or its Contributor
+Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution become
+effective for each Contribution on the date the Contributor first distributes
+such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under this
+License. No additional rights or licenses will be implied from the distribution
+or licensing of Covered Software under this License. Notwithstanding Section
+2.1(b) above, no patent license is granted by a Contributor:
+
+for any code that a Contributor has removed from Covered Software; or
+
+for infringements caused by: (i) Your and any other third party\u2019s
+modifications of Covered Software, or (ii) the combination of its Contributions
+with other software (except as part of its Contributor Version); or
+
+under Patent Claims infringed by Covered Software in the absence of its
+Contributions.
+
+This License does not grant any rights in the trademarks, service marks, or
+logos of any Contributor (except as may be necessary to comply with the notice
+requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to distribute
+the Covered Software under a subsequent version of this License (see Section
+10.2) or under the terms of a Secondary License (if permitted under the terms of
+Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its Contributions are
+its original creation(s) or it has sufficient rights to grant the rights to its
+Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under applicable
+copyright doctrines of fair use, fair dealing, or other equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
+Section 2.1.
+
+3. Responsibilities
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under the
+terms of this License. You must inform recipients that the Source Code Form of
+the Covered Software is governed by the terms of this License, and how they can
+obtain a copy of this License. You may not attempt to alter or restrict the
+recipients\u2019 rights in the Source Code Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+such Covered Software must also be made available in Source Code Form, as
+described in Section 3.1, and You must inform recipients of the Executable Form
+how they can obtain a copy of such Source Code Form by reasonable means in a
+timely manner, at a charge no more than the cost of distribution to the
+recipient; and
+
+You may distribute such Executable Form under the terms of this License, or
+sublicense it under different terms, provided that the license for the
+Executable Form does not attempt to limit or alter the recipients\u2019 rights in
+the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice, provided
+that You also comply with the requirements of this License for the Covered
+Software. If the Larger Work is a combination of Covered Software with a work
+governed by one or more Secondary Licenses, and the Covered Software is not
+Incompatible With Secondary Licenses, this License permits You to additionally
+distribute such Covered Software under the terms of such Secondary License(s),
+so that the recipient of the Larger Work may, at their option, further
+distribute the Covered Software under the terms of either this License or such
+Secondary License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices (including
+copyright notices, patent notices, disclaimers of warranty, or limitations of
+liability) contained within the Source Code Form of the Covered Software, except
+that You may alter any license notices to the extent required to remedy known
+factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support, indemnity
+or liability obligations to one or more recipients of Covered Software. However,
+You may do so only on Your own behalf, and not on behalf of any Contributor. You
+must make it absolutely clear that any such warranty, support, indemnity, or
+liability obligation is offered by You alone, and You hereby agree to indemnify
+every Contributor for any liability incurred by such Contributor as a result of
+warranty, support, indemnity or liability terms You offer. You may include
+additional disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+
+If it is impossible for You to comply with any of the terms of this License with
+respect to some or all of the Covered Software due to statute, judicial order,
+or regulation then You must: (a) comply with the terms of this License to the
+maximum extent possible; and (b) describe the limitations and the code they
+affect. Such description must be placed in a text file included with all
+distributions of the Covered Software under this License. Except to the extent
+prohibited by statute or regulation, such description must be sufficiently
+detailed for a recipient of ordinary skill to be able to understand it.
+
+5. Termination
+
+5.1. The rights granted under this License will terminate automatically if You
+fail to comply with any of its terms. However, if You become compliant, then the
+rights granted under this License from a particular Contributor are reinstated
+(a) provisionally, unless and until such Contributor explicitly and finally
+terminates Your grants, and (b) on an ongoing basis, if such Contributor fails
+to notify You of the non-compliance by some reasonable means prior to 60 days
+after You have come back into compliance. Moreover, Your grants from a
+particular Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the first
+time You have received notice of non-compliance with this License from such
+Contributor, and You become compliant prior to 30 days after Your receipt of the
+notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions, counter-claims, and
+cross-claims) alleging that a Contributor Version directly or indirectly
+infringes any patent, then the rights granted to You by any and all Contributors
+for the Covered Software under Section 2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
+license agreements (excluding distributors and resellers) which have been
+validly granted by You or Your distributors under this License prior to
+termination shall survive termination.
+
+6. Disclaimer of Warranty
+
+Covered Software is provided under this License on an \u201cas is\u201d basis, without
+warranty of any kind, either expressed, implied, or statutory, including,
+without limitation, warranties that the Covered Software is free of defects,
+merchantable, fit for a particular purpose or non-infringing. The entire risk as
+to the quality and performance of the Covered Software is with You. Should any
+Covered Software prove defective in any respect, You (not any Contributor)
+assume the cost of any necessary servicing, repair, or correction. This
+disclaimer of warranty constitutes an essential part of this License. No use of
+any Covered Software is authorized under this License except under this
+disclaimer.
+
+7. Limitation of Liability
+
+Under no circumstances and under no legal theory, whether tort (including
+negligence), contract, or otherwise, shall any Contributor, or anyone who
+distributes Covered Software as permitted above, be liable to You for any
+direct, indirect, special, incidental, or consequential damages of any character
+including, without limitation, damages for lost profits, loss of goodwill, work
+stoppage, computer failure or malfunction, or any and all other commercial
+damages or losses, even if such party shall have been informed of the
+possibility of such damages. This limitation of liability shall not apply to
+liability for death or personal injury resulting from such party\u2019s negligence
+to the extent applicable law prohibits such limitation. Some jurisdictions do
+not allow the exclusion or limitation of incidental or consequential damages, so
+this exclusion and limitation may not apply to You.
+
+8. Litigation
+
+Any litigation relating to this License may be brought only in the courts of a
+jurisdiction where the defendant maintains its principal place of business and
+such litigation shall be governed by laws of that jurisdiction, without
+reference to its conflict-of-law provisions. Nothing in this Section shall
+prevent a party\u2019s ability to bring cross-claims or counter-claims.
+
+9. Miscellaneous
+
+This License represents the complete agreement concerning the subject matter
+hereof. If any provision of this License is held to be unenforceable, such
+provision shall be reformed only to the extent necessary to make it enforceable.
+Any law or regulation which provides that the language of a contract shall be
+construed against the drafter shall not be used to construe this License against
+a Contributor.
+
+10. Versions of the License
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section 10.3,
+no one other than the license steward has the right to modify or publish new
+versions of this License. Each version will be given a distinguishing version
+number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version of the
+License under which You originally received the Covered Software, or under the
+terms of any subsequent version published by the license steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to create a
+new license for such software, you may create and use a modified version of this
+License if you rename the license and remove any references to the name of the
+license steward (except to note that such modified license differs from this
+License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With Secondary
+Licenses under the terms of this version of the License, the notice described in
+Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+
+This Source Code Form is subject to the terms of the Mozilla Public License, v.
+2.0. If a copy of the MPL was not distributed with this file, You can obtain one
+at https://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular file, then
+You may include the notice in a location (such as a LICENSE file in a relevant
+directory) where a recipient would be likely to look for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - \u201cIncompatible With Secondary Licenses\u201d Notice
+
+This Source Code Form is \u201cIncompatible With Secondary Licenses\u201d, as defined
+by the Mozilla Public License, v. 2.0.
The binary distribution of this product bundles these dependencies under the
following license:
-D3 v3.5.6
+JDOM 1.1
--------------------------------------------------------------------------------
-(3-clause BSD license)
-All rights reserved.
+/*--
+
+ Copyright (C) 2000-2004 Jason Hunter & Brett McLaughlin.
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions, and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions, and the disclaimer that follows
+ these conditions in the documentation and/or other materials
+ provided with the distribution.
+
+ 3. The name "JDOM" must not be used to endorse or promote products
+ derived from this software without prior written permission. For
+ written permission, please contact <request_AT_jdom_DOT_org>.
+
+ 4. Products derived from this software may not be called "JDOM", nor
+ may "JDOM" appear in their name, without prior written permission
+ from the JDOM Project Management <request_AT_jdom_DOT_org>.
+
+ In addition, we request (but do not require) that you include in the
+ end-user documentation provided with the redistribution and/or in the
+ software itself an acknowledgement equivalent to the following:
+ "This product includes software developed by the
+ JDOM Project (http://www.jdom.org/)."
+ Alternatively, the acknowledgment may be graphical using the logos
+ available at http://www.jdom.org/images/logos.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ SUCH DAMAGE.
+
+ This software consists of voluntary contributions made by many
+ individuals on behalf of the JDOM Project and was originally
+ created by Jason Hunter <jhunter_AT_jdom_DOT_org> and
+ Brett McLaughlin <brett_AT_jdom_DOT_org>. For more information
+ on the JDOM Project, please see <http://www.jdom.org/>.
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this list
- of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice, this
- list of conditions and the following disclaimer in the documentation and/or
- other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its contributors may
- be used to endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
-IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
-INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
-NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
+ */
+
+The binary distribution of this product bundles these dependencies under the
+following license:
+Hbase Server 1.2.4
+--------------------------------------------------------------------------------
+This project bundles a derivative image for our Orca Logo. This image is
+available under the Creative Commons By Attribution 3.0 License.
+
+ Creative Commons Legal Code
+
+ Attribution 3.0 Unported
+
+ CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+ LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN
+ ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+ INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+ REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR
+ DAMAGES RESULTING FROM ITS USE.
+
+ License
+
+ THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE
+ COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY
+ COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS
+ AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.
+
+ BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE
+ TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY
+ BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS
+ CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND
+ CONDITIONS.
+
+ 1. Definitions
+
+ a. "Adaptation" means a work based upon the Work, or upon the Work and
+ other pre-existing works, such as a translation, adaptation,
+ derivative work, arrangement of music or other alterations of a
+ literary or artistic work, or phonogram or performance and includes
+ cinematographic adaptations or any other form in which the Work may be
+ recast, transformed, or adapted including in any form recognizably
+ derived from the original, except that a work that constitutes a
+ Collection will not be considered an Adaptation for the purpose of
+ this License. For the avoidance of doubt, where the Work is a musical
+ work, performance or phonogram, the synchronization of the Work in
+ timed-relation with a moving image ("synching") will be considered an
+ Adaptation for the purpose of this License.
+ b. "Collection" means a collection of literary or artistic works, such as
+ encyclopedias and anthologies, or performances, phonograms or
+ broadcasts, or other works or subject matter other than works listed
+ in Section 1(f) below, which, by reason of the selection and
+ arrangement of their contents, constitute intellectual creations, in
+ which the Work is included in its entirety in unmodified form along
+ with one or more other contributions, each constituting separate and
+ independent works in themselves, which together are assembled into a
+ collective whole. A work that constitutes a Collection will not be
+ considered an Adaptation (as defined above) for the purposes of this
+ License.
+ c. "Distribute" means to make available to the public the original and
+ copies of the Work or Adaptation, as appropriate, through sale or
+ other transfer of ownership.
+ d. "Licensor" means the individual, individuals, entity or entities that
+ offer(s) the Work under the terms of this License.
+ e. "Original Author" means, in the case of a literary or artistic work,
+ the individual, individuals, entity or entities who created the Work
+ or if no individual or entity can be identified, the publisher; and in
+ addition (i) in the case of a performance the actors, singers,
+ musicians, dancers, and other persons who act, sing, deliver, declaim,
+ play in, interpret or otherwise perform literary or artistic works or
+ expressions of folklore; (ii) in the case of a phonogram the producer
+ being the person or legal entity who first fixes the sounds of a
+ performance or other sounds; and, (iii) in the case of broadcasts, the
+ organization that transmits the broadcast.
+ f. "Work" means the literary and/or artistic work offered under the terms
+ of this License including without limitation any production in the
+ literary, scientific and artistic domain, whatever may be the mode or
+ form of its expression including digital form, such as a book,
+ pamphlet and other writing; a lecture, address, sermon or other work
+ of the same nature; a dramatic or dramatico-musical work; a
+ choreographic work or entertainment in dumb show; a musical
+ composition with or without words; a cinematographic work to which are
+ assimilated works expressed by a process analogous to cinematography;
+ a work of drawing, painting, architecture, sculpture, engraving or
+ lithography; a photographic work to which are assimilated works
+ expressed by a process analogous to photography; a work of applied
+ art; an illustration, map, plan, sketch or three-dimensional work
+ relative to geography, topography, architecture or science; a
+ performance; a broadcast; a phonogram; a compilation of data to the
+ extent it is protected as a copyrightable work; or a work performed by
+ a variety or circus performer to the extent it is not otherwise
+ considered a literary or artistic work.
+ g. "You" means an individual or entity exercising rights under this
+ License who has not previously violated the terms of this License with
+ respect to the Work, or who has received express permission from the
+ Licensor to exercise rights under this License despite a previous
+ violation.
+ h. "Publicly Perform" means to perform public recitations of the Work and
+ to communicate to the public those public recitations, by any means or
+ process, including by wire or wireless means or public digital
+ performances; to make available to the public Works in such a way that
+ members of the public may access these Works from a place and at a
+ place individually chosen by them; to perform the Work to the public
+ by any means or process and the communication to the public of the
+ performances of the Work, including by public digital performance; to
+ broadcast and rebroadcast the Work by any means including signs,
+ sounds or images.
+ i. "Reproduce" means to make copies of the Work by any means including
+ without limitation by sound or visual recordings and the right of
+ fixation and reproducing fixations of the Work, including storage of a
+ protected performance or phonogram in digital form or other electronic
+ medium.
+
+ 2. Fair Dealing Rights. Nothing in this License is intended to reduce,
+ limit, or restrict any uses free from copyright or rights arising from
+ limitations or exceptions that are provided for in connection with the
+ copyright protection under copyright law or other applicable laws.
+
+ 3. License Grant. Subject to the terms and conditions of this License,
+ Licensor hereby grants You a worldwide, royalty-free, non-exclusive,
+ perpetual (for the duration of the applicable copyright) license to
+ exercise the rights in the Work as stated below:
+
+ a. to Reproduce the Work, to incorporate the Work into one or more
+ Collections, and to Reproduce the Work as incorporated in the
+ Collections;
+ b. to create and Reproduce Adaptations provided that any such Adaptation,
+ including any translation in any medium, takes reasonable steps to
+ clearly label, demarcate or otherwise identify that changes were made
+ to the original Work. For example, a translation could be marked "The
+ original work was translated from English to Spanish," or a
+ modification could indicate "The original work has been modified.";
+ c. to Distribute and Publicly Perform the Work including as incorporated
+ in Collections; and,
+ d. to Distribute and Publicly Perform Adaptations.
+ e. For the avoidance of doubt:
+
+ i. Non-waivable Compulsory License Schemes. In those jurisdictions in
+ which the right to collect royalties through any statutory or
+ compulsory licensing scheme cannot be waived, the Licensor
+ reserves the exclusive right to collect such royalties for any
+ exercise by You of the rights granted under this License;
+ ii. Waivable Compulsory License Schemes. In those jurisdictions in
+ which the right to collect royalties through any statutory or
+ compulsory licensing scheme can be waived, the Licensor waives the
+ exclusive right to collect such royalties for any exercise by You
+ of the rights granted under this License; and,
+ iii. Voluntary License Schemes. The Licensor waives the right to
+ collect royalties, whether individually or, in the event that the
+ Licensor is a member of a collecting society that administers
+ voluntary licensing schemes, via that society, from any exercise
+ by You of the rights granted under this License.
+
+ The above rights may be exercised in all media and formats whether now
+ known or hereafter devised. The above rights include the right to make
+ such modifications as are technically necessary to exercise the rights in
+ other media and formats. Subject to Section 8(f), all rights not expressly
+ granted by Licensor are hereby reserved.
+
+ 4. Restrictions. The license granted in Section 3 above is expressly made
+ subject to and limited by the following restrictions:
+
+ a. You may Distribute or Publicly Perform the Work only under the terms
+ of this License. You must include a copy of, or the Uniform Resource
+ Identifier (URI) for, this License with every copy of the Work You
+ Distribute or Publicly Perform. You may not offer or impose any terms
+ on the Work that restrict the terms of this License or the ability of
+ the recipient of the Work to exercise the rights granted to that
+ recipient under the terms of the License. You may not sublicense the
+ Work. You must keep intact all notices that refer to this License and
+ to the disclaimer of warranties with every copy of the Work You
+ Distribute or Publicly Perform. When You Distribute or Publicly
+ Perform the Work, You may not impose any effective technological
+ measures on the Work that restrict the ability of a recipient of the
+ Work from You to exercise the rights granted to that recipient under
+ the terms of the License. This Section 4(a) applies to the Work as
+ incorporated in a Collection, but this does not require the Collection
+ apart from the Work itself to be made subject to the terms of this
+ License. If You create a Collection, upon notice from any Licensor You
+ must, to the extent practicable, remove from the Collection any credit
+ as required by Section 4(b), as requested. If You create an
+ Adaptation, upon notice from any Licensor You must, to the extent
+ practicable, remove from the Adaptation any credit as required by
+ Section 4(b), as requested.
+ b. If You Distribute, or Publicly Perform the Work or any Adaptations or
+ Collections, You must, unless a request has been made pursuant to
+ Section 4(a), keep intact all copyright notices for the Work and
+ provide, reasonable to the medium or means You are utilizing: (i) the
+ name of the Original Author (or pseudonym, if applicable) if supplied,
+ and/or if the Original Author and/or Licensor designate another party
+ or parties (e.g., a sponsor institute, publishing entity, journal) for
+ attribution ("Attribution Parties") in Licensor's copyright notice,
+ terms of service or by other reasonable means, the name of such party
+ or parties; (ii) the title of the Work if supplied; (iii) to the
+ extent reasonably practicable, the URI, if any, that Licensor
+ specifies to be associated with the Work, unless such URI does not
+ refer to the copyright notice or licensing information for the Work;
+ and (iv) , consistent with Section 3(b), in the case of an Adaptation,
+ a credit identifying the use of the Work in the Adaptation (e.g.,
+ "French translation of the Work by Original Author," or "Screenplay
+ based on original Work by Original Author"). The credit required by
+ this Section 4 (b) may be implemented in any reasonable manner;
+ provided, however, that in the case of a Adaptation or Collection, at
+ a minimum such credit will appear, if a credit for all contributing
+ authors of the Adaptation or Collection appears, then as part of these
+ credits and in a manner at least as prominent as the credits for the
+ other contributing authors. For the avoidance of doubt, You may only
+ use the credit required by this Section for the purpose of attribution
+ in the manner set out above and, by exercising Your rights under this
+ License, You may not implicitly or explicitly assert or imply any
+ connection with, sponsorship or endorsement by the Original Author,
+ Licensor and/or Attribution Parties, as appropriate, of You or Your
+ use of the Work, without the separate, express prior written
+ permission of the Original Author, Licensor and/or Attribution
+ Parties.
+ c. Except as otherwise agreed in writing by the Licensor or as may be
+ otherwise permitted by applicable law, if You Reproduce, Distribute or
+ Publicly Perform the Work either by itself or as part of any
+ Adaptations or Collections, You must not distort, mutilate, modify or
+ take other derogatory action in relation to the Work which would be
+ prejudicial to the Original Author's honor or reputation. Licensor
+ agrees that in those jurisdictions (e.g. Japan), in which any exercise
+ of the right granted in Section 3(b) of this License (the right to
+ make Adaptations) would be deemed to be a distortion, mutilation,
+ modification or other derogatory action prejudicial to the Original
+ Author's honor and reputation, the Licensor will waive or not assert,
+ as appropriate, this Section, to the fullest extent permitted by the
+ applicable national law, to enable You to reasonably exercise Your
+ right under Section 3(b) of this License (right to make Adaptations)
+ but not otherwise.
+
+ 5. Representations, Warranties and Disclaimer
+
+ UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR
+ OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY
+ KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE,
+ INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY,
+ FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF
+ LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS,
+ WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION
+ OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
+
+ 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE
+ LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR
+ ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES
+ ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS
+ BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+ 7. Termination
+
+ a. This License and the rights granted hereunder will terminate
+ automatically upon any breach by You of the terms of this License.
+ Individuals or entities who have received Adaptations or Collections
+ from You under this License, however, will not have their licenses
+ terminated provided such individuals or entities remain in full
+ compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will
+ survive any termination of this License.
+ b. Subject to the above terms and conditions, the license granted here is
+ perpetual (for the duration of the applicable copyright in the Work).
+ Notwithstanding the above, Licensor reserves the right to release the
+ Work under different license terms or to stop distributing the Work at
+ any time; provided, however that any such election will not serve to
+ withdraw this License (or any other license that has been, or is
+ required to be, granted under the terms of this License), and this
+ License will continue in full force and effect unless terminated as
+ stated above.
+
+ 8. Miscellaneous
+
+ a. Each time You Distribute or Publicly Perform the Work or a Collection,
+ the Licensor offers to the recipient a license to the Work on the same
+ terms and conditions as the license granted to You under this License.
+ b. Each time You Distribute or Publicly Perform an Adaptation, Licensor
+ offers to the recipient a license to the original Work on the same
+ terms and conditions as the license granted to You under this License.
+ c. If any provision of this License is invalid or unenforceable under
+ applicable law, it shall not affect the validity or enforceability of
+ the remainder of the terms of this License, and without further action
+ by the parties to this agreement, such provision shall be reformed to
+ the minimum extent necessary to make such provision valid and
+ enforceable.
+ d. No term or provision of this License shall be deemed waived and no
+ breach consented to unless such waiver or consent shall be in writing
+ and signed by the party to be charged with such waiver or consent.
+ e. This License constitutes the entire agreement between the parties with
+ respect to the Work licensed here. There are no understandings,
+ agreements or representations with respect to the Work not specified
+ here. Licensor shall not be bound by any additional provisions that
+ may appear in any communication from You. This License may not be
+ modified without the mutual written agreement of the Licensor and You.
+ f. The rights granted under, and the subject matter referenced, in this
+ License were drafted utilizing the terminology of the Berne Convention
+ for the Protection of Literary and Artistic Works (as amended on
+ September 28, 1979), the Rome Convention of 1961, the WIPO Copyright
+ Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996
+ and the Universal Copyright Convention (as revised on July 24, 1971).
+ These rights and subject matter take effect in the relevant
+ jurisdiction in which the License terms are sought to be enforced
+ according to the corresponding provisions of the implementation of
+ those treaty provisions in the applicable national law. If the
+ standard suite of rights granted under applicable copyright law
+ includes additional rights not granted under this License, such
+ additional rights are deemed to be included in the License; this
+ License is not intended to restrict the license of any rights under
+ applicable law.
+
+
+ Creative Commons Notice
+
+ Creative Commons is not a party to this License, and makes no warranty
+ whatsoever in connection with the Work. Creative Commons will not be
+ liable to You or any party on any legal theory for any damages
+ whatsoever, including without limitation any general, special,
+ incidental or consequential damages arising in connection to this
+ license. Notwithstanding the foregoing two (2) sentences, if Creative
+ Commons has expressly identified itself as the Licensor hereunder, it
+ shall have all rights and obligations of Licensor.
+
+ Except for the limited purpose of indicating to the public that the
+ Work is licensed under the CCPL, Creative Commons does not authorize
+ the use by either party of the trademark "Creative Commons" or any
+ related trademark or logo of Creative Commons without the prior
+ written consent of Creative Commons. Any permitted use will be in
+ compliance with Creative Commons' then-current trademark usage
+ guidelines, as may be published on its website or otherwise made
+ available upon request from time to time. For the avoidance of doubt,
+ this trademark restriction does not form part of this License.
+
+ Creative Commons may be contacted at https://creativecommons.org/.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8850c056/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index 33c12ed..c41972b 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -17,7 +17,11 @@ which has the following notices:
* This product includes software developed by IBM Corporation and others.
The binary distribution of this product bundles binaries of
-AWS Java SDK 1.10.6,
+AWS SDK for Java - Core 1.11.45,
+AWS Java SDK for AWS KMS 1.11.45,
+AWS Java SDK for Amazon S3 1.11.45,
+AWS Java SDK for AWS STS 1.11.45,
+JMES Path Query library 1.0,
which has the following notices:
* This software includes third party software subject to the following
copyrights: - XML parsing and utility functions from JetS3t - Copyright
@@ -257,6 +261,13 @@ provides utilities for the java.lang API, which can be obtained at:
* HOMEPAGE:
* https://commons.apache.org/proper/commons-lang/
+This product contains a modified portion of 'JDOM 1.1', which can be obtained at:
+
+ * LICENSE:
+ * https://github.com/hunterhacker/jdom/blob/jdom-1.1/core/LICENSE.txt
+ * HOMEPAGE:
+ * http://www.jdom.org/
+
The binary distribution of this product bundles binaries of
Commons Codec 1.4,
which has the following notices:
@@ -283,7 +294,7 @@ which has the following notices:
Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams
The binary distribution of this product bundles binaries of
-Java Concurrency in Practice book annotations 1.0,
+"Java Concurrency in Practice" book annotations 1.0,
which has the following notices:
* Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative
Commons Attribution License (http://creativecommons.org/licenses/by/2.5)
@@ -292,7 +303,15 @@ which has the following notices:
notice.
The binary distribution of this product bundles binaries of
-Jetty 6.1.26,
+Jetty :: Http Utility 9.3.11.,
+Jetty :: IO Utility 9.3.11.,
+Jetty :: Security 9.3.11.,
+Jetty :: Server Core 9.3.11.,
+Jetty :: Servlet Handling 9.3.11.,
+Jetty :: Utilities 9.3.11.,
+Jetty :: Utilities :: Ajax,
+Jetty :: Webapp Application Support 9.3.11.,
+Jetty :: XML utilities 9.3.11.,
which has the following notices:
* ==============================================================
Jetty Web Container
@@ -453,3 +472,107 @@ which has the following notices:
- voluntary contributions made by Paul Eng on behalf of the
Apache Software Foundation that were originally developed at iClick, Inc.,
software copyright (c) 1999.
+
+The binary distribution of this product bundles binaries of
+Logback Classic Module 1.1.2,
+Logback Core Module 1.1.2,
+which has the following notices:
+ * Logback: the reliable, generic, fast and flexible logging framework.
+ Copyright (C) 1999-2012, QOS.ch. All rights reserved.
+
+The binary distribution of this product bundles binaries of
+Apache HBase - Annotations 1.2.4,
+Apache HBase - Client 1.2.4,
+Apache HBase - Common 1.2.4,
+Apache HBase - Hadoop Compatibility 1.2.4,
+Apache HBase - Hadoop Two Compatibility 1.2.4,
+Apache HBase - Prefix Tree 1.2.4,
+Apache HBase - Procedure 1.2.4,
+Apache HBase - Protocol 1.2.4,
+Apache HBase - Server 1.2.4,
+which has the following notices:
+ * Apache HBase
+ Copyright 2007-2015 The Apache Software Foundation
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+ --
+ This product incorporates portions of the 'Hadoop' project
+
+ Copyright 2007-2009 The Apache Software Foundation
+
+ Licensed under the Apache License v2.0
+ --
+ Our Orca logo we got here: http://www.vectorfree.com/jumping-orca
+ It is licensed Creative Commons Attribution 3.0.
+ See https://creativecommons.org/licenses/by/3.0/us/
+ We changed the logo by stripping the colored background, inverting
+ it and then rotating it some.
+
+ Later we found that vectorfree.com image is not properly licensed.
+ The original is owned by vectorportal.com. The original was
+ relicensed so we could use it as Creative Commons Attribution 3.0.
+ The license is bundled with the download available here:
+ http://www.vectorportal.com/subcategory/205/KILLER-WHALE-FREE-VECTOR.eps/ifile/9136/detailtest.asp
+ --
+ This product includes portions of the Bootstrap project v3.0.0
+
+ Copyright 2013 Twitter, Inc.
+
+ Licensed under the Apache License v2.0
+
+ This product uses the Glyphicons Halflings icon set.
+
+ http://glyphicons.com/
+
+ Copyright Jan Kova\u0159�k
+
+ Licensed under the Apache License v2.0 as a part of the Bootstrap project.
+
+ --
+ This product includes portions of the Guava project v14, specifically
+ 'hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java'
+
+ Copyright (C) 2007 The Guava Authors
+
+ Licensed under the Apache License, Version 2.0
+
+The binary distribution of this product bundles binaries of
+Phoenix Core 4.7.0,
+which has the following notices:
+ Apache Phoenix
+ Copyright 2013-2016 The Apache Software Foundation
+
+ This product includes software developed by The Apache Software
+ Foundation (http://www.apache.org/).
+
+ This also includes:
+
+ The phoenix-spark module has been adapted from the phoenix-spark library
+ distributed under the terms of the Apache 2 license. Original source copyright:
+ Copyright 2014 Simply Measured, Inc.
+ Copyright 2015 Interset Software Inc.
+
+ The file bin/daemon.py is based on the file of the same name in python-daemon 2.0.5
+ (https://pypi.python.org/pypi/python-daemon/). Original source copyright:
+ # Copyright � 2008\u20132015 Ben Finney <be...@benfinney.id.au>
+ # Copyright � 2007\u20132008 Robert Niederreiter, Jens Klein
+ # Copyright � 2004\u20132005 Chad J. Schroeder
+ # Copyright � 2003 Clark Evans
+ # Copyright � 2002 Noah Spurrier
+ # Copyright � 2001 J�rgen Hermann
+
+The binary distribution of this product bundles binaries of
+Plexus Cipher: encryption/decryption Component 1.4,
+which has the following notices:
+ * The code in this component contains a class - Base64 taken from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/Base64.java
+ which is Apache license: http://www.apache.org/licenses/LICENSE-2.0
+
+ The PBE key processing routine PBECipher.createCipher() is adopted from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/OpenSSL.java
+ which is also Apache APL-2.0 license: http://www.apache.org/licenses/LICENSE-2.0
+
+The binary distribution of this product bundles binaries of
+software.amazon.ion:ion-java 1.0.1,
+which has the following notices:
+ * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-3955. Support for application
priority ACLs in queues of CapacityScheduler. (Sunil G via wangda)
Posted by st...@apache.org.
YARN-3955. Support for application priority ACLs in queues of CapacityScheduler. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/287d3d68
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/287d3d68
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/287d3d68
Branch: refs/heads/HADOOP-13345
Commit: 287d3d6804a869723ae36605a3c2d2b3eae3941e
Parents: db490ec
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Jan 9 08:40:39 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Jan 9 08:40:39 2017 -0800
----------------------------------------------------------------------
.../sls/scheduler/ResourceSchedulerWrapper.java | 2 +-
.../apache/hadoop/yarn/security/AccessType.java | 2 +
.../conf/capacity-scheduler.xml | 9 +
.../server/resourcemanager/ClientRMService.java | 3 +-
.../server/resourcemanager/RMAppManager.java | 16 +-
.../scheduler/AbstractYarnScheduler.java | 10 +-
.../scheduler/YarnScheduler.java | 11 +-
.../AppPriorityACLConfigurationParser.java | 219 ++++++++++++++++++
.../scheduler/capacity/AppPriorityACLGroup.java | 108 +++++++++
.../scheduler/capacity/CapacityScheduler.java | 153 +++++++-----
.../CapacitySchedulerConfiguration.java | 44 +++-
.../capacity/CapacitySchedulerContext.java | 7 +
.../capacity/CapacitySchedulerQueueManager.java | 23 +-
.../scheduler/capacity/LeafQueue.java | 16 ++
.../security/AppPriorityACLsManager.java | 230 +++++++++++++++++++
.../server/resourcemanager/ACLsTestBase.java | 5 +-
.../TestApplicationMasterService.java | 5 +-
.../resourcemanager/TestClientRMService.java | 2 +-
.../capacity/TestApplicationPriority.java | 25 +-
...TestApplicationPriorityACLConfiguration.java | 120 ++++++++++
.../capacity/TestApplicationPriorityACLs.java | 206 +++++++++++++++++
.../scheduler/capacity/TestParentQueue.java | 6 +-
22 files changed, 1127 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
index e66de2f..5517362 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
@@ -963,7 +963,7 @@ final public class ResourceSchedulerWrapper
@Override
public Priority checkAndGetApplicationPriority(Priority priority,
- String user, String queueName, ApplicationId applicationId)
+ UserGroupInformation user, String queueName, ApplicationId applicationId)
throws YarnException {
// TODO Dummy implementation.
return Priority.newInstance(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java
index 32459b9..fb4484b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java
@@ -30,4 +30,6 @@ public enum AccessType {
// queue
SUBMIT_APP,
ADMINISTER_QUEUE,
+ // application
+ APPLICATION_MAX_PRIORITY,
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
index 6ac726e..47db01f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
@@ -98,6 +98,15 @@
</property>
<property>
+ <name>yarn.scheduler.capacity.root.default.acl_application_max_priority</name>
+ <value>*</value>
+ <description>
+ The ACL of who can submit applications with configured priority.
+ For e.g, [user={name} group={name} max_priority={priority} default_priority={priority}]
+ </description>
+ </property>
+
+ <property>
<name>yarn.scheduler.capacity.node-locality-delay</name>
<value>40</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index cdf30a1..add522b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1614,7 +1614,8 @@ public class ClientRMService extends AbstractService implements
}
try {
- rmAppManager.updateApplicationPriority(applicationId, newAppPriority);
+ rmAppManager.updateApplicationPriority(callerUGI, applicationId,
+ newAppPriority);
} catch (YarnException ex) {
RMAuditLogger.logFailure(callerUGI.getShortUserName(),
AuditConstants.UPDATE_APP_PRIORITY, "UNKNOWN", "ClientRMService",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 8232b88..47eec54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -353,19 +353,19 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
RMServerUtils.validateApplicationTimeouts(
submissionContext.getApplicationTimeouts());
}
-
+
ApplicationId applicationId = submissionContext.getApplicationId();
ResourceRequest amReq =
validateAndCreateResourceRequest(submissionContext, isRecovery);
// Verify and get the update application priority and set back to
// submissionContext
+ UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
Priority appPriority = scheduler.checkAndGetApplicationPriority(
- submissionContext.getPriority(), user, submissionContext.getQueue(),
+ submissionContext.getPriority(), userUgi, submissionContext.getQueue(),
applicationId);
submissionContext.setPriority(appPriority);
- UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
// Since FairScheduler queue mapping is done inside scheduler,
// if FairScheduler is used and the queue doesn't exist, we should not
// fail here because queue will be created inside FS. Ideally, FS queue
@@ -569,12 +569,14 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
/**
* updateApplicationPriority will invoke scheduler api to update the
* new priority to RM and StateStore.
+ * @param callerUGI user
* @param applicationId Application Id
* @param newAppPriority proposed new application priority
* @throws YarnException Handle exceptions
*/
- public void updateApplicationPriority(ApplicationId applicationId,
- Priority newAppPriority) throws YarnException {
+ public void updateApplicationPriority(UserGroupInformation callerUGI,
+ ApplicationId applicationId, Priority newAppPriority)
+ throws YarnException {
RMApp app = this.rmContext.getRMApps().get(applicationId);
synchronized (applicationId) {
@@ -587,8 +589,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
// Invoke scheduler api to update priority in scheduler and to
// State Store.
- Priority appPriority = rmContext.getScheduler()
- .updateApplicationPriority(newAppPriority, applicationId, future);
+ Priority appPriority = rmContext.getScheduler().updateApplicationPriority(
+ newAppPriority, applicationId, future, callerUGI);
if (app.getApplicationPriority().equals(appPriority)) {
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 89f9ffa..9ff0b24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -35,6 +35,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -788,9 +789,9 @@ public abstract class AbstractYarnScheduler
}
@Override
- public Priority checkAndGetApplicationPriority(Priority priorityFromContext,
- String user, String queueName, ApplicationId applicationId)
- throws YarnException {
+ public Priority checkAndGetApplicationPriority(
+ Priority priorityRequestedByApp, UserGroupInformation user,
+ String queueName, ApplicationId applicationId) throws YarnException {
// Dummy Implementation till Application Priority changes are done in
// specific scheduler.
return Priority.newInstance(0);
@@ -798,7 +799,8 @@ public abstract class AbstractYarnScheduler
@Override
public Priority updateApplicationPriority(Priority newPriority,
- ApplicationId applicationId, SettableFuture<Object> future)
+ ApplicationId applicationId, SettableFuture<Object> future,
+ UserGroupInformation user)
throws YarnException {
// Dummy Implementation till Application Priority changes are done in
// specific scheduler.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 5579efd..08e0603 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -307,7 +307,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
* Verify whether a submitted application priority is valid as per configured
* Queue
*
- * @param priorityFromContext
+ * @param priorityRequestedByApp
* Submitted Application priority.
* @param user
* User who submitted the Application
@@ -317,8 +317,8 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
* Application ID
* @return Updated Priority from scheduler
*/
- public Priority checkAndGetApplicationPriority(Priority priorityFromContext,
- String user, String queueName, ApplicationId applicationId)
+ public Priority checkAndGetApplicationPriority(Priority priorityRequestedByApp,
+ UserGroupInformation user, String queueName, ApplicationId applicationId)
throws YarnException;
/**
@@ -330,12 +330,13 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
* @param applicationId Application ID
*
* @param future Sets any type of exception happened from StateStore
+ * @param user who submitted the application
*
* @return updated priority
*/
public Priority updateApplicationPriority(Priority newPriority,
- ApplicationId applicationId, SettableFuture<Object> future)
- throws YarnException;
+ ApplicationId applicationId, SettableFuture<Object> future,
+ UserGroupInformation user) throws YarnException;
/**
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AppPriorityACLConfigurationParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AppPriorityACLConfigurationParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AppPriorityACLConfigurationParser.java
new file mode 100644
index 0000000..4489a01
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AppPriorityACLConfigurationParser.java
@@ -0,0 +1,219 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.Priority;
+
+/**
+ *
+ * PriorityACLConfiguration class is used to parse Application Priority ACL
+ * configuration from capcity-scheduler.xml
+ */
+public class AppPriorityACLConfigurationParser {
+
+ private static final Log LOG = LogFactory
+ .getLog(AppPriorityACLConfigurationParser.class);
+
+ public enum AppPriorityACLKeyType {
+ USER(1), GROUP(2), MAX_PRIORITY(3), DEFAULT_PRIORITY(4);
+
+ private final int id;
+
+ AppPriorityACLKeyType(int id) {
+ this.id = id;
+ }
+
+ public int getId() {
+ return this.id;
+ }
+ }
+
+ public static final String PATTERN_FOR_PRIORITY_ACL = "\\[([^\\]]+)";
+
+ @Private
+ public static final String ALL_ACL = "*";
+
+ @Private
+ public static final String NONE_ACL = " ";
+
+ public List<AppPriorityACLGroup> getPriorityAcl(Priority clusterMaxPriority,
+ String aclString) {
+
+ List<AppPriorityACLGroup> aclList = new ArrayList<AppPriorityACLGroup>();
+ Matcher matcher = Pattern.compile(PATTERN_FOR_PRIORITY_ACL)
+ .matcher(aclString);
+
+ /*
+ * Each ACL group will be separated by "[]". Syntax of each ACL group could
+ * be like below "user=b1,b2 group=g1 max-priority=a2 default-priority=a1"
+ * Ideally this means "for this given user/group, maximum possible priority
+ * is a2 and if the user has not specified any priority, then it is a1."
+ */
+ while (matcher.find()) {
+ // Get the first ACL sub-group.
+ String aclSubGroup = matcher.group(1);
+ if (aclSubGroup.trim().isEmpty()) {
+ continue;
+ }
+
+ /*
+ * Internal storage is PriorityACLGroup which stores each parsed priority
+ * ACLs group. This will help while looking for a user to priority mapping
+ * during app submission time. ACLs will be passed in below order only. 1.
+ * user/group 2. max-priority 3. default-priority
+ */
+ AppPriorityACLGroup userPriorityACL = new AppPriorityACLGroup();
+
+ // userAndGroupName will hold user acl and group acl as interim storage
+ // since both user/group acl comes with separate key value pairs.
+ List<StringBuilder> userAndGroupName = new ArrayList<>();
+
+ for (String kvPair : aclSubGroup.trim().split(" +")) {
+ /*
+ * There are 3 possible options for key here: 1. user/group 2.
+ * max-priority 3. default-priority
+ */
+ String[] splits = kvPair.split("=");
+
+ // Ensure that each ACL sub string is key value pair separated by '='.
+ if (splits != null && splits.length > 1) {
+ parsePriorityACLType(userPriorityACL, splits, userAndGroupName);
+ }
+ }
+
+ // If max_priority is higher to clusterMaxPriority, its better to
+ // handle here.
+ if (userPriorityACL.getMaxPriority().getPriority() > clusterMaxPriority
+ .getPriority()) {
+ LOG.warn("ACL configuration for '" + userPriorityACL.getMaxPriority()
+ + "' is greater that cluster max priority. Resetting ACLs to "
+ + clusterMaxPriority);
+ userPriorityACL.setMaxPriority(
+ Priority.newInstance(clusterMaxPriority.getPriority()));
+ }
+
+ AccessControlList acl = createACLStringForPriority(userAndGroupName);
+ userPriorityACL.setACLList(acl);
+ aclList.add(userPriorityACL);
+ }
+
+ return aclList;
+ }
+
+ /*
+ * Parse different types of ACLs sub parts for on priority group and store in
+ * a map for later processing.
+ */
+ private void parsePriorityACLType(AppPriorityACLGroup userPriorityACL,
+ String[] splits, List<StringBuilder> userAndGroupName) {
+ // Here splits will have the key value pair at index 0 and 1 respectively.
+ // To parse all keys, its better to convert to PriorityACLConfig enum.
+ AppPriorityACLKeyType aclType = AppPriorityACLKeyType
+ .valueOf(StringUtils.toUpperCase(splits[0].trim()));
+ switch (aclType) {
+ case MAX_PRIORITY :
+ userPriorityACL
+ .setMaxPriority(Priority.newInstance(Integer.parseInt(splits[1])));
+ break;
+ case USER :
+ userAndGroupName.add(getUserOrGroupACLStringFromConfig(splits[1]));
+ break;
+ case GROUP :
+ userAndGroupName.add(getUserOrGroupACLStringFromConfig(splits[1]));
+ break;
+ case DEFAULT_PRIORITY :
+ int defaultPriority = Integer.parseInt(splits[1]);
+ Priority priority = (defaultPriority < 0)
+ ? Priority.newInstance(0)
+ : Priority.newInstance(defaultPriority);
+ userPriorityACL.setDefaultPriority(priority);
+ break;
+ default:
+ break;
+ }
+ }
+
+ /*
+ * This method will help to append different types of ACLs keys against one
+ * priority. For eg,USER will be appended with GROUP as "user2,user4 group1".
+ */
+ private AccessControlList createACLStringForPriority(
+ List<StringBuilder> acls) {
+
+ String finalACL = "";
+ String userACL = acls.get(0).toString();
+
+ // If any of user/group is *, consider it as acceptable for all.
+ // "user" is at index 0, and "group" is at index 1.
+ if (userACL.trim().equals(ALL_ACL)) {
+ finalACL = ALL_ACL;
+ } else if (userACL.equals(NONE_ACL)) {
+ finalACL = NONE_ACL;
+ } else {
+
+ // Get USER segment
+ if (!userACL.trim().isEmpty()) {
+ // skip last appended ","
+ finalACL = acls.get(0).toString();
+ }
+
+ // Get GROUP segment if any
+ if (acls.size() > 1) {
+ String groupACL = acls.get(1).toString();
+ if (!groupACL.trim().isEmpty()) {
+ finalACL = finalACL + " "
+ + acls.get(1).toString();
+ }
+ }
+ }
+
+ // Here ACL will look like "user1,user2 group" in ideal cases.
+ return new AccessControlList(finalACL.trim());
+ }
+
+ /*
+ * This method will help to append user/group acl string against given
+ * priority. For example "user1,user2 group1,group2"
+ */
+ private StringBuilder getUserOrGroupACLStringFromConfig(String value) {
+
+ // ACL strings could be generate for USER or GRUOP.
+ // aclList in map contains two entries. 1. USER, 2. GROUP.
+ StringBuilder aclTypeName = new StringBuilder();
+
+ if (value.trim().equals(ALL_ACL)) {
+ aclTypeName.setLength(0);
+ aclTypeName.append(ALL_ACL);
+ return aclTypeName;
+ }
+
+ aclTypeName.append(value.trim());
+ return aclTypeName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AppPriorityACLGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AppPriorityACLGroup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AppPriorityACLGroup.java
new file mode 100644
index 0000000..cb5ebcb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AppPriorityACLGroup.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.api.records.Priority;
+
+/**
+ * PriorityACLGroup will hold all ACL related information per priority.
+ *
+ */
+public class AppPriorityACLGroup implements Comparable<AppPriorityACLGroup> {
+
+ private Priority maxPriority = null;
+ private Priority defaultPriority = null;
+ private AccessControlList aclList = null;
+
+ public AppPriorityACLGroup(Priority maxPriority, Priority defaultPriority,
+ AccessControlList aclList) {
+ this.setMaxPriority(Priority.newInstance(maxPriority.getPriority()));
+ this.setDefaultPriority(
+ Priority.newInstance(defaultPriority.getPriority()));
+ this.setACLList(aclList);
+ }
+
+ public AppPriorityACLGroup() {
+ }
+
+ @Override
+ public int compareTo(AppPriorityACLGroup o) {
+ return getMaxPriority().compareTo(o.getMaxPriority());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+
+ if (obj == null) {
+ return false;
+ }
+
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+
+ AppPriorityACLGroup other = (AppPriorityACLGroup) obj;
+ if (getMaxPriority() != other.getMaxPriority()) {
+ return false;
+ }
+
+ if (getDefaultPriority() != other.getDefaultPriority()) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 517861;
+ int result = 9511;
+ result = prime * result + getMaxPriority().getPriority();
+ result = prime * result + getDefaultPriority().getPriority();
+ return result;
+ }
+
+ public Priority getMaxPriority() {
+ return maxPriority;
+ }
+
+ public Priority getDefaultPriority() {
+ return defaultPriority;
+ }
+
+ public AccessControlList getACLList() {
+ return aclList;
+ }
+
+ public void setMaxPriority(Priority maxPriority) {
+ this.maxPriority = maxPriority;
+ }
+
+ public void setDefaultPriority(Priority defaultPriority) {
+ this.defaultPriority = defaultPriority;
+ }
+
+ public void setACLList(AccessControlList accessControlList) {
+ this.aclList = accessControlList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 55ffe25..c475967 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -137,6 +137,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -227,6 +228,7 @@ public class CapacityScheduler extends
private List<AsyncScheduleThread> asyncSchedulerThreads;
private ResourceCommitterService resourceCommitterService;
private RMNodeLabelsManager labelManager;
+ private AppPriorityACLsManager appPriorityACLManager;
/**
* EXPERT
@@ -308,8 +310,9 @@ public class CapacityScheduler extends
this.usePortForNodeName = this.conf.getUsePortForNodeName();
this.applications = new ConcurrentHashMap<>();
this.labelManager = rmContext.getNodeLabelManager();
+ this.appPriorityACLManager = new AppPriorityACLsManager(conf);
this.queueManager = new CapacitySchedulerQueueManager(yarnConf,
- this.labelManager);
+ this.labelManager, this.appPriorityACLManager);
this.queueManager.setCapacitySchedulerContext(this);
this.activitiesManager = new ActivitiesManager(rmContext);
@@ -2188,86 +2191,110 @@ public class CapacityScheduler extends
}
@Override
- public Priority checkAndGetApplicationPriority(Priority priorityFromContext,
- String user, String queueName, ApplicationId applicationId)
- throws YarnException {
- Priority appPriority = null;
-
- // ToDo: Verify against priority ACLs
-
- // Verify the scenario where priority is null from submissionContext.
- if (null == priorityFromContext) {
- // Get the default priority for the Queue. If Queue is non-existent, then
- // use default priority
- priorityFromContext = this.queueManager.getDefaultPriorityForQueue(
- queueName);
+ public Priority checkAndGetApplicationPriority(
+ Priority priorityRequestedByApp, UserGroupInformation user,
+ String queueName, ApplicationId applicationId) throws YarnException {
+ try {
+ readLock.lock();
+ Priority appPriority = priorityRequestedByApp;
+
+ // Verify the scenario where priority is null from submissionContext.
+ if (null == appPriority) {
+ // Verify whether submitted user has any default priority set. If so,
+ // user's default priority will get precedence over queue default.
+ // for updateApplicationPriority call flow, this check is done in
+ // CientRMService itself.
+ appPriority = this.appPriorityACLManager.getDefaultPriority(queueName,
+ user);
+
+ // Get the default priority for the Queue. If Queue is non-existent,
+ // then
+ // use default priority. Do it only if user doesnt have any default.
+ if (null == appPriority) {
+ appPriority = this.queueManager.getDefaultPriorityForQueue(queueName);
+ }
- LOG.info("Application '" + applicationId
- + "' is submitted without priority "
- + "hence considering default queue/cluster priority: "
- + priorityFromContext.getPriority());
- }
+ LOG.info(
+ "Application '" + applicationId + "' is submitted without priority "
+ + "hence considering default queue/cluster priority: "
+ + appPriority.getPriority());
+ }
- // Verify whether submitted priority is lesser than max priority
- // in the cluster. If it is out of found, defining a max cap.
- if (priorityFromContext.compareTo(getMaxClusterLevelAppPriority()) < 0) {
- priorityFromContext = Priority
- .newInstance(getMaxClusterLevelAppPriority().getPriority());
- }
+ // Verify whether submitted priority is lesser than max priority
+ // in the cluster. If it is out of found, defining a max cap.
+ if (appPriority.getPriority() > getMaxClusterLevelAppPriority()
+ .getPriority()) {
+ appPriority = Priority
+ .newInstance(getMaxClusterLevelAppPriority().getPriority());
+ }
- appPriority = priorityFromContext;
+ // Lets check for ACLs here.
+ if (!appPriorityACLManager.checkAccess(user, queueName, appPriority)) {
+ throw new YarnException(new AccessControlException(
+ "User " + user + " does not have permission to submit/update "
+ + applicationId + " for " + appPriority));
+ }
- LOG.info("Priority '" + appPriority.getPriority()
- + "' is acceptable in queue : " + queueName + " for application: "
- + applicationId + " for the user: " + user);
+ LOG.info("Priority '" + appPriority.getPriority()
+ + "' is acceptable in queue : " + queueName + " for application: "
+ + applicationId);
- return appPriority;
+ return appPriority;
+ } finally {
+ readLock.unlock();
+ }
}
@Override
public Priority updateApplicationPriority(Priority newPriority,
- ApplicationId applicationId, SettableFuture<Object> future)
+ ApplicationId applicationId, SettableFuture<Object> future,
+ UserGroupInformation user)
throws YarnException {
- Priority appPriority = null;
- SchedulerApplication<FiCaSchedulerApp> application = applications
- .get(applicationId);
+ try {
+ writeLock.lock();
+ Priority appPriority = null;
+ SchedulerApplication<FiCaSchedulerApp> application = applications
+ .get(applicationId);
- if (application == null) {
- throw new YarnException("Application '" + applicationId
- + "' is not present, hence could not change priority.");
- }
+ if (application == null) {
+ throw new YarnException("Application '" + applicationId
+ + "' is not present, hence could not change priority.");
+ }
- RMApp rmApp = rmContext.getRMApps().get(applicationId);
+ RMApp rmApp = rmContext.getRMApps().get(applicationId);
- appPriority = checkAndGetApplicationPriority(newPriority, rmApp.getUser(),
- rmApp.getQueue(), applicationId);
+ appPriority = checkAndGetApplicationPriority(newPriority, user,
+ rmApp.getQueue(), applicationId);
- if (application.getPriority().equals(appPriority)) {
- future.set(null);
- return appPriority;
- }
+ if (application.getPriority().equals(appPriority)) {
+ future.set(null);
+ return appPriority;
+ }
- // Update new priority in Submission Context to update to StateStore.
- rmApp.getApplicationSubmissionContext().setPriority(appPriority);
+ // Update new priority in Submission Context to update to StateStore.
+ rmApp.getApplicationSubmissionContext().setPriority(appPriority);
- // Update to state store
- ApplicationStateData appState = ApplicationStateData.newInstance(
- rmApp.getSubmitTime(), rmApp.getStartTime(),
- rmApp.getApplicationSubmissionContext(), rmApp.getUser(),
- rmApp.getCallerContext());
- appState.setApplicationTimeouts(rmApp.getApplicationTimeouts());
- rmContext.getStateStore().updateApplicationStateSynchronously(appState,
- false, future);
+ // Update to state store
+ ApplicationStateData appState = ApplicationStateData.newInstance(
+ rmApp.getSubmitTime(), rmApp.getStartTime(),
+ rmApp.getApplicationSubmissionContext(), rmApp.getUser(),
+ rmApp.getCallerContext());
+ appState.setApplicationTimeouts(rmApp.getApplicationTimeouts());
+ rmContext.getStateStore().updateApplicationStateSynchronously(appState,
+ false, future);
- // As we use iterator over a TreeSet for OrderingPolicy, once we change
- // priority then reinsert back to make order correct.
- LeafQueue queue = (LeafQueue) getQueue(rmApp.getQueue());
- queue.updateApplicationPriority(application, appPriority);
+ // As we use iterator over a TreeSet for OrderingPolicy, once we change
+ // priority then reinsert back to make order correct.
+ LeafQueue queue = (LeafQueue) getQueue(rmApp.getQueue());
+ queue.updateApplicationPriority(application, appPriority);
- LOG.info("Priority '" + appPriority + "' is updated in queue :"
- + rmApp.getQueue() + " for application: " + applicationId
- + " for the user: " + rmApp.getUser());
- return appPriority;
+ LOG.info("Priority '" + appPriority + "' is updated in queue :"
+ + rmApp.getQueue() + " for application: " + applicationId
+ + " for the user: " + rmApp.getUser());
+ return appPriority;
+ } finally {
+ writeLock.unlock();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index bfaeba4..eb148d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.ReservationACL;
@@ -49,6 +50,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AppPriorityACLConfigurationParser.AppPriorityACLKeyType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
@@ -63,7 +65,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
private static final Log LOG =
LogFactory.getLog(CapacitySchedulerConfiguration.class);
-
+
private static final String CS_CONFIGURATION_FILE = "capacity-scheduler.xml";
@Private
@@ -274,6 +276,8 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final boolean DEFAULT_LAZY_PREEMPTION_ENABLED = false;
+ AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
+
public CapacitySchedulerConfiguration() {
this(new Configuration());
}
@@ -602,6 +606,10 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return "acl_" + StringUtils.toLowerCase(acl.toString());
}
+ private static String getAclKey(AccessType acl) {
+ return "acl_" + StringUtils.toLowerCase(acl.toString());
+ }
+
@Override
public Map<ReservationACL, AccessControlList> getReservationAcls(String
queue) {
@@ -627,6 +635,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
set(queuePrefix + getAclKey(acl), aclString);
}
+ private void setAcl(String queue, AccessType acl, String aclString) {
+ String queuePrefix = getQueuePrefix(queue);
+ set(queuePrefix + getAclKey(acl), aclString);
+ }
+
public Map<AccessType, AccessControlList> getAcls(String queue) {
Map<AccessType, AccessControlList> acls =
new HashMap<AccessType, AccessControlList>();
@@ -650,6 +663,35 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
}
+ @VisibleForTesting
+ public void setPriorityAcls(String queue, Priority priority,
+ Priority defaultPriority, String[] acls) {
+ StringBuilder aclString = new StringBuilder();
+
+ StringBuilder userAndGroup = new StringBuilder();
+ for (int i = 0; i < acls.length; i++) {
+ userAndGroup.append(AppPriorityACLKeyType.values()[i] + "=" + acls[i].trim())
+ .append(" ");
+ }
+
+ aclString.append("[" + userAndGroup.toString().trim() + " "
+ + "max_priority=" + priority.getPriority() + " " + "default_priority="
+ + defaultPriority.getPriority() + "]");
+
+ setAcl(queue, AccessType.APPLICATION_MAX_PRIORITY, aclString.toString());
+ }
+
+ public List<AppPriorityACLGroup> getPriorityAcls(String queue,
+ Priority clusterMaxPriority) {
+ String queuePrefix = getQueuePrefix(queue);
+ String defaultAcl = ALL_ACL;
+ String aclString = get(
+ queuePrefix + getAclKey(AccessType.APPLICATION_MAX_PRIORITY),
+ defaultAcl);
+
+ return priorityACLConfig.getPriorityAcl(clusterMaxPriority, aclString);
+ }
+
public String[] getQueues(String queue) {
LOG.debug("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue));
String[] queues = getStrings(getQueuePrefix(queue) + QUEUES);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
index 7d29619..504acb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
@@ -23,6 +23,7 @@ import java.util.Comparator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
@@ -85,4 +86,10 @@ public interface CapacitySchedulerContext {
ActivitiesManager getActivitiesManager();
CapacitySchedulerQueueManager getCapacitySchedulerQueueManager();
+
+ /**
+ *
+ * @return Max Cluster level App priority.
+ */
+ Priority getMaxClusterLevelAppPriority();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index 6a3c08a..ddcbc0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
/**
*
@@ -86,6 +87,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
private final Map<String, CSQueue> queues = new ConcurrentHashMap<>();
private CSQueue root;
private final RMNodeLabelsManager labelManager;
+ private AppPriorityACLsManager appPriorityACLManager;
private QueueStateManager<CSQueue, CapacitySchedulerConfiguration>
queueStateManager;
@@ -94,12 +96,15 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
* Construct the service.
* @param conf the configuration
* @param labelManager the labelManager
+ * @param appPriorityACLManager App priority ACL manager
*/
public CapacitySchedulerQueueManager(Configuration conf,
- RMNodeLabelsManager labelManager) {
+ RMNodeLabelsManager labelManager,
+ AppPriorityACLsManager appPriorityACLManager) {
this.authorizer = YarnAuthorizationProvider.getInstance(conf);
this.labelManager = labelManager;
this.queueStateManager = new QueueStateManager<>();
+ this.appPriorityACLManager = appPriorityACLManager;
}
@Override
@@ -145,7 +150,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
throws IOException {
root = parseQueue(this.csContext, conf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues, NOOP);
- setQueueAcls(authorizer, queues);
+ setQueueAcls(authorizer, appPriorityACLManager, queues);
labelManager.reinitializeQueueLabels(getQueueToLabels());
this.queueStateManager.initialize(this);
LOG.info("Initialized root queue " + root);
@@ -168,7 +173,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
// Re-configure queues
root.reinitialize(newRoot, this.csContext.getClusterResource());
- setQueueAcls(authorizer, queues);
+ setQueueAcls(authorizer, appPriorityACLManager, queues);
// Re-calculate headroom for active applications
Resource clusterResource = this.csContext.getClusterResource();
@@ -305,12 +310,22 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
* @throws IOException if fails to set queue acls
*/
public static void setQueueAcls(YarnAuthorizationProvider authorizer,
- Map<String, CSQueue> queues) throws IOException {
+ AppPriorityACLsManager appPriorityACLManager, Map<String, CSQueue> queues)
+ throws IOException {
List<Permission> permissions = new ArrayList<>();
for (CSQueue queue : queues.values()) {
AbstractCSQueue csQueue = (AbstractCSQueue) queue;
permissions.add(
new Permission(csQueue.getPrivilegedEntity(), csQueue.getACLs()));
+
+ if (queue instanceof LeafQueue) {
+ LeafQueue lQueue = (LeafQueue) queue;
+
+ // Clear Priority ACLs first since reinitialize also call same.
+ appPriorityACLManager.clearPriorityACLs(lQueue.getQueueName());
+ appPriorityACLManager.addPrioirityACLs(lQueue.getPriorityACLs(),
+ lQueue.getQueueName());
+ }
}
authorizer.setPermission(permissions,
UserGroupInformation.getCurrentUser());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 18b38f4..6d4dbe0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -140,6 +140,9 @@ public class LeafQueue extends AbstractCSQueue {
private Map<String, TreeSet<RMContainer>> ignorePartitionExclusivityRMContainers =
new ConcurrentHashMap<>();
+ List<AppPriorityACLGroup> priorityAcls =
+ new ArrayList<AppPriorityACLGroup>();
+
@SuppressWarnings({ "unchecked", "rawtypes" })
public LeafQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
@@ -205,6 +208,9 @@ public class LeafQueue extends AbstractCSQueue {
conf.getMaximumApplicationMasterResourcePerQueuePercent(
getQueuePath());
+ priorityAcls = conf.getPriorityAcls(getQueuePath(),
+ scheduler.getMaxClusterLevelAppPriority());
+
if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels,
this.defaultLabelExpression, null)) {
throw new IOException(
@@ -497,6 +503,16 @@ public class LeafQueue extends AbstractCSQueue {
}
}
+ @Private
+ public List<AppPriorityACLGroup> getPriorityACLs() {
+ try {
+ readLock.lock();
+ return new ArrayList<>(priorityAcls);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
@Override
public void reinitialize(
CSQueue newlyParsedQueue, Resource clusterResource)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AppPriorityACLsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AppPriorityACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AppPriorityACLsManager.java
new file mode 100644
index 0000000..c1fd0a6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AppPriorityACLsManager.java
@@ -0,0 +1,230 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.security;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AppPriorityACLGroup;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ *
+ * Manager class to store and check permission for Priority ACLs.
+ */
+public class AppPriorityACLsManager {
+
+ private static final Log LOG = LogFactory
+ .getLog(AppPriorityACLsManager.class);
+
+ /*
+ * An internal class to store ACLs specific to each priority. This will be
+ * used to read and process acl's during app submission time as well.
+ */
+ private static class PriorityACL {
+ private Priority priority;
+ private Priority defaultPriority;
+ private AccessControlList acl;
+
+ PriorityACL(Priority priority, Priority defaultPriority,
+ AccessControlList acl) {
+ this.setPriority(priority);
+ this.setDefaultPriority(defaultPriority);
+ this.setAcl(acl);
+ }
+
+ public Priority getPriority() {
+ return priority;
+ }
+
+ public void setPriority(Priority maxPriority) {
+ this.priority = maxPriority;
+ }
+
+ public Priority getDefaultPriority() {
+ return defaultPriority;
+ }
+
+ public void setDefaultPriority(Priority defaultPriority) {
+ this.defaultPriority = defaultPriority;
+ }
+
+ public AccessControlList getAcl() {
+ return acl;
+ }
+
+ public void setAcl(AccessControlList acl) {
+ this.acl = acl;
+ }
+ }
+
+ private boolean isACLsEnable;
+ private final ConcurrentMap<String, List<PriorityACL>> allAcls =
+ new ConcurrentHashMap<>();
+
+ public AppPriorityACLsManager(Configuration conf) {
+ this.isACLsEnable = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
+ YarnConfiguration.DEFAULT_YARN_ACL_ENABLE);
+ }
+
+ /**
+ * Clear priority acl during refresh.
+ *
+ * @param queueName
+ * Queue Name
+ */
+ public void clearPriorityACLs(String queueName) {
+ allAcls.remove(queueName);
+ }
+
+ /**
+ * Each Queue could have configured with different priority acl's groups. This
+ * method helps to store each such ACL list against queue.
+ *
+ * @param priorityACLGroups
+ * List of Priority ACL Groups.
+ * @param queueName
+ * Queue Name associate with priority acl groups.
+ */
+ public void addPrioirityACLs(List<AppPriorityACLGroup> priorityACLGroups,
+ String queueName) {
+
+ List<PriorityACL> priorityACL = allAcls.get(queueName);
+ if (null == priorityACL) {
+ priorityACL = new ArrayList<PriorityACL>();
+ allAcls.put(queueName, priorityACL);
+ }
+
+ // Ensure lowest priority PriorityACLGroup comes first in the list.
+ Collections.sort(priorityACLGroups);
+
+ for (AppPriorityACLGroup priorityACLGroup : priorityACLGroups) {
+ priorityACL.add(new PriorityACL(priorityACLGroup.getMaxPriority(),
+ priorityACLGroup.getDefaultPriority(),
+ priorityACLGroup.getACLList()));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Priority ACL group added: max-priority - "
+ + priorityACLGroup.getMaxPriority() + "default-priority - "
+ + priorityACLGroup.getDefaultPriority());
+ }
+ }
+ }
+
+ /**
+ * Priority based checkAccess to ensure that given user has enough permission
+ * to submit application at a given priority level.
+ *
+ * @param callerUGI
+ * User who submits the application.
+ * @param queueName
+ * Queue to which application is submitted.
+ * @param submittedPriority
+ * priority of the application.
+ * @return True or False to indicate whether application can be submitted at
+ * submitted priority level or not.
+ */
+ public boolean checkAccess(UserGroupInformation callerUGI, String queueName,
+ Priority submittedPriority) {
+ if (!isACLsEnable) {
+ return true;
+ }
+
+ List<PriorityACL> acls = allAcls.get(queueName);
+ if (acls == null || acls.isEmpty()) {
+ return true;
+ }
+
+ PriorityACL approvedPriorityACL = getMappedPriorityAclForUGI(acls,
+ callerUGI, submittedPriority);
+ if (approvedPriorityACL == null) {
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * If an application is submitted without any priority, and submitted user has
+ * a default priority, this method helps to update this default priority as
+ * app's priority.
+ *
+ * @param queueName
+ * Submitted queue
+ * @param user
+ * User who submitted this application
+ * @return Default priority associated with given user.
+ */
+ public Priority getDefaultPriority(String queueName,
+ UserGroupInformation user) {
+ if (!isACLsEnable) {
+ return null;
+ }
+
+ List<PriorityACL> acls = allAcls.get(queueName);
+ if (acls == null || acls.isEmpty()) {
+ return null;
+ }
+
+ PriorityACL approvedPriorityACL = getMappedPriorityAclForUGI(acls, user,
+ null);
+ if (approvedPriorityACL == null) {
+ return null;
+ }
+
+ Priority defaultPriority = Priority
+ .newInstance(approvedPriorityACL.getDefaultPriority().getPriority());
+ return defaultPriority;
+ }
+
+ private PriorityACL getMappedPriorityAclForUGI(List<PriorityACL> acls ,
+ UserGroupInformation user, Priority submittedPriority) {
+
+ // Iterate through all configured ACLs starting from lower priority.
+ // If user is found corresponding to a configured priority, then store
+ // that entry. if failed, continue iterate through whole acl list.
+ PriorityACL selectedAcl = null;
+ for (PriorityACL entry : acls) {
+ AccessControlList list = entry.getAcl();
+
+ if (list.isUserAllowed(user)) {
+ selectedAcl = entry;
+
+ // If submittedPriority is passed through the argument, also check
+ // whether submittedPriority is under max-priority of each ACL group.
+ if (submittedPriority != null) {
+ selectedAcl = null;
+ if (submittedPriority.getPriority() <= entry.getPriority()
+ .getPriority()) {
+ return entry;
+ }
+ }
+ }
+ }
+ return selectedAcl;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
index e661703..fbd5ac3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
@@ -43,6 +43,8 @@ public abstract class ACLsTestBase {
protected static final String COMMON_USER = "common_user";
protected static final String QUEUE_A_USER = "queueA_user";
protected static final String QUEUE_B_USER = "queueB_user";
+ protected static final String QUEUE_A_GROUP = "queueA_group";
+ protected static final String QUEUE_B_GROUP = "queueB_group";
protected static final String ROOT_ADMIN = "root_admin";
protected static final String QUEUE_A_ADMIN = "queueA_admin";
protected static final String QUEUE_B_ADMIN = "queueB_admin";
@@ -53,7 +55,7 @@ public abstract class ACLsTestBase {
protected static final Log LOG = LogFactory.getLog(TestApplicationACLs.class);
- MockRM resourceManager;
+ protected MockRM resourceManager;
Configuration conf;
YarnRPC rpc;
InetSocketAddress rmAddress;
@@ -68,6 +70,7 @@ public abstract class ACLsTestBase {
AccessControlList adminACL = new AccessControlList("");
conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
+ conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
resourceManager = new MockRM(conf) {
protected ClientRMService createClientRMService() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index 00466ae..23bed22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -29,6 +29,7 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
@@ -463,7 +464,9 @@ public class TestApplicationMasterService {
// Change the priority of App1 to 8
Priority appPriority2 = Priority.newInstance(8);
- rm.getRMAppManager().updateApplicationPriority(app1.getApplicationId(),
+ UserGroupInformation ugi = UserGroupInformation
+ .createRemoteUser(app1.getUser());
+ rm.getRMAppManager().updateApplicationPriority(ugi, app1.getApplicationId(),
appPriority2);
AllocateResponse response2 = am1.allocate(allocateRequest);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index c135384..5ecba12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -1123,7 +1123,7 @@ public class TestClientRMService {
when(yarnScheduler.getResourceCalculator()).thenReturn(rs);
when(yarnScheduler.checkAndGetApplicationPriority(any(Priority.class),
- anyString(), anyString(), any(ApplicationId.class)))
+ any(UserGroupInformation.class), anyString(), any(ApplicationId.class)))
.thenReturn(Priority.newInstance(0));
return yarnScheduler;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index 164ca20..ff52efd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -344,7 +345,10 @@ public class TestApplicationPriority {
// Change the priority of App1 to 8
Priority appPriority2 = Priority.newInstance(8);
- cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null);
+ UserGroupInformation ugi = UserGroupInformation
+ .createRemoteUser(app1.getUser());
+ cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null,
+ ugi);
// get scheduler app
FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications()
@@ -378,7 +382,10 @@ public class TestApplicationPriority {
// Change the priority of App1 to 15
Priority appPriority2 = Priority.newInstance(15);
- cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null);
+ UserGroupInformation ugi = UserGroupInformation
+ .createRemoteUser(app1.getUser());
+ cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null,
+ ugi);
// get scheduler app
FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications()
@@ -428,7 +435,10 @@ public class TestApplicationPriority {
// Change the priority of App1 to 8
Priority appPriority2 = Priority.newInstance(8);
- cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null);
+ UserGroupInformation ugi = UserGroupInformation
+ .createRemoteUser(app1.getUser());
+ cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null,
+ ugi);
// let things settle down
Thread.sleep(1000);
@@ -557,7 +567,10 @@ public class TestApplicationPriority {
// Change the priority of App1 to 3 (lowest)
Priority appPriority3 = Priority.newInstance(3);
- cs.updateApplicationPriority(appPriority3, app2.getApplicationId(), null);
+ UserGroupInformation ugi = UserGroupInformation
+ .createRemoteUser(app2.getUser());
+ cs.updateApplicationPriority(appPriority3, app2.getApplicationId(), null,
+ ugi);
// add request for containers App2
am2.allocate("127.0.0.1", 2 * GB, 3, new ArrayList<ContainerId>());
@@ -788,8 +801,10 @@ public class TestApplicationPriority {
int appsPendingExpected, int activeAppsExpected, RMApp app)
throws YarnException {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ UserGroupInformation ugi = UserGroupInformation
+ .createRemoteUser(app.getUser());
cs.updateApplicationPriority(Priority.newInstance(2),
- app.getApplicationId(), null);
+ app.getApplicationId(), null, ugi);
SchedulerEvent removeAttempt;
removeAttempt = new AppAttemptRemovedSchedulerEvent(
app.getCurrentAppAttempt().getAppAttemptId(), RMAppAttemptState.KILLED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java
new file mode 100644
index 0000000..598bd49
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java
@@ -0,0 +1,120 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class TestApplicationPriorityACLConfiguration {
+
+ private final int defaultPriorityQueueA = 3;
+ private final int defaultPriorityQueueB = -1;
+ private final int maxPriorityQueueA = 5;
+ private final int maxPriorityQueueB = 10;
+ private final int clusterMaxPriority = 10;
+
+ private static final String QUEUE_A_USER = "queueA_user";
+ private static final String QUEUE_B_USER = "queueB_user";
+ private static final String QUEUE_A_GROUP = "queueA_group";
+
+ private static final String QUEUEA = "queueA";
+ private static final String QUEUEB = "queueB";
+ private static final String QUEUEC = "queueC";
+
+ @Test
+ public void testSimpleACLConfiguration() throws Exception {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 25f);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEC, 25f);
+
+ // Success case: Configure one user/group level priority acl for queue A.
+ String[] aclsForA = new String[2];
+ aclsForA[0] = QUEUE_A_USER;
+ aclsForA[1] = QUEUE_A_GROUP;
+ csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
+ Priority.newInstance(maxPriorityQueueA),
+ Priority.newInstance(defaultPriorityQueueA), aclsForA);
+
+ // Try to get the ACL configs and make sure there are errors/exceptions
+ List<AppPriorityACLGroup> pGroupA = csConf.getPriorityAcls(
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
+ Priority.newInstance(clusterMaxPriority));
+
+ // Validate!
+ verifyACLs(pGroupA, QUEUE_A_USER, QUEUE_A_GROUP, maxPriorityQueueA,
+ defaultPriorityQueueA);
+ }
+
+ @Test
+ public void testACLConfigurationForInvalidCases() throws Exception {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 25f);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEC, 25f);
+
+ // Success case: Configure one user/group level priority acl for queue A.
+ String[] aclsForA = new String[2];
+ aclsForA[0] = QUEUE_A_USER;
+ aclsForA[1] = QUEUE_A_GROUP;
+ csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
+ Priority.newInstance(maxPriorityQueueA),
+ Priority.newInstance(defaultPriorityQueueA), aclsForA);
+
+ String[] aclsForB = new String[1];
+ aclsForB[0] = QUEUE_B_USER;
+ csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB,
+ Priority.newInstance(maxPriorityQueueB),
+ Priority.newInstance(defaultPriorityQueueB), aclsForB);
+
+ // Try to get the ACL configs and make sure there are errors/exceptions
+ List<AppPriorityACLGroup> pGroupA = csConf.getPriorityAcls(
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
+ Priority.newInstance(clusterMaxPriority));
+ List<AppPriorityACLGroup> pGroupB = csConf.getPriorityAcls(
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEB,
+ Priority.newInstance(clusterMaxPriority));
+
+ // Validate stored ACL values with configured ones.
+ verifyACLs(pGroupA, QUEUE_A_USER, QUEUE_A_GROUP, maxPriorityQueueA,
+ defaultPriorityQueueA);
+ verifyACLs(pGroupB, QUEUE_B_USER, "", maxPriorityQueueB, 0);
+ }
+
+ private void verifyACLs(List<AppPriorityACLGroup> pGroup, String queueUser,
+ String queueGroup, int maxPriority, int defaultPriority) {
+ AppPriorityACLGroup group = pGroup.get(0);
+ String aclString = queueUser + " " + queueGroup;
+
+ Assert.assertEquals(aclString.trim(),
+ group.getACLList().getAclString().trim());
+ Assert.assertEquals(maxPriority, group.getMaxPriority().getPriority());
+ Assert.assertEquals(defaultPriority,
+ group.getDefaultPriority().getPriority());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/287d3d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java
new file mode 100644
index 0000000..b41ba83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java
@@ -0,0 +1,206 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.ACLsTestBase;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class TestApplicationPriorityACLs extends ACLsTestBase {
+
+ private final int defaultPriorityQueueA = 3;
+ private final int defaultPriorityQueueB = 10;
+ private final int maxPriorityQueueA = 5;
+ private final int maxPriorityQueueB = 11;
+ private final int clusterMaxPriority = 10;
+
+ @Test
+ public void testApplicationACLs() throws Exception {
+
+ /*
+ * Cluster Max-priority is 10. User 'queueA_user' has permission to submit
+ * apps only at priority 5. Default priority for this user is 3.
+ */
+
+ // Case 1: App will be submitted with priority 5.
+ verifyAppSubmitWithPrioritySuccess(QUEUE_A_USER, QUEUEA, 5);
+
+ // Case 2: App will be rejected as submitted priority was 6.
+ verifyAppSubmitWithPriorityFailure(QUEUE_A_USER, QUEUEA, 6);
+
+ // Case 3: App will be submitted w/o priority, hence consider default 3.
+ verifyAppSubmitWithPrioritySuccess(QUEUE_A_USER, QUEUEA, -1);
+
+ // Case 4: App will be submitted with priority 11.
+ verifyAppSubmitWithPrioritySuccess(QUEUE_B_USER, QUEUEB, 11);
+ }
+
+ private void verifyAppSubmitWithPrioritySuccess(String submitter,
+ String queueName, int priority) throws Exception {
+ Priority appPriority = null;
+ if (priority > 0) {
+ appPriority = Priority.newInstance(priority);
+ } else {
+ // RM will consider default priority for the submitted user. So update
+ // priority to the default value to compare.
+ priority = defaultPriorityQueueA;
+ }
+
+ ApplicationSubmissionContext submissionContext = prepareForAppSubmission(
+ submitter, queueName, appPriority);
+ submitAppToRMWithValidAcl(submitter, submissionContext);
+
+ // Ideally get app report here and check the priority.
+ verifyAppPriorityIsAccepted(submitter, submissionContext.getApplicationId(),
+ priority);
+ }
+
+ private void verifyAppSubmitWithPriorityFailure(String submitter,
+ String queueName, int priority) throws Exception {
+ Priority appPriority = Priority.newInstance(priority);
+ ApplicationSubmissionContext submissionContext = prepareForAppSubmission(
+ submitter, queueName, appPriority);
+ submitAppToRMWithInValidAcl(submitter, submissionContext);
+ }
+
+ private ApplicationSubmissionContext prepareForAppSubmission(String submitter,
+ String queueName, Priority priority) throws Exception {
+
+ GetNewApplicationRequest newAppRequest = GetNewApplicationRequest
+ .newInstance();
+
+ ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
+ ApplicationId applicationId = submitterClient
+ .getNewApplication(newAppRequest).getApplicationId();
+
+ Resource resource = BuilderUtils.newResource(1024, 1);
+
+ ContainerLaunchContext amContainerSpec = ContainerLaunchContext
+ .newInstance(null, null, null, null, null, null);
+ ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext
+ .newInstance(applicationId, "applicationName", queueName, null,
+ amContainerSpec, false, true, 1, resource, "applicationType");
+ appSubmissionContext.setApplicationId(applicationId);
+ appSubmissionContext.setQueue(queueName);
+ if (null != priority) {
+ appSubmissionContext.setPriority(priority);
+ }
+
+ return appSubmissionContext;
+ }
+
+ private void submitAppToRMWithValidAcl(String submitter,
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException, IOException, InterruptedException {
+ ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
+ SubmitApplicationRequest submitRequest = SubmitApplicationRequest
+ .newInstance(appSubmissionContext);
+ submitterClient.submitApplication(submitRequest);
+ resourceManager.waitForState(appSubmissionContext.getApplicationId(),
+ RMAppState.ACCEPTED);
+ }
+
+ private void submitAppToRMWithInValidAcl(String submitter,
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException, IOException, InterruptedException {
+ ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
+ SubmitApplicationRequest submitRequest = SubmitApplicationRequest
+ .newInstance(appSubmissionContext);
+ try {
+ submitterClient.submitApplication(submitRequest);
+ } catch (YarnException ex) {
+ Assert.assertTrue(ex.getCause() instanceof RemoteException);
+ }
+ }
+
+ private void verifyAppPriorityIsAccepted(String submitter,
+ ApplicationId applicationId, int priority)
+ throws IOException, InterruptedException {
+ ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
+
+ /**
+ * If priority is greater than cluster max, RM will auto set to cluster max
+ * Consider this scenario as a special case.
+ */
+ if (priority > clusterMaxPriority) {
+ priority = clusterMaxPriority;
+ }
+
+ GetApplicationReportRequest request = GetApplicationReportRequest
+ .newInstance(applicationId);
+ try {
+ GetApplicationReportResponse response = submitterClient
+ .getApplicationReport(request);
+ Assert.assertEquals(response.getApplicationReport().getPriority(),
+ Priority.newInstance(priority));
+ } catch (YarnException e) {
+ Assert.fail("Application submission should not fail.");
+ }
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 25f);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEC, 25f);
+
+ String[] aclsForA = new String[2];
+ aclsForA[0] = QUEUE_A_USER;
+ aclsForA[1] = QUEUE_A_GROUP;
+ csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
+ Priority.newInstance(maxPriorityQueueA),
+ Priority.newInstance(defaultPriorityQueueA), aclsForA);
+
+ String[] aclsForB = new String[2];
+ aclsForB[0] = QUEUE_B_USER;
+ aclsForB[1] = QUEUE_B_GROUP;
+ csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB,
+ Priority.newInstance(maxPriorityQueueB),
+ Priority.newInstance(defaultPriorityQueueB), aclsForB);
+
+ csConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ csConf.set(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class.getName());
+
+ return csConf;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDFS-11301. Double wrapping over
RandomAccessFile in LocalReplicaInPipeline#createStreams. Contributed by
Hanisha Koneru.
Posted by st...@apache.org.
HDFS-11301. Double wrapping over RandomAccessFile in LocalReplicaInPipeline#createStreams. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91bf5044
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91bf5044
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91bf5044
Branch: refs/heads/HADOOP-13345
Commit: 91bf504440967ccdff1cb1cbe7801a5ce2ba88ab
Parents: 287d3d6
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Jan 9 10:32:18 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Jan 9 10:32:18 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/LocalReplicaInPipeline.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91bf5044/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplicaInPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplicaInPipeline.java
index 003f96f..345c329 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplicaInPipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplicaInPipeline.java
@@ -308,9 +308,7 @@ public class LocalReplicaInPipeline extends LocalReplica
FileOutputStream crcOut = null;
try {
blockOut = fileIoProvider.getFileOutputStream(
- getVolume(),
- fileIoProvider.getRandomAccessFile(getVolume(), blockFile, "rw")
- .getFD());
+ getVolume(), new RandomAccessFile(blockFile, "rw").getFD());
crcOut = fileIoProvider.getFileOutputStream(getVolume(), metaRAF.getFD());
if (!isCreate) {
blockOut.getChannel().position(blockDiskSize);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDFS-11299. Support multiple
Datanode File IO hooks. Contributed by Hanisha Koneru.
Posted by st...@apache.org.
HDFS-11299. Support multiple Datanode File IO hooks. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4046794a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4046794a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4046794a
Branch: refs/heads/HADOOP-13345
Commit: 4046794a5365f80f9fa002e3889e41c6d29e13a8
Parents: c18590f
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Jan 10 10:43:02 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Jan 10 10:43:11 2017 -0800
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 2 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 10 +-
.../server/datanode/CountingFileIoEvents.java | 106 -------
.../hadoop/hdfs/server/datanode/DataNode.java | 5 -
.../hdfs/server/datanode/DataNodeMXBean.java | 5 -
.../server/datanode/DefaultFileIoEvents.java | 67 ----
.../datanode/FaultInjectorFileIoEvents.java | 55 ++++
.../hdfs/server/datanode/FileIoEvents.java | 115 -------
.../hdfs/server/datanode/FileIoProvider.java | 318 +++++++++++--------
.../server/datanode/ProfilingFileIoEvents.java | 110 ++++---
.../datanode/TestDataNodeVolumeMetrics.java | 4 +-
.../hadoop/tools/TestHdfsConfigFields.java | 4 +-
12 files changed, 310 insertions(+), 491 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 541f5dc..9a19a9b 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -326,7 +326,7 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
FsVolume
--------
-Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.fileio.events.class` to **org.apache.hadoop.hdfs.server.datanode.ProfilingFileIoEvents**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics.
+Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics.
| Name | Description |
|:---- |:---- |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 50217a2..cf9c805 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -698,8 +698,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
public static final String DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory";
public static final String DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY = "dfs.datanode.fsdataset.volume.choosing.policy";
- public static final String DFS_DATANODE_FILE_IO_EVENTS_CLASS_KEY =
- "dfs.datanode.fileio.events.class";
+ public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
+ "dfs.datanode.enable.fileio.profiling";
+ public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
+ false;
+ public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY =
+ "dfs.datanode.enable.fileio.fault.injection";
+ public static final boolean
+ DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_DEFAULT = false;
public static final String DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_KEY = "dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold";
public static final long DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_DEFAULT = 1024L * 1024L * 1024L * 10L; // 10 GB
public static final String DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = "dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/CountingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/CountingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/CountingFileIoEvents.java
deleted file mode 100644
index 7c6bfd6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/CountingFileIoEvents.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.datanode;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
-
-import javax.annotation.Nullable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * {@link FileIoEvents} that simply counts the number of operations.
- * Not meant to be used outside of testing.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class CountingFileIoEvents extends FileIoEvents {
- private final Map<OPERATION, Counts> counts;
-
- private static class Counts {
- private final AtomicLong successes = new AtomicLong(0);
- private final AtomicLong failures = new AtomicLong(0);
-
- @JsonProperty("Successes")
- public long getSuccesses() {
- return successes.get();
- }
-
- @JsonProperty("Failures")
- public long getFailures() {
- return failures.get();
- }
- }
-
- public CountingFileIoEvents() {
- counts = new HashMap<>();
- for (OPERATION op : OPERATION.values()) {
- counts.put(op, new Counts());
- }
- }
-
- @Override
- public long beforeMetadataOp(
- @Nullable FsVolumeSpi volume, OPERATION op) {
- return 0;
- }
-
- @Override
- public void afterMetadataOp(
- @Nullable FsVolumeSpi volume, OPERATION op, long begin) {
- counts.get(op).successes.incrementAndGet();
- }
-
- @Override
- public long beforeFileIo(
- @Nullable FsVolumeSpi volume, OPERATION op, long len) {
- return 0;
- }
-
- @Override
- public void afterFileIo(
- @Nullable FsVolumeSpi volume, OPERATION op, long begin, long len) {
- counts.get(op).successes.incrementAndGet();
- }
-
- @Override
- public void onFailure(
- @Nullable FsVolumeSpi volume, OPERATION op, Exception e, long begin) {
- counts.get(op).failures.incrementAndGet();
- }
-
- @Override
- public String getStatistics() {
- ObjectMapper objectMapper = new ObjectMapper();
- try {
- return objectMapper.writeValueAsString(counts);
- } catch (JsonProcessingException e) {
- // Failed to serialize. Don't log the exception call stack.
- FileIoProvider.LOG.error("Failed to serialize statistics" + e);
- return null;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 28d627a..090d8b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -3023,11 +3023,6 @@ public class DataNode extends ReconfigurableBase
return "";
}
}
-
- @Override // DataNodeMXBean
- public String getFileIoProviderStatistics() {
- return fileIoProvider.getStatistics();
- }
public void refreshNamenodes(Configuration conf) throws IOException {
blockPoolManager.refreshNamenodes(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java
index ccc5f92..fb79a86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java
@@ -122,11 +122,6 @@ public interface DataNodeMXBean {
String getDiskBalancerStatus();
/**
- * Gets the {@link FileIoProvider} statistics.
- */
- String getFileIoProviderStatistics();
-
- /**
* Gets the average info (e.g. time) of SendPacketDownstream when the DataNode
* acts as the penultimate (2nd to the last) node in pipeline.
* <p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DefaultFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DefaultFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DefaultFileIoEvents.java
deleted file mode 100644
index 6a12aae..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DefaultFileIoEvents.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.datanode;
-
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
-
-import javax.annotation.Nullable;
-
-/**
- * The default implementation of {@link FileIoEvents} that do nothing.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public final class DefaultFileIoEvents extends FileIoEvents {
- @Override
- public long beforeMetadataOp(
- @Nullable FsVolumeSpi volume, OPERATION op) {
- return 0;
- }
-
- @Override
- public void afterMetadataOp(
- @Nullable FsVolumeSpi volume, OPERATION op, long begin) {
- }
-
- @Override
- public long beforeFileIo(
- @Nullable FsVolumeSpi volume, OPERATION op, long len) {
- return 0;
- }
-
- @Override
- public void afterFileIo(
- @Nullable FsVolumeSpi volume, OPERATION op, long begin, long len) {
- }
-
- @Override
- public void onFailure(
- @Nullable FsVolumeSpi volume, OPERATION op, Exception e, long begin) {
- }
-
- @Override
- public @Nullable String getStatistics() {
- // null is valid JSON.
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FaultInjectorFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FaultInjectorFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FaultInjectorFileIoEvents.java
new file mode 100644
index 0000000..ead6ed9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FaultInjectorFileIoEvents.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+
+import javax.annotation.Nullable;
+
+/**
+ * Injects faults in the metadata and data related operations on datanode
+ * volumes.
+ */
+@InterfaceAudience.Private
+public class FaultInjectorFileIoEvents {
+
+ private final boolean isEnabled;
+
+ public FaultInjectorFileIoEvents(@Nullable Configuration conf) {
+ if (conf != null) {
+ isEnabled = conf.getBoolean(DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY, DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_DEFAULT);
+ } else {
+ isEnabled = false;
+ }
+ }
+
+ public void beforeMetadataOp(
+ @Nullable FsVolumeSpi volume, FileIoProvider.OPERATION op) {
+ }
+
+ public void beforeFileIo(
+ @Nullable FsVolumeSpi volume, FileIoProvider.OPERATION op, long len) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoEvents.java
deleted file mode 100644
index 10f2a0c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoEvents.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.datanode;
-
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
-
-import javax.annotation.Nullable;
-
-/**
- * The following hooks can be implemented for instrumentation/fault
- * injection.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public abstract class FileIoEvents {
-
- /**
- * Invoked before a filesystem metadata operation.
- *
- * @param volume target volume for the operation. Null if unavailable.
- * @param op type of operation.
- * @return timestamp at which the operation was started. 0 if
- * unavailable.
- */
- abstract long beforeMetadataOp(@Nullable FsVolumeSpi volume, OPERATION op);
-
- /**
- * Invoked after a filesystem metadata operation has completed.
- *
- * @param volume target volume for the operation. Null if unavailable.
- * @param op type of operation.
- * @param begin timestamp at which the operation was started. 0
- * if unavailable.
- */
- abstract void afterMetadataOp(@Nullable FsVolumeSpi volume, OPERATION op,
- long begin);
-
- /**
- * Invoked before a read/write/flush/channel transfer operation.
- *
- * @param volume target volume for the operation. Null if unavailable.
- * @param op type of operation.
- * @param len length of the file IO. 0 for flush.
- * @return timestamp at which the operation was started. 0 if
- * unavailable.
- */
- abstract long beforeFileIo(@Nullable FsVolumeSpi volume, OPERATION op,
- long len);
-
-
- /**
- * Invoked after a read/write/flush/channel transfer operation
- * has completed.
- *
- * @param volume target volume for the operation. Null if unavailable.
- * @param op type of operation.
- * @param len of the file IO. 0 for flush.
- * @return timestamp at which the operation was started. 0 if
- * unavailable.
- */
- abstract void afterFileIo(@Nullable FsVolumeSpi volume, OPERATION op,
- long begin, long len);
-
- /**
- * Invoked if an operation fails with an exception.
- * @param volume target volume for the operation. Null if unavailable.
- * @param op type of operation.
- * @param e Exception encountered during the operation.
- * @param begin time at which the operation was started.
- */
- abstract void onFailure(
- @Nullable FsVolumeSpi volume, OPERATION op, Exception e, long begin);
-
- /**
- * Invoked by FileIoProvider if an operation fails with an exception.
- * @param datanode datanode that runs volume check upon volume io failure
- * @param volume target volume for the operation. Null if unavailable.
- * @param op type of operation.
- * @param e Exception encountered during the operation.
- * @param begin time at which the operation was started.
- */
- void onFailure(DataNode datanode,
- @Nullable FsVolumeSpi volume, OPERATION op, Exception e, long begin) {
- onFailure(volume, op, e, begin);
- if (datanode != null && volume != null) {
- datanode.checkDiskErrorAsync(volume);
- }
- }
-
- /**
- * Return statistics as a JSON string.
- * @return
- */
- @Nullable abstract String getStatistics();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index f961049..9def2e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.datanode;
-
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -34,7 +33,6 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIOException;
import org.apache.hadoop.net.SocketOutputStream;
-import org.apache.hadoop.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,11 +57,14 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*;
/**
* This class abstracts out various file IO operations performed by the
- * DataNode and invokes event hooks before and after each file IO.
+ * DataNode and invokes profiling (for collecting stats) and fault injection
+ * (for testing) event hooks before and after each file IO.
*
- * Behavior can be injected into these events by implementing
- * {@link FileIoEvents} and replacing the default implementation
- * with {@link DFSConfigKeys#DFS_DATANODE_FILE_IO_EVENTS_CLASS_KEY}.
+ * Behavior can be injected into these events by enabling the
+ * profiling and/or fault injection event hooks through
+ * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and
+ * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}.
+ * These event hooks are disabled by default.
*
* Most functions accept an optional {@link FsVolumeSpi} parameter for
* instrumentation/logging.
@@ -78,9 +79,12 @@ public class FileIoProvider {
public static final Logger LOG = LoggerFactory.getLogger(
FileIoProvider.class);
- private final FileIoEvents eventHooks;
+ private final ProfilingFileIoEvents profilingEventHook;
+ private final FaultInjectorFileIoEvents faultInjectorEventHook;
private final DataNode datanode;
+ private static final int LEN_INT = 4;
+
/**
* @param conf Configuration object. May be null. When null,
* the event handlers are no-ops.
@@ -89,15 +93,8 @@ public class FileIoProvider {
*/
public FileIoProvider(@Nullable Configuration conf,
final DataNode datanode) {
- if (conf != null) {
- final Class<? extends FileIoEvents> clazz = conf.getClass(
- DFSConfigKeys.DFS_DATANODE_FILE_IO_EVENTS_CLASS_KEY,
- DefaultFileIoEvents.class,
- FileIoEvents.class);
- eventHooks = ReflectionUtils.newInstance(clazz, conf);
- } else {
- eventHooks = new DefaultFileIoEvents();
- }
+ profilingEventHook = new ProfilingFileIoEvents(conf);
+ faultInjectorEventHook = new FaultInjectorFileIoEvents(conf);
this.datanode = datanode;
}
@@ -123,15 +120,6 @@ public class FileIoProvider {
}
/**
- * Retrieve statistics from the underlying {@link FileIoEvents}
- * implementation as a JSON string, if it maintains them.
- * @return statistics as a JSON string. May be null.
- */
- public @Nullable String getStatistics() {
- return eventHooks.getStatistics();
- }
-
- /**
* See {@link Flushable#flush()}.
*
* @param volume target volume. null if unavailable.
@@ -139,12 +127,13 @@ public class FileIoProvider {
*/
public void flush(
@Nullable FsVolumeSpi volume, Flushable f) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, FLUSH, 0);
+ final long begin = profilingEventHook.beforeFileIo(volume, FLUSH, 0);
try {
+ faultInjectorEventHook.beforeFileIo(volume, FLUSH, 0);
f.flush();
- eventHooks.afterFileIo(volume, FLUSH, begin, 0);
+ profilingEventHook.afterFileIo(volume, FLUSH, begin, 0);
} catch (Exception e) {
- eventHooks.onFailure(datanode, volume, FLUSH, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -157,12 +146,13 @@ public class FileIoProvider {
*/
public void sync(
@Nullable FsVolumeSpi volume, FileOutputStream fos) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, SYNC, 0);
+ final long begin = profilingEventHook.beforeFileIo(volume, SYNC, 0);
try {
+ faultInjectorEventHook.beforeFileIo(volume, SYNC, 0);
fos.getChannel().force(true);
- eventHooks.afterFileIo(volume, SYNC, begin, 0);
+ profilingEventHook.afterFileIo(volume, SYNC, begin, 0);
} catch (Exception e) {
- eventHooks.onFailure(datanode, volume, SYNC, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -176,12 +166,13 @@ public class FileIoProvider {
public void syncFileRange(
@Nullable FsVolumeSpi volume, FileDescriptor outFd,
long offset, long numBytes, int flags) throws NativeIOException {
- final long begin = eventHooks.beforeFileIo(volume, SYNC, 0);
+ final long begin = profilingEventHook.beforeFileIo(volume, SYNC, 0);
try {
+ faultInjectorEventHook.beforeFileIo(volume, SYNC, 0);
NativeIO.POSIX.syncFileRangeIfPossible(outFd, offset, numBytes, flags);
- eventHooks.afterFileIo(volume, SYNC, begin, 0);
+ profilingEventHook.afterFileIo(volume, SYNC, begin, 0);
} catch (Exception e) {
- eventHooks.onFailure(datanode, volume, SYNC, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -195,13 +186,14 @@ public class FileIoProvider {
public void posixFadvise(
@Nullable FsVolumeSpi volume, String identifier, FileDescriptor outFd,
long offset, long length, int flags) throws NativeIOException {
- final long begin = eventHooks.beforeMetadataOp(volume, FADVISE);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, FADVISE);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, FADVISE);
NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(
identifier, outFd, offset, length, flags);
- eventHooks.afterMetadataOp(volume, FADVISE, begin);
+ profilingEventHook.afterMetadataOp(volume, FADVISE, begin);
} catch (Exception e) {
- eventHooks.onFailure(datanode, volume, FADVISE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -213,13 +205,14 @@ public class FileIoProvider {
* @return true if the file was successfully deleted.
*/
public boolean delete(@Nullable FsVolumeSpi volume, File f) {
- final long begin = eventHooks.beforeMetadataOp(volume, DELETE);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, DELETE);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, DELETE);
boolean deleted = f.delete();
- eventHooks.afterMetadataOp(volume, DELETE, begin);
+ profilingEventHook.afterMetadataOp(volume, DELETE, begin);
return deleted;
} catch (Exception e) {
- eventHooks.onFailure(datanode, volume, DELETE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -232,16 +225,17 @@ public class FileIoProvider {
* existed.
*/
public boolean deleteWithExistsCheck(@Nullable FsVolumeSpi volume, File f) {
- final long begin = eventHooks.beforeMetadataOp(volume, DELETE);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, DELETE);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, DELETE);
boolean deleted = !f.exists() || f.delete();
- eventHooks.afterMetadataOp(volume, DELETE, begin);
+ profilingEventHook.afterMetadataOp(volume, DELETE, begin);
if (!deleted) {
LOG.warn("Failed to delete file {}", f);
}
return deleted;
} catch (Exception e) {
- eventHooks.onFailure(datanode, volume, DELETE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -263,13 +257,14 @@ public class FileIoProvider {
@Nullable FsVolumeSpi volume, SocketOutputStream sockOut,
FileChannel fileCh, long position, int count,
LongWritable waitTime, LongWritable transferTime) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, TRANSFER, count);
+ final long begin = profilingEventHook.beforeFileIo(volume, TRANSFER, count);
try {
+ faultInjectorEventHook.beforeFileIo(volume, TRANSFER, count);
sockOut.transferToFully(fileCh, position, count,
waitTime, transferTime);
- eventHooks.afterFileIo(volume, TRANSFER, begin, count);
+ profilingEventHook.afterFileIo(volume, TRANSFER, begin, count);
} catch (Exception e) {
- eventHooks.onFailure(datanode, volume, TRANSFER, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -284,13 +279,14 @@ public class FileIoProvider {
*/
public boolean createFile(
@Nullable FsVolumeSpi volume, File f) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, OPEN);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, OPEN);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, OPEN);
boolean created = f.createNewFile();
- eventHooks.afterMetadataOp(volume, OPEN, begin);
+ profilingEventHook.afterMetadataOp(volume, OPEN, begin);
return created;
} catch (Exception e) {
- eventHooks.onFailure(datanode, volume, OPEN, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -309,15 +305,16 @@ public class FileIoProvider {
*/
public FileInputStream getFileInputStream(
@Nullable FsVolumeSpi volume, File f) throws FileNotFoundException {
- final long begin = eventHooks.beforeMetadataOp(volume, OPEN);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, OPEN);
FileInputStream fis = null;
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, OPEN);
fis = new WrappedFileInputStream(volume, f);
- eventHooks.afterMetadataOp(volume, OPEN, begin);
+ profilingEventHook.afterMetadataOp(volume, OPEN, begin);
return fis;
} catch(Exception e) {
org.apache.commons.io.IOUtils.closeQuietly(fis);
- eventHooks.onFailure(datanode, volume, OPEN, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -339,15 +336,16 @@ public class FileIoProvider {
public FileOutputStream getFileOutputStream(
@Nullable FsVolumeSpi volume, File f,
boolean append) throws FileNotFoundException {
- final long begin = eventHooks.beforeMetadataOp(volume, OPEN);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, OPEN);
FileOutputStream fos = null;
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, OPEN);
fos = new WrappedFileOutputStream(volume, f, append);
- eventHooks.afterMetadataOp(volume, OPEN, begin);
+ profilingEventHook.afterMetadataOp(volume, OPEN, begin);
return fos;
} catch(Exception e) {
org.apache.commons.io.IOUtils.closeQuietly(fos);
- eventHooks.onFailure(datanode, volume, OPEN, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -403,16 +401,17 @@ public class FileIoProvider {
public FileInputStream getShareDeleteFileInputStream(
@Nullable FsVolumeSpi volume, File f,
long offset) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, OPEN);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, OPEN);
FileInputStream fis = null;
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, OPEN);
fis = new WrappedFileInputStream(volume,
NativeIO.getShareDeleteFileDescriptor(f, offset));
- eventHooks.afterMetadataOp(volume, OPEN, begin);
+ profilingEventHook.afterMetadataOp(volume, OPEN, begin);
return fis;
} catch(Exception e) {
org.apache.commons.io.IOUtils.closeQuietly(fis);
- eventHooks.onFailure(datanode, volume, OPEN, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -434,16 +433,17 @@ public class FileIoProvider {
*/
public FileInputStream openAndSeek(
@Nullable FsVolumeSpi volume, File f, long offset) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, OPEN);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, OPEN);
FileInputStream fis = null;
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, OPEN);
fis = new WrappedFileInputStream(volume,
FsDatasetUtil.openAndSeek(f, offset));
- eventHooks.afterMetadataOp(volume, OPEN, begin);
+ profilingEventHook.afterMetadataOp(volume, OPEN, begin);
return fis;
} catch(Exception e) {
org.apache.commons.io.IOUtils.closeQuietly(fis);
- eventHooks.onFailure(datanode, volume, OPEN, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -465,15 +465,16 @@ public class FileIoProvider {
public RandomAccessFile getRandomAccessFile(
@Nullable FsVolumeSpi volume, File f,
String mode) throws FileNotFoundException {
- final long begin = eventHooks.beforeMetadataOp(volume, OPEN);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, OPEN);
RandomAccessFile raf = null;
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, OPEN);
raf = new WrappedRandomAccessFile(volume, f, mode);
- eventHooks.afterMetadataOp(volume, OPEN, begin);
+ profilingEventHook.afterMetadataOp(volume, OPEN, begin);
return raf;
} catch(Exception e) {
org.apache.commons.io.IOUtils.closeQuietly(raf);
- eventHooks.onFailure(datanode, volume, OPEN, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -486,13 +487,14 @@ public class FileIoProvider {
* @return true on success false on failure.
*/
public boolean fullyDelete(@Nullable FsVolumeSpi volume, File dir) {
- final long begin = eventHooks.beforeMetadataOp(volume, DELETE);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, DELETE);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, DELETE);
boolean deleted = FileUtil.fullyDelete(dir);
- eventHooks.afterMetadataOp(volume, DELETE, begin);
+ profilingEventHook.afterMetadataOp(volume, DELETE, begin);
return deleted;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, DELETE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -508,12 +510,13 @@ public class FileIoProvider {
*/
public void replaceFile(
@Nullable FsVolumeSpi volume, File src, File target) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, MOVE);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, MOVE);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, MOVE);
FileUtil.replaceFile(src, target);
- eventHooks.afterMetadataOp(volume, MOVE, begin);
+ profilingEventHook.afterMetadataOp(volume, MOVE, begin);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, MOVE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -530,12 +533,13 @@ public class FileIoProvider {
public void rename(
@Nullable FsVolumeSpi volume, File src, File target)
throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, MOVE);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, MOVE);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, MOVE);
Storage.rename(src, target);
- eventHooks.afterMetadataOp(volume, MOVE, begin);
+ profilingEventHook.afterMetadataOp(volume, MOVE, begin);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, MOVE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -552,12 +556,13 @@ public class FileIoProvider {
public void moveFile(
@Nullable FsVolumeSpi volume, File src, File target)
throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, MOVE);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, MOVE);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, MOVE);
FileUtils.moveFile(src, target);
- eventHooks.afterMetadataOp(volume, MOVE, begin);
+ profilingEventHook.afterMetadataOp(volume, MOVE, begin);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, MOVE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -576,12 +581,13 @@ public class FileIoProvider {
public void move(
@Nullable FsVolumeSpi volume, Path src, Path target,
CopyOption... options) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, MOVE);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, MOVE);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, MOVE);
Files.move(src, target, options);
- eventHooks.afterMetadataOp(volume, MOVE, begin);
+ profilingEventHook.afterMetadataOp(volume, MOVE, begin);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, MOVE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -600,12 +606,14 @@ public class FileIoProvider {
@Nullable FsVolumeSpi volume, File src, File target,
boolean preserveFileDate) throws IOException {
final long length = src.length();
- final long begin = eventHooks.beforeFileIo(volume, NATIVE_COPY, length);
+ final long begin = profilingEventHook.beforeFileIo(volume, NATIVE_COPY,
+ length);
try {
+ faultInjectorEventHook.beforeFileIo(volume, NATIVE_COPY, length);
Storage.nativeCopyFileUnbuffered(src, target, preserveFileDate);
- eventHooks.afterFileIo(volume, NATIVE_COPY, begin, length);
+ profilingEventHook.afterFileIo(volume, NATIVE_COPY, begin, length);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, NATIVE_COPY, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -622,15 +630,16 @@ public class FileIoProvider {
*/
public boolean mkdirs(
@Nullable FsVolumeSpi volume, File dir) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, MKDIRS);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, MKDIRS);
boolean created = false;
boolean isDirectory;
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, MKDIRS);
created = dir.mkdirs();
isDirectory = !created && dir.isDirectory();
- eventHooks.afterMetadataOp(volume, MKDIRS, begin);
+ profilingEventHook.afterMetadataOp(volume, MKDIRS, begin);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, MKDIRS, e, begin);
+ onFailure(volume, begin);
throw e;
}
@@ -650,13 +659,14 @@ public class FileIoProvider {
*/
public void mkdirsWithExistsCheck(
@Nullable FsVolumeSpi volume, File dir) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, MKDIRS);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, MKDIRS);
boolean succeeded = false;
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, MKDIRS);
succeeded = dir.isDirectory() || dir.mkdirs();
- eventHooks.afterMetadataOp(volume, MKDIRS, begin);
+ profilingEventHook.afterMetadataOp(volume, MKDIRS, begin);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, MKDIRS, e, begin);
+ onFailure(volume, begin);
throw e;
}
@@ -676,13 +686,14 @@ public class FileIoProvider {
*/
public File[] listFiles(
@Nullable FsVolumeSpi volume, File dir) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, LIST);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, LIST);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, LIST);
File[] children = FileUtil.listFiles(dir);
- eventHooks.afterMetadataOp(volume, LIST, begin);
+ profilingEventHook.afterMetadataOp(volume, LIST, begin);
return children;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, LIST, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -698,13 +709,14 @@ public class FileIoProvider {
*/
public String[] list(
@Nullable FsVolumeSpi volume, File dir) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, LIST);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, LIST);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, LIST);
String[] children = FileUtil.list(dir);
- eventHooks.afterMetadataOp(volume, LIST, begin);
+ profilingEventHook.afterMetadataOp(volume, LIST, begin);
return children;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, LIST, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -721,13 +733,14 @@ public class FileIoProvider {
public List<String> listDirectory(
@Nullable FsVolumeSpi volume, File dir,
FilenameFilter filter) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, LIST);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, LIST);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, LIST);
List<String> children = IOUtils.listDirectory(dir, filter);
- eventHooks.afterMetadataOp(volume, LIST, begin);
+ profilingEventHook.afterMetadataOp(volume, LIST, begin);
return children;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, LIST, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -743,13 +756,14 @@ public class FileIoProvider {
*/
public int getHardLinkCount(
@Nullable FsVolumeSpi volume, File f) throws IOException {
- final long begin = eventHooks.beforeMetadataOp(volume, LIST);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, LIST);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, LIST);
int count = HardLink.getLinkCount(f);
- eventHooks.afterMetadataOp(volume, LIST, begin);
+ profilingEventHook.afterMetadataOp(volume, LIST, begin);
return count;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, LIST, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -762,13 +776,14 @@ public class FileIoProvider {
* @return true if the file exists.
*/
public boolean exists(@Nullable FsVolumeSpi volume, File f) {
- final long begin = eventHooks.beforeMetadataOp(volume, EXISTS);
+ final long begin = profilingEventHook.beforeMetadataOp(volume, EXISTS);
try {
+ faultInjectorEventHook.beforeMetadataOp(volume, EXISTS);
boolean exists = f.exists();
- eventHooks.afterMetadataOp(volume, EXISTS, begin);
+ profilingEventHook.afterMetadataOp(volume, EXISTS, begin);
return exists;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, EXISTS, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -803,13 +818,14 @@ public class FileIoProvider {
*/
@Override
public int read() throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, READ, 1);
+ final long begin = profilingEventHook.beforeFileIo(volume, READ, LEN_INT);
try {
+ faultInjectorEventHook.beforeFileIo(volume, READ, LEN_INT);
int b = super.read();
- eventHooks.afterFileIo(volume, READ, begin, 1);
+ profilingEventHook.afterFileIo(volume, READ, begin, LEN_INT);
return b;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, READ, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -819,13 +835,15 @@ public class FileIoProvider {
*/
@Override
public int read(@Nonnull byte[] b) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, READ, b.length);
+ final long begin = profilingEventHook.beforeFileIo(volume, READ, b
+ .length);
try {
+ faultInjectorEventHook.beforeFileIo(volume, READ, b.length);
int numBytesRead = super.read(b);
- eventHooks.afterFileIo(volume, READ, begin, numBytesRead);
+ profilingEventHook.afterFileIo(volume, READ, begin, numBytesRead);
return numBytesRead;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, READ, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -835,13 +853,14 @@ public class FileIoProvider {
*/
@Override
public int read(@Nonnull byte[] b, int off, int len) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, READ, len);
+ final long begin = profilingEventHook.beforeFileIo(volume, READ, len);
try {
+ faultInjectorEventHook.beforeFileIo(volume, READ, len);
int numBytesRead = super.read(b, off, len);
- eventHooks.afterFileIo(volume, READ, begin, numBytesRead);
+ profilingEventHook.afterFileIo(volume, READ, begin, numBytesRead);
return numBytesRead;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, READ, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -878,12 +897,14 @@ public class FileIoProvider {
*/
@Override
public void write(int b) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, WRITE, 1);
+ final long begin = profilingEventHook.beforeFileIo(volume, WRITE,
+ LEN_INT);
try {
+ faultInjectorEventHook.beforeFileIo(volume, WRITE, LEN_INT);
super.write(b);
- eventHooks.afterFileIo(volume, WRITE, begin, 1);
+ profilingEventHook.afterFileIo(volume, WRITE, begin, LEN_INT);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, WRITE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -893,12 +914,14 @@ public class FileIoProvider {
*/
@Override
public void write(@Nonnull byte[] b) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, WRITE, b.length);
+ final long begin = profilingEventHook.beforeFileIo(volume, WRITE, b
+ .length);
try {
+ faultInjectorEventHook.beforeFileIo(volume, WRITE, b.length);
super.write(b);
- eventHooks.afterFileIo(volume, WRITE, begin, b.length);
+ profilingEventHook.afterFileIo(volume, WRITE, begin, b.length);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, WRITE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -908,12 +931,13 @@ public class FileIoProvider {
*/
@Override
public void write(@Nonnull byte[] b, int off, int len) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, WRITE, len);
+ final long begin = profilingEventHook.beforeFileIo(volume, WRITE, len);
try {
+ faultInjectorEventHook.beforeFileIo(volume, WRITE, len);
super.write(b, off, len);
- eventHooks.afterFileIo(volume, WRITE, begin, len);
+ profilingEventHook.afterFileIo(volume, WRITE, begin, len);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, WRITE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@@ -935,77 +959,93 @@ public class FileIoProvider {
@Override
public int read() throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, READ, 1);
+ final long begin = profilingEventHook.beforeFileIo(volume, READ, LEN_INT);
try {
+ faultInjectorEventHook.beforeFileIo(volume, READ, LEN_INT);
int b = super.read();
- eventHooks.afterFileIo(volume, READ, begin, 1);
+ profilingEventHook.afterFileIo(volume, READ, begin, LEN_INT);
return b;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, READ, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, READ, len);
+ final long begin = profilingEventHook.beforeFileIo(volume, READ, len);
try {
+ faultInjectorEventHook.beforeFileIo(volume, READ, len);
int numBytesRead = super.read(b, off, len);
- eventHooks.afterFileIo(volume, READ, begin, numBytesRead);
+ profilingEventHook.afterFileIo(volume, READ, begin, numBytesRead);
return numBytesRead;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, READ, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@Override
public int read(byte[] b) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, READ, b.length);
+ final long begin = profilingEventHook.beforeFileIo(volume, READ, b
+ .length);
try {
+ faultInjectorEventHook.beforeFileIo(volume, READ, b.length);
int numBytesRead = super.read(b);
- eventHooks.afterFileIo(volume, READ, begin, numBytesRead);
+ profilingEventHook.afterFileIo(volume, READ, begin, numBytesRead);
return numBytesRead;
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, READ, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@Override
public void write(int b) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, WRITE, 1);
+ final long begin = profilingEventHook.beforeFileIo(volume, WRITE,
+ LEN_INT);
try {
+ faultInjectorEventHook.beforeFileIo(volume, WRITE, LEN_INT);
super.write(b);
- eventHooks.afterFileIo(volume, WRITE, begin, 1);
+ profilingEventHook.afterFileIo(volume, WRITE, begin, LEN_INT);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, WRITE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@Override
public void write(@Nonnull byte[] b) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, WRITE, b.length);
+ final long begin = profilingEventHook.beforeFileIo(volume, WRITE, b
+ .length);
try {
+ faultInjectorEventHook.beforeFileIo(volume, WRITE, b.length);
super.write(b);
- eventHooks.afterFileIo(volume, WRITE, begin, b.length);
+ profilingEventHook.afterFileIo(volume, WRITE, begin, b.length);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, WRITE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
- final long begin = eventHooks.beforeFileIo(volume, WRITE, len);
+ final long begin = profilingEventHook.beforeFileIo(volume, WRITE, len);
try {
+ faultInjectorEventHook.beforeFileIo(volume, WRITE, len);
super.write(b, off, len);
- eventHooks.afterFileIo(volume, WRITE, begin, len);
+ profilingEventHook.afterFileIo(volume, WRITE, begin, len);
} catch(Exception e) {
- eventHooks.onFailure(datanode, volume, WRITE, e, begin);
+ onFailure(volume, begin);
throw e;
}
}
}
+
+ private void onFailure(@Nullable FsVolumeSpi volume, long begin) {
+ if (datanode != null && volume != null) {
+ datanode.checkDiskErrorAsync(volume);
+ }
+ profilingEventHook.onFailure(volume, begin);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index affd093..43ac495 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hdfs.server.datanode;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.util.Time;
@@ -26,84 +28,96 @@ import org.apache.hadoop.util.Time;
import javax.annotation.Nullable;
/**
- * {@link FileIoEvents} that profiles the performance of the metadata and data
- * related operations on datanode volumes.
+ * Profiles the performance of the metadata and data related operations on
+ * datanode volumes.
*/
@InterfaceAudience.Private
-class ProfilingFileIoEvents extends FileIoEvents {
+class ProfilingFileIoEvents {
+
+ private final boolean isEnabled;
+
+ public ProfilingFileIoEvents(@Nullable Configuration conf) {
+ if (conf != null) {
+ isEnabled = conf.getBoolean(DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
+ } else {
+ isEnabled = false;
+ }
+ }
- @Override
public long beforeMetadataOp(@Nullable FsVolumeSpi volume,
FileIoProvider.OPERATION op) {
- DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
- if (metrics != null) {
- return Time.monotonicNow();
+ if (isEnabled) {
+ DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
+ if (metrics != null) {
+ return Time.monotonicNow();
+ }
}
return 0;
}
- @Override
public void afterMetadataOp(@Nullable FsVolumeSpi volume,
FileIoProvider.OPERATION op, long begin) {
- DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
- if (metrics != null) {
- metrics.addMetadastaOperationLatency(Time.monotonicNow() - begin);
+ if (isEnabled) {
+ DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
+ if (metrics != null) {
+ metrics.addMetadastaOperationLatency(Time.monotonicNow() - begin);
+ }
}
}
- @Override
public long beforeFileIo(@Nullable FsVolumeSpi volume,
FileIoProvider.OPERATION op, long len) {
- DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
- if (metrics != null) {
- return Time.monotonicNow();
+ if (isEnabled) {
+ DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
+ if (metrics != null) {
+ return Time.monotonicNow();
+ }
}
return 0;
}
- @Override
public void afterFileIo(@Nullable FsVolumeSpi volume,
FileIoProvider.OPERATION op, long begin, long len) {
- DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
- if (metrics != null) {
- long latency = Time.monotonicNow() - begin;
- metrics.addDataFileIoLatency(latency);
- switch (op) {
- case SYNC:
- metrics.addSyncIoLatency(latency);
- break;
- case FLUSH:
- metrics.addFlushIoLatency(latency);
- break;
- case READ:
- metrics.addReadIoLatency(latency);
- break;
- case WRITE:
- metrics.addWriteIoLatency(latency);
- break;
- default:
+ if (isEnabled) {
+ DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
+ if (metrics != null) {
+ long latency = Time.monotonicNow() - begin;
+ metrics.addDataFileIoLatency(latency);
+ switch (op) {
+ case SYNC:
+ metrics.addSyncIoLatency(latency);
+ break;
+ case FLUSH:
+ metrics.addFlushIoLatency(latency);
+ break;
+ case READ:
+ metrics.addReadIoLatency(latency);
+ break;
+ case WRITE:
+ metrics.addWriteIoLatency(latency);
+ break;
+ default:
+ }
}
}
}
- @Override
- public void onFailure(@Nullable FsVolumeSpi volume,
- FileIoProvider.OPERATION op, Exception e, long begin) {
- DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
- if (metrics != null) {
- metrics.addFileIoError(Time.monotonicNow() - begin);
+ public void onFailure(@Nullable FsVolumeSpi volume, long begin) {
+ if (isEnabled) {
+ DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
+ if (metrics != null) {
+ metrics.addFileIoError(Time.monotonicNow() - begin);
+ }
}
}
- @Nullable
- @Override
- public String getStatistics() {
- return null;
- }
-
private DataNodeVolumeMetrics getVolumeMetrics(final FsVolumeSpi volume) {
- if (volume != null) {
- return volume.getMetrics();
+ if (isEnabled) {
+ if (volume != null) {
+ return volume.getMetrics();
+ }
}
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
index 407c3e9..6a8ac9c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
@@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics {
private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException {
Configuration conf = new HdfsConfiguration();
- conf.set(DFSConfigKeys.DFS_DATANODE_FILE_IO_EVENTS_CLASS_KEY,
- "org.apache.hadoop.hdfs.server.datanode.ProfilingFileIoEvents");
+ conf.setBoolean(DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true);
SimulatedFSDataset.setFactory(conf);
return new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATANODES)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4046794a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 3bac7b9..51ef5d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -100,7 +100,9 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
configurationPropsToSkipCompare
- .add(DFSConfigKeys.DFS_DATANODE_FILE_IO_EVENTS_CLASS_KEY);
+ .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
+ configurationPropsToSkipCompare.add(DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
// Allocate
xmlPropsToSkipCompare = new HashSet<String>();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-6015. AsyncDispatcher thread
name can be set to improved debugging. Contributed by Ajith S.
Posted by st...@apache.org.
YARN-6015. AsyncDispatcher thread name can be set to improved debugging. Contributed by Ajith S.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a59df157
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a59df157
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a59df157
Branch: refs/heads/HADOOP-13345
Commit: a59df15757fac7f917cb96fc8fcfeb7017475e4f
Parents: 0203164
Author: Naganarasimha <na...@apache.org>
Authored: Sat Jan 7 08:11:58 2017 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Sat Jan 7 08:11:58 2017 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/event/AsyncDispatcher.java | 16 +++++++++++++++-
.../yarn/nodelabels/CommonNodeLabelsManager.java | 2 +-
.../hadoop/yarn/server/nodemanager/NodeManager.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 2 +-
.../timelineservice/NMTimelinePublisher.java | 2 +-
.../server/resourcemanager/ResourceManager.java | 2 +-
.../ahs/RMApplicationHistoryWriter.java | 2 +-
.../metrics/AbstractSystemMetricsPublisher.java | 2 +-
.../resourcemanager/recovery/RMStateStore.java | 2 +-
9 files changed, 23 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index a3d306f..913b8ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -74,6 +74,11 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
protected final Map<Class<? extends Enum>, EventHandler> eventDispatchers;
private boolean exitOnDispatchException;
+ /**
+ * The thread name for dispatcher.
+ */
+ private String dispatcherThreadName = "AsyncDispatcher event handler";
+
public AsyncDispatcher() {
this(new LinkedBlockingQueue<Event>());
}
@@ -84,6 +89,15 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
this.eventDispatchers = new HashMap<Class<? extends Enum>, EventHandler>();
}
+ /**
+ * Set a name for this dispatcher thread.
+ * @param dispatcherName name of the dispatcher thread
+ */
+ public AsyncDispatcher(String dispatcherName) {
+ this();
+ dispatcherThreadName = dispatcherName;
+ }
+
Runnable createThread() {
return new Runnable() {
@Override
@@ -130,7 +144,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
//start all the components
super.serviceStart();
eventHandlingThread = new Thread(createThread());
- eventHandlingThread.setName("AsyncDispatcher event handler");
+ eventHandlingThread.setName(dispatcherThreadName);
eventHandlingThread.start();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 1a83632..f3f4ba0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -211,7 +211,7 @@ public class CommonNodeLabelsManager extends AbstractService {
// for UT purpose
protected void initDispatcher(Configuration conf) {
// create async handler
- dispatcher = new AsyncDispatcher();
+ dispatcher = new AsyncDispatcher("NodeLabelManager dispatcher");
AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher;
asyncDispatcher.init(conf);
asyncDispatcher.setDrainEventsOnStop();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 7de5317..60bd00b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -319,7 +319,7 @@ public class NodeManager extends CompositeService
addService(del);
// NodeManager level dispatcher
- this.dispatcher = new AsyncDispatcher();
+ this.dispatcher = new AsyncDispatcher("NM Event dispatcher");
dirsHandler = new LocalDirsHandlerService(metrics);
nodeHealthChecker =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 8f4478e..cb39862 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -223,7 +223,7 @@ public class ContainerManagerImpl extends CompositeService implements
this.dirsHandler = dirsHandler;
// ContainerManager level dispatcher.
- dispatcher = new AsyncDispatcher();
+ dispatcher = new AsyncDispatcher("NM ContainerManager dispatcher");
this.deletionService = deletionContext;
this.metrics = metrics;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index d54189f..8994582 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -86,7 +86,7 @@ public class NMTimelinePublisher extends CompositeService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
- dispatcher = new AsyncDispatcher();
+ dispatcher = new AsyncDispatcher("NM Timeline dispatcher");
dispatcher.register(NMTimelineEventType.class,
new ForwardingEventHandler());
addIfService(dispatcher);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 110f2c8..0c9d095 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -414,7 +414,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
protected Dispatcher createDispatcher() {
- return new AsyncDispatcher();
+ return new AsyncDispatcher("RM Event dispatcher");
}
protected ResourceScheduler createScheduler() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
index 6a15465..19c5c27 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
@@ -355,7 +355,7 @@ public class RMApplicationHistoryWriter extends CompositeService {
}
protected AsyncDispatcher createDispatcher() {
- return new AsyncDispatcher();
+ return new AsyncDispatcher("RM ApplicationHistory dispatcher");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
index db2b0af..e56fccf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
@@ -111,7 +111,7 @@ public abstract class AbstractSystemMetricsPublisher extends CompositeService
}
protected AsyncDispatcher createDispatcher() {
- return new AsyncDispatcher();
+ return new AsyncDispatcher("RM Timeline dispatcher");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 0fd346f..6ede3b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -678,7 +678,7 @@ public abstract class RMStateStore extends AbstractService {
@Override
protected void serviceInit(Configuration conf) throws Exception{
// create async handler
- dispatcher = new AsyncDispatcher();
+ dispatcher = new AsyncDispatcher("RM StateStore dispatcher");
dispatcher.init(conf);
rmStateStoreEventHandler = new ForwardingEventHandler();
dispatcher.register(RMStateStoreEventType.class,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HADOOP-13958. Bump up release year
to 2017. Contributed by Junping Du.
Posted by st...@apache.org.
HADOOP-13958. Bump up release year to 2017. Contributed by Junping Du.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94f1d06f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94f1d06f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94f1d06f
Branch: refs/heads/HADOOP-13345
Commit: 94f1d06f205eb5679475569b932eb2792f53d891
Parents: 85826f6
Author: Junping Du <ju...@apache.org>
Authored: Sun Jan 8 15:31:07 2017 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Sun Jan 8 15:31:07 2017 -0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94f1d06f/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 4751fc3..e35e9ca 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -31,7 +31,7 @@
<properties>
<!-- Set the Release year during release -->
- <release-year>2016</release-year>
+ <release-year>2017</release-year>
<failIfNoTests>false</failIfNoTests>
<maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-11282. Document the missing
metrics of DataNode Volume IO operations. Contributed by Yiqun Lin.
Posted by st...@apache.org.
HDFS-11282. Document the missing metrics of DataNode Volume IO operations. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a659ff4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a659ff4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a659ff4
Branch: refs/heads/HADOOP-13345
Commit: 4a659ff40fca7c263d62ac7514afc100a4dbb1ed
Parents: 0b8a7c1
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Jan 5 19:34:32 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Jan 5 19:34:32 2017 -0800
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 31 ++++++++++++++++++++
1 file changed, 31 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a659ff4/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 1d5ea5e..541f5dc 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -323,6 +323,37 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
| `RemoteBytesWritten` | Number of bytes written by remote clients |
| `BPServiceActorInfo` | The information about a block pool service actor |
+FsVolume
+--------
+
+Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.fileio.events.class` to **org.apache.hadoop.hdfs.server.datanode.ProfilingFileIoEvents**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `TotalMetadataOperations` | Total number (monotonically increasing) of metadata operations. Metadata operations include stat, list, mkdir, delete, move, open and posix_fadvise. |
+| `MetadataOperationRateNumOps` | The number of metadata operations within an interval time of metric |
+| `MetadataOperationRateAvgTime` | Mean time of metadata operations in milliseconds |
+| `MetadataOperationLatency`*num*`s(50|75|90|95|99)thPercentileLatency` | The 50/75/90/95/99th percentile of metadata operations latency in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
+| `TotalDataFileIos` | Total number (monotonically increasing) of data file io operations |
+| `DataFileIoRateNumOps` | The number of data file io operations within an interval time of metric |
+| `DataFileIoRateAvgTime` | Mean time of data file io operations in milliseconds |
+| `DataFileIoLatency`*num*`s(50|75|90|95|99)thPercentileLatency` | The 50/75/90/95/99th percentile of data file io operations latency in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
+| `FlushIoRateNumOps` | The number of file flush io operations within an interval time of metric |
+| `FlushIoRateAvgTime` | Mean time of file flush io operations in milliseconds |
+| `FlushIoLatency`*num*`s(50|75|90|95|99)thPercentileLatency` | The 50/75/90/95/99th percentile of file flush io operations latency in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
+| `SyncIoRateNumOps` | The number of file sync io operations within an interval time of metric |
+| `SyncIoRateAvgTime` | Mean time of file sync io operations in milliseconds |
+| `SyncIoLatency`*num*`s(50|75|90|95|99)thPercentileLatency` | The 50/75/90/95/99th percentile of file sync io operations latency in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
+| `ReadIoRateNumOps` | The number of file read io operations within an interval time of metric |
+| `ReadIoRateAvgTime` | Mean time of file read io operations in milliseconds |
+| `ReadIoLatency`*num*`s(50|75|90|95|99)thPercentileLatency` | The 50/75/90/95/99th percentile of file read io operations latency in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
+| `WriteIoRateNumOps` | The number of file write io operations within an interval time of metric |
+| `WriteIoRateAvgTime` | Mean time of file write io operations in milliseconds |
+| `WriteIoLatency`*num*`s(50|75|90|95|99)thPercentileLatency` | The 50/75/90/95/99th percentile of file write io operations latency in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
+| `TotalFileIoErrors` | Total number (monotonically increasing) of file io error operations |
+| `FileIoErrorRateNumOps` | The number of file io error operations within an interval time of metric |
+| `FileIoErrorRateAvgTime` | It measures the mean time in milliseconds from the start of an operation to hitting a failure |
+
yarn context
============
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HADOOP-12733. Remove references to
obsolete io.seqfile configuration variables. Contributed by Ray Chiang.
Posted by st...@apache.org.
HADOOP-12733. Remove references to obsolete io.seqfile configuration variables. Contributed by Ray Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01d31fe9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01d31fe9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01d31fe9
Branch: refs/heads/HADOOP-13345
Commit: 01d31fe9389ccdc153d7f4bf6574bf8e509867c1
Parents: 87bb1c4
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Jan 4 14:10:36 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Jan 4 14:10:36 2017 +0900
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 16 ----------------
.../hadoop/conf/TestCommonConfigurationFields.java | 2 --
.../test/resources/job_1329348432655_0001_conf.xml | 2 --
.../src/main/data/2jobs2min-rumen-jh.json | 6 ------
4 files changed, 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01d31fe9/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index b4a34db..ee2cc2e 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1262,22 +1262,6 @@
</description>
</property>
-<property>
- <name>io.seqfile.lazydecompress</name>
- <value>true</value>
- <description>Should values of block-compressed SequenceFiles be decompressed
- only when necessary.
- </description>
-</property>
-
-<property>
- <name>io.seqfile.sorter.recordlimit</name>
- <value>1000000</value>
- <description>The limit on number of records to be kept in memory in a spill
- in SequenceFiles.Sorter
- </description>
-</property>
-
<property>
<name>io.mapfile.bloom.size</name>
<value>1048576</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01d31fe9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 571dfae..a3a4026 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -124,8 +124,6 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare.add("dr.who");
// XML deprecated properties.
- xmlPropsToSkipCompare.add("io.seqfile.lazydecompress");
- xmlPropsToSkipCompare.add("io.seqfile.sorter.recordlimit");
// - org.apache.hadoop.hdfs.client.HdfsClientConfigKeys
xmlPropsToSkipCompare
.add("io.bytes.per.checksum");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01d31fe9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
index e4619d6..4c73e8b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
@@ -16,7 +16,6 @@
<property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.memory.limit.percent</name><value>0.25</value></property>
<property><!--Loaded from job.xml--><name>hadoop.http.authentication.kerberos.keytab</name><value>${user.home}/hadoop.keytab</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.keytab</name><value>/etc/krb5.keytab</value></property>
-<property><!--Loaded from job.xml--><name>io.seqfile.sorter.recordlimit</name><value>1000000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.io.sort.factor</name><value>10</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.disk-health-checker.interval-ms</name><value>120000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.working.dir</name><value>hdfs://localhost:8021/user/user</value></property>
@@ -102,7 +101,6 @@
<property><!--Loaded from job.xml--><name>dfs.client.block.write.retries</name><value>3</value></property>
<property><!--Loaded from job.xml--><name>hadoop.proxyuser.user.groups</name><value>users</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.name.dir.restore</name><value>false</value></property>
-<property><!--Loaded from job.xml--><name>io.seqfile.lazydecompress</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.merge.inmem.threshold</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.input.fileinputformat.split.minsize</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>dfs.replication</name><value>3</value></property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01d31fe9/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
index 095cfd5..c252539 100644
--- a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
+++ b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
@@ -4545,7 +4545,6 @@
"hadoop.ssl.keystores.factory.class" : "org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory",
"hadoop.http.authentication.kerberos.keytab" : "${user.home}/hadoop.keytab",
"yarn.nodemanager.keytab" : "/etc/krb5.keytab",
- "io.seqfile.sorter.recordlimit" : "1000000",
"mapreduce.task.io.sort.factor" : "10",
"yarn.nodemanager.disk-health-checker.interval-ms" : "120000",
"mapreduce.job.working.dir" : "hdfs://a2115.smile.com:9820/user/jenkins",
@@ -4662,7 +4661,6 @@
"rpc.engine.org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
"dfs.datanode.hdfs-blocks-metadata.enabled" : "true",
"ha.zookeeper.parent-znode" : "/hadoop-ha",
- "io.seqfile.lazydecompress" : "true",
"mapreduce.reduce.merge.inmem.threshold" : "1000",
"mapreduce.input.fileinputformat.split.minsize" : "0",
"dfs.replication" : "3",
@@ -9627,7 +9625,6 @@
"hadoop.ssl.keystores.factory.class" : "org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory",
"hadoop.http.authentication.kerberos.keytab" : "${user.home}/hadoop.keytab",
"yarn.nodemanager.keytab" : "/etc/krb5.keytab",
- "io.seqfile.sorter.recordlimit" : "1000000",
"mapreduce.task.io.sort.factor" : "10",
"yarn.nodemanager.disk-health-checker.interval-ms" : "120000",
"mapreduce.job.working.dir" : "hdfs://a2115.smile.com:9820/user/jenkins",
@@ -9744,7 +9741,6 @@
"rpc.engine.org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
"dfs.datanode.hdfs-blocks-metadata.enabled" : "true",
"ha.zookeeper.parent-znode" : "/hadoop-ha",
- "io.seqfile.lazydecompress" : "true",
"mapreduce.reduce.merge.inmem.threshold" : "1000",
"mapreduce.input.fileinputformat.split.minsize" : "0",
"dfs.replication" : "3",
@@ -10209,7 +10205,6 @@
"hadoop.ssl.keystores.factory.class" : "org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory",
"hadoop.http.authentication.kerberos.keytab" : "${user.home}/hadoop.keytab",
"yarn.nodemanager.keytab" : "/etc/krb5.keytab",
-"io.seqfile.sorter.recordlimit" : "1000000",
"mapreduce.task.io.sort.factor" : "10",
"yarn.nodemanager.disk-health-checker.interval-ms" : "120000",
"mapreduce.job.working.dir" : "hdfs://a2115.smile.com:9820/user/jenkins",
@@ -10327,7 +10322,6 @@
"rpc.engine.org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
"dfs.datanode.hdfs-blocks-metadata.enabled" : "true",
"ha.zookeeper.parent-znode" : "/hadoop-ha",
-"io.seqfile.lazydecompress" : "true",
"mapreduce.reduce.merge.inmem.threshold" : "1000",
"mapreduce.input.fileinputformat.split.minsize" : "0",
"dfs.replication" : "3",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-5991. Yarn Distributed Shell
does not print throwable t to App Master When failed to start container
(Contributed by Jim Frankola via Daniel Templeton)
Posted by st...@apache.org.
YARN-5991. Yarn Distributed Shell does not print throwable t to App Master When failed to start container
(Contributed by Jim Frankola via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71a4acf7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71a4acf7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71a4acf7
Branch: refs/heads/HADOOP-13345
Commit: 71a4acf74bc9ca34f0e57835c9d6e3efbe7c0567
Parents: 2977bc6
Author: Daniel Templeton <te...@apache.org>
Authored: Fri Jan 6 14:26:08 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Fri Jan 6 14:26:08 2017 -0800
----------------------------------------------------------------------
.../yarn/applications/distributedshell/ApplicationMaster.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a4acf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 17dae6b..5a06ef6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -1001,7 +1001,7 @@ public class ApplicationMaster {
@Override
public void onStartContainerError(ContainerId containerId, Throwable t) {
- LOG.error("Failed to start Container " + containerId);
+ LOG.error("Failed to start Container " + containerId, t);
containers.remove(containerId);
applicationMaster.numCompletedContainers.incrementAndGet();
applicationMaster.numFailedContainers.incrementAndGet();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-5988. RM unable to start in
secure setup. Contributed by Ajith S.
Posted by st...@apache.org.
YARN-5988. RM unable to start in secure setup. Contributed by Ajith S.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e49e0a6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e49e0a6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e49e0a6e
Branch: refs/heads/HADOOP-13345
Commit: e49e0a6e37f4a32535d7d4a07015fbf9eb33c74a
Parents: 01d31fe
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Jan 4 10:29:31 2017 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Jan 4 12:09:29 2017 +0530
----------------------------------------------------------------------
.../server/resourcemanager/AdminService.java | 8 +++
.../resourcemanager/TestRMAdminService.java | 56 +++++++++++++++-----
2 files changed, 50 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e49e0a6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 028b6f0..04ee5d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -536,6 +536,7 @@ public class AdminService extends CompositeService implements
checkRMStatus(user.getShortUserName(), operation, "refresh Service ACLs.");
refreshServiceAcls();
+ refreshActiveServicesAcls();
RMAuditLogger.logSuccess(user.getShortUserName(), operation,
"AdminService");
@@ -549,6 +550,13 @@ public class AdminService extends CompositeService implements
YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
refreshServiceAcls(conf, policyProvider);
+ }
+
+ private void refreshActiveServicesAcls() throws IOException, YarnException {
+ PolicyProvider policyProvider = RMPolicyProvider.getInstance();
+ Configuration conf =
+ getConfiguration(new Configuration(false),
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider);
rmContext.getApplicationMasterService().refreshServiceAcls(
conf, policyProvider);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e49e0a6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index fa01b55..9ae28c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -770,20 +770,7 @@ public class TestRMAdminService {
YarnException {
StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(
HAServiceProtocol.RequestSource.REQUEST_BY_USER);
- configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
- "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
- configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
- configuration.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
- int base = 100;
- for (String confKey : YarnConfiguration
- .getServiceAddressConfKeys(configuration)) {
- configuration.set(HAUtil.addSuffix(confKey, "rm1"), "0.0.0.0:"
- + (base + 20));
- configuration.set(HAUtil.addSuffix(confKey, "rm2"), "0.0.0.0:"
- + (base + 40));
- base = base * 2;
- }
+ updateConfigurationForRMHA();
Configuration conf1 = new Configuration(configuration);
conf1.set(YarnConfiguration.RM_HA_ID, "rm1");
Configuration conf2 = new Configuration(configuration);
@@ -1340,4 +1327,45 @@ public class TestRMAdminService {
}
}
+ @Test
+ public void testSecureRMBecomeActive() throws IOException,
+ YarnException {
+ StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+ updateConfigurationForRMHA();
+ configuration.setBoolean(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true);
+ configuration.set(YarnConfiguration.RM_HA_ID, "rm1");
+ // upload default configurations
+ uploadDefaultConfiguration();
+
+ ResourceManager resourceManager = new ResourceManager();
+ try {
+ resourceManager.init(configuration);
+ resourceManager.start();
+ Assert.assertTrue(resourceManager.getRMContext().getHAServiceState()
+ == HAServiceState.STANDBY);
+ resourceManager.adminService.transitionToActive(requestInfo);
+ } finally {
+ resourceManager.close();
+ }
+ }
+
+ private void updateConfigurationForRMHA() {
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+ configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ configuration.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
+
+ int base = 1500;
+ for (String confKey : YarnConfiguration
+ .getServiceAddressConfKeys(configuration)) {
+ configuration.set(HAUtil.addSuffix(confKey, "rm1"),
+ "0.0.0.0:" + (base + 20));
+ configuration.set(HAUtil.addSuffix(confKey, "rm2"),
+ "0.0.0.1:" + (base + 40));
+ base = base * 2;
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-5959. RM changes to support
change of container ExecutionType. (Arun Suresh via wangda)
Posted by st...@apache.org.
YARN-5959. RM changes to support change of container ExecutionType. (Arun Suresh via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a55bd84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a55bd84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a55bd84
Branch: refs/heads/HADOOP-13345
Commit: 0a55bd841ec0f2eb89a0383f4c589526e8b138d4
Parents: a605ff3
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Jan 5 10:31:05 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Jan 5 10:31:05 2017 -0800
----------------------------------------------------------------------
.../v2/app/rm/TestRMContainerAllocator.java | 12 +-
.../sls/scheduler/ResourceSchedulerWrapper.java | 9 +-
.../sls/scheduler/SLSCapacityScheduler.java | 9 +-
.../yarn/api/records/ContainerUpdateType.java | 9 +-
.../yarn/api/records/UpdateContainerError.java | 6 +
.../api/records/UpdateContainerRequest.java | 10 +
.../src/main/proto/yarn_service_protos.proto | 3 +-
.../api/impl/TestAMRMClientOnRMRestart.java | 11 +-
.../OpportunisticContainerAllocator.java | 16 +-
.../OpportunisticContainerContext.java | 8 +-
.../server/scheduler/SchedulerRequestKey.java | 51 ++-
.../ApplicationMasterService.java | 53 ++-
...pportunisticContainerAllocatorAMService.java | 18 +-
.../server/resourcemanager/RMServerUtils.java | 142 +++---
.../rmapp/attempt/RMAppAttemptImpl.java | 6 +-
.../rmcontainer/RMContainerImpl.java | 32 +-
.../scheduler/AbstractYarnScheduler.java | 104 ++++-
.../resourcemanager/scheduler/Allocation.java | 26 +-
.../scheduler/AppSchedulingInfo.java | 70 +--
.../scheduler/ContainerUpdateContext.java | 267 +++++++++++
.../scheduler/ContainerUpdates.java | 68 +++
.../scheduler/SchedulerApplicationAttempt.java | 170 ++++++-
.../scheduler/SchedulerUtils.java | 22 +
.../scheduler/YarnScheduler.java | 9 +-
.../scheduler/capacity/CapacityScheduler.java | 16 +-
.../allocator/RegularContainerAllocator.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 13 +-
.../scheduler/fair/FSAppAttempt.java | 4 +-
.../scheduler/fair/FairScheduler.java | 41 +-
.../scheduler/fifo/FifoAppAttempt.java | 4 +-
.../scheduler/fifo/FifoScheduler.java | 5 +-
.../LocalitySchedulingPlacementSet.java | 46 +-
.../placement/SchedulingPlacementSet.java | 5 +-
.../server/resourcemanager/Application.java | 4 +-
.../yarn/server/resourcemanager/MockAM.java | 7 +
.../yarn/server/resourcemanager/MockNM.java | 6 +
.../resourcemanager/TestClientRMService.java | 5 +-
...pportunisticContainerAllocatorAMService.java | 456 ++++++++++++++++++-
.../attempt/TestRMAppAttemptTransitions.java | 19 +-
.../rmcontainer/TestRMContainerImpl.java | 7 +-
.../capacity/TestCapacityScheduler.java | 37 +-
.../scheduler/capacity/TestChildQueueOrder.java | 5 +-
.../scheduler/capacity/TestReservations.java | 9 +-
.../scheduler/fair/FairSchedulerTestBase.java | 14 +-
.../fair/TestContinuousScheduling.java | 9 +-
.../scheduler/fair/TestFairScheduler.java | 35 +-
.../scheduler/fifo/TestFifoScheduler.java | 46 +-
47 files changed, 1612 insertions(+), 314 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index f9ee9cc..e6aee6e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -110,7 +110,6 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -132,6 +131,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
@@ -1707,8 +1707,7 @@ public class TestRMContainerAllocator {
ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
List<ContainerId> release, List<String> blacklistAdditions,
List<String> blacklistRemovals,
- List<UpdateContainerRequest> increaseRequests,
- List<UpdateContainerRequest> decreaseRequests) {
+ ContainerUpdates updateRequests) {
List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
for (ResourceRequest req : ask) {
ResourceRequest reqCopy = ResourceRequest.newInstance(req
@@ -1723,7 +1722,7 @@ public class TestRMContainerAllocator {
lastBlacklistRemovals = blacklistRemovals;
Allocation allocation = super.allocate(
applicationAttemptId, askCopy, release, blacklistAdditions,
- blacklistRemovals, increaseRequests, decreaseRequests);
+ blacklistRemovals, updateRequests);
if (forceResourceLimit != null) {
// Test wants to force the non-default resource limit
allocation.setResourceLimit(forceResourceLimit);
@@ -1754,8 +1753,7 @@ public class TestRMContainerAllocator {
ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
List<ContainerId> release, List<String> blacklistAdditions,
List<String> blacklistRemovals,
- List<UpdateContainerRequest> increaseRequest,
- List<UpdateContainerRequest> decreaseRequests) {
+ ContainerUpdates updateRequests) {
List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
for (ResourceRequest req : ask) {
ResourceRequest reqCopy = ResourceRequest.newInstance(req
@@ -1766,7 +1764,7 @@ public class TestRMContainerAllocator {
SecurityUtil.setTokenServiceUseIp(false);
Allocation normalAlloc = super.allocate(
applicationAttemptId, askCopy, release,
- blacklistAdditions, blacklistRemovals, null, null);
+ blacklistAdditions, blacklistRemovals, updateRequests);
List<Container> containers = normalAlloc.getContainers();
if(containers.size() > 0) {
// allocate excess container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
index 79f934c..e66de2f 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
@@ -59,7 +59,6 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
@@ -68,6 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
@@ -206,14 +206,13 @@ final public class ResourceSchedulerWrapper
public Allocation allocate(ApplicationAttemptId attemptId,
List<ResourceRequest> resourceRequests, List<ContainerId> containerIds,
List<String> strings, List<String> strings2,
- List<UpdateContainerRequest> increaseRequests,
- List<UpdateContainerRequest> decreaseRequests) {
+ ContainerUpdates updateRequests) {
if (metricsON) {
final Timer.Context context = schedulerAllocateTimer.time();
Allocation allocation = null;
try {
allocation = scheduler.allocate(attemptId, resourceRequests,
- containerIds, strings, strings2, null, null);
+ containerIds, strings, strings2, updateRequests);
return allocation;
} finally {
context.stop();
@@ -227,7 +226,7 @@ final public class ResourceSchedulerWrapper
}
} else {
return scheduler.allocate(attemptId,
- resourceRequests, containerIds, strings, strings2, null, null);
+ resourceRequests, containerIds, strings, strings2, updateRequests);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index 89f9ad3..8388273 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -51,10 +51,10 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@@ -182,15 +182,14 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
public Allocation allocate(ApplicationAttemptId attemptId,
List<ResourceRequest> resourceRequests, List<ContainerId> containerIds,
List<String> strings, List<String> strings2,
- List<UpdateContainerRequest> increaseRequests,
- List<UpdateContainerRequest> decreaseRequests) {
+ ContainerUpdates updateRequests) {
if (metricsON) {
final Timer.Context context = schedulerAllocateTimer.time();
Allocation allocation = null;
try {
allocation = super
.allocate(attemptId, resourceRequests, containerIds, strings,
- strings2, increaseRequests, decreaseRequests);
+ strings2, updateRequests);
return allocation;
} finally {
context.stop();
@@ -204,7 +203,7 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
}
} else {
return super.allocate(attemptId, resourceRequests, containerIds, strings,
- strings2, increaseRequests, decreaseRequests);
+ strings2, updateRequests);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerUpdateType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerUpdateType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerUpdateType.java
index 978ea09..6109cdb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerUpdateType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerUpdateType.java
@@ -39,7 +39,12 @@ public enum ContainerUpdateType {
DECREASE_RESOURCE,
/**
- * Execution Type change.
+ * Execution Type promotion.
*/
- UPDATE_EXECUTION_TYPE
+ PROMOTE_EXECUTION_TYPE,
+
+ /**
+ * Execution Type demotion.
+ */
+ DEMOTE_EXECUTION_TYPE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerError.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerError.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerError.java
index 7102f7b..e7458cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerError.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerError.java
@@ -87,6 +87,12 @@ public abstract class UpdateContainerError {
}
@Override
+ public String toString() {
+ return "UpdateContainerError{reason=" + getReason() + ", "
+ + "req=" + getUpdateContainerRequest() + "}";
+ }
+
+ @Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
index e4f7a82..925a7979 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
@@ -150,11 +150,13 @@ public abstract class UpdateContainerRequest extends AbstractResourceRequest {
ContainerId cId = getContainerId();
ExecutionType execType = getExecutionType();
Resource capability = getCapability();
+ ContainerUpdateType updateType = getContainerUpdateType();
result =
prime * result + ((capability == null) ? 0 : capability.hashCode());
result = prime * result + ((cId == null) ? 0 : cId.hashCode());
result = prime * result + getContainerVersion();
result = prime * result + ((execType == null) ? 0 : execType.hashCode());
+ result = prime * result + ((updateType== null) ? 0 : updateType.hashCode());
return result;
}
@@ -208,6 +210,14 @@ public abstract class UpdateContainerRequest extends AbstractResourceRequest {
} else if (!execType.equals(other.getExecutionType())) {
return false;
}
+ ContainerUpdateType updateType = getContainerUpdateType();
+ if (updateType == null) {
+ if (other.getContainerUpdateType() != null) {
+ return false;
+ }
+ } else if (!updateType.equals(other.getContainerUpdateType())) {
+ return false;
+ }
return true;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index d9230d4..aed1580 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -63,7 +63,8 @@ message FinishApplicationMasterResponseProto {
enum ContainerUpdateTypeProto {
INCREASE_RESOURCE = 0;
DECREASE_RESOURCE = 1;
- UPDATE_EXECUTION_TYPE = 2;
+ PROMOTE_EXECUTION_TYPE = 2;
+ DEMOTE_EXECUTION_TYPE = 3;
}
message UpdateContainerRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
index f1c49f2..ac77446 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
@@ -574,8 +575,7 @@ public class TestAMRMClientOnRMRestart {
ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
List<ContainerId> release, List<String> blacklistAdditions,
List<String> blacklistRemovals,
- List<UpdateContainerRequest> increaseRequests,
- List<UpdateContainerRequest> decreaseRequests) {
+ ContainerUpdates updateRequests) {
List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
for (ResourceRequest req : ask) {
ResourceRequest reqCopy =
@@ -586,13 +586,12 @@ public class TestAMRMClientOnRMRestart {
}
lastAsk = ask;
lastRelease = release;
- lastIncrease = increaseRequests;
- lastDecrease = decreaseRequests;
+ lastIncrease = updateRequests.getIncreaseRequests();
+ lastDecrease = updateRequests.getDecreaseRequests();
lastBlacklistAdditions = blacklistAdditions;
lastBlacklistRemovals = blacklistRemovals;
return super.allocate(applicationAttemptId, askCopy, release,
- blacklistAdditions, blacklistRemovals, increaseRequests,
- decreaseRequests);
+ blacklistAdditions, blacklistRemovals, updateRequests);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
index c1300b2..6fd5228 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
@@ -321,13 +321,21 @@ public class OpportunisticContainerAllocator {
// before accepting an ask)
Resource capability = normalizeCapability(appParams, rr);
+ return createContainer(
+ rmIdentifier, appParams.getContainerTokenExpiryInterval(),
+ SchedulerRequestKey.create(rr), userName, node, cId, capability);
+ }
+
+ private Container createContainer(long rmIdentifier, long tokenExpiry,
+ SchedulerRequestKey schedulerKey, String userName, RemoteNode node,
+ ContainerId cId, Resource capability) {
long currTime = System.currentTimeMillis();
ContainerTokenIdentifier containerTokenIdentifier =
new ContainerTokenIdentifier(
cId, 0, node.getNodeId().toString(), userName,
- capability, currTime + appParams.containerTokenExpiryInterval,
+ capability, currTime + tokenExpiry,
tokenSecretManager.getCurrentKey().getKeyId(), rmIdentifier,
- rr.getPriority(), currTime,
+ schedulerKey.getPriority(), currTime,
null, CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK,
ExecutionType.OPPORTUNISTIC);
byte[] pwd =
@@ -336,9 +344,9 @@ public class OpportunisticContainerAllocator {
containerTokenIdentifier);
Container container = BuilderUtils.newContainer(
cId, node.getNodeId(), node.getHttpAddress(),
- capability, rr.getPriority(), containerToken,
+ capability, schedulerKey.getPriority(), containerToken,
containerTokenIdentifier.getExecutionType(),
- rr.getAllocationRequestId());
+ schedulerKey.getAllocationRequestId());
return container;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java
index a2f9f4d..1b1c5b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java
@@ -150,8 +150,9 @@ public class OpportunisticContainerContext {
resourceRequest.getNumContainers() + request.getNumContainers());
}
if (ResourceRequest.isAnyLocation(request.getResourceName())) {
- LOG.info("# of outstandingOpReqs in ANY (at" +
- "priority = "+ schedulerKey.getPriority()
+ LOG.info("# of outstandingOpReqs in ANY (at "
+ + "priority = " + schedulerKey.getPriority()
+ + ", allocationReqId = " + schedulerKey.getAllocationRequestId()
+ ", with capability = " + request.getCapability() + " ) : "
+ resourceRequest.getNumContainers());
}
@@ -167,7 +168,8 @@ public class OpportunisticContainerContext {
public void matchAllocationToOutstandingRequest(Resource capability,
List<Container> allocatedContainers) {
for (Container c : allocatedContainers) {
- SchedulerRequestKey schedulerKey = SchedulerRequestKey.extractFrom(c);
+ SchedulerRequestKey schedulerKey =
+ SchedulerRequestKey.extractFrom(c);
Map<Resource, ResourceRequest> asks =
outstandingOpReqs.get(schedulerKey);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
index 9b7edbe..36a9149 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
@@ -18,9 +18,12 @@
package org.apache.hadoop.yarn.server.scheduler;
+import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
/**
* Composite key for outstanding scheduler requests for any schedulable entity.
@@ -31,6 +34,7 @@ public final class SchedulerRequestKey implements
private final Priority priority;
private final long allocationRequestId;
+ private final ContainerId containerToUpdate;
/**
* Factory method to generate a SchedulerRequestKey from a ResourceRequest.
@@ -39,7 +43,13 @@ public final class SchedulerRequestKey implements
*/
public static SchedulerRequestKey create(ResourceRequest req) {
return new SchedulerRequestKey(req.getPriority(),
- req.getAllocationRequestId());
+ req.getAllocationRequestId(), null);
+ }
+
+ public static SchedulerRequestKey create(UpdateContainerRequest req,
+ SchedulerRequestKey schedulerRequestKey) {
+ return new SchedulerRequestKey(schedulerRequestKey.getPriority(),
+ schedulerRequestKey.getAllocationRequestId(), req.getContainerId());
}
/**
@@ -50,12 +60,16 @@ public final class SchedulerRequestKey implements
*/
public static SchedulerRequestKey extractFrom(Container container) {
return new SchedulerRequestKey(container.getPriority(),
- container.getAllocationRequestId());
+ container.getAllocationRequestId(), null);
}
- SchedulerRequestKey(Priority priority, long allocationRequestId) {
+
+
+ public SchedulerRequestKey(Priority priority, long allocationRequestId,
+ ContainerId containerToUpdate) {
this.priority = priority;
this.allocationRequestId = allocationRequestId;
+ this.containerToUpdate = containerToUpdate;
}
/**
@@ -76,6 +90,10 @@ public final class SchedulerRequestKey implements
return allocationRequestId;
}
+ public ContainerId getContainerToUpdate() {
+ return containerToUpdate;
+ }
+
@Override
public int compareTo(SchedulerRequestKey o) {
if (o == null) {
@@ -85,6 +103,15 @@ public final class SchedulerRequestKey implements
return 1;
}
}
+
+ // Ensure updates are ranked higher
+ if (this.containerToUpdate == null && o.containerToUpdate != null) {
+ return -1;
+ }
+ if (this.containerToUpdate != null && o.containerToUpdate == null) {
+ return 1;
+ }
+
int priorityCompare = o.getPriority().compareTo(priority);
// we first sort by priority and then by allocationRequestId
if (priorityCompare != 0) {
@@ -107,16 +134,21 @@ public final class SchedulerRequestKey implements
if (getAllocationRequestId() != that.getAllocationRequestId()) {
return false;
}
- return getPriority() != null ?
- getPriority().equals(that.getPriority()) :
- that.getPriority() == null;
+ if (!getPriority().equals(that.getPriority())) {
+ return false;
+ }
+ return containerToUpdate != null ?
+ containerToUpdate.equals(that.containerToUpdate) :
+ that.containerToUpdate == null;
}
@Override
public int hashCode() {
- int result = getPriority() != null ? getPriority().hashCode() : 0;
- result = 31 * result + (int) (getAllocationRequestId() ^ (
- getAllocationRequestId() >>> 32));
+ int result = priority != null ? priority.hashCode() : 0;
+ result = 31 * result + (int) (allocationRequestId ^ (allocationRequestId
+ >>> 32));
+ result = 31 * result + (containerToUpdate != null ? containerToUpdate
+ .hashCode() : 0);
return result;
}
@@ -125,6 +157,7 @@ public final class SchedulerRequestKey implements
return "SchedulerRequestKey{" +
"priority=" + priority +
", allocationRequestId=" + allocationRequestId +
+ ", containerToUpdate=" + containerToUpdate +
'}';
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 9fd1845..70a46a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -68,7 +68,6 @@ import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
import org.apache.hadoop.yarn.api.records.UpdateContainerError;
-import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
@@ -93,7 +92,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler
+ .AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security
@@ -559,12 +561,10 @@ public class ApplicationMasterService extends AbstractService implements
// Split Update Resource Requests into increase and decrease.
// No Exceptions are thrown here. All update errors are aggregated
// and returned to the AM.
- List<UpdateContainerRequest> increaseResourceReqs = new ArrayList<>();
- List<UpdateContainerRequest> decreaseResourceReqs = new ArrayList<>();
- List<UpdateContainerError> updateContainerErrors =
+ List<UpdateContainerError> updateErrors = new ArrayList<>();
+ ContainerUpdates containerUpdateRequests =
RMServerUtils.validateAndSplitUpdateResourceRequests(
- rmContext, request, maximumCapacity,
- increaseResourceReqs, decreaseResourceReqs);
+ rmContext, request, maximumCapacity, updateErrors);
// Send new requests to appAttempt.
Allocation allocation;
@@ -580,7 +580,7 @@ public class ApplicationMasterService extends AbstractService implements
allocation =
this.rScheduler.allocate(appAttemptId, ask, release,
blacklistAdditions, blacklistRemovals,
- increaseResourceReqs, decreaseResourceReqs);
+ containerUpdateRequests);
}
if (!blacklistAdditions.isEmpty() || !blacklistRemovals.isEmpty()) {
@@ -596,7 +596,7 @@ public class ApplicationMasterService extends AbstractService implements
}
// Notify the AM of container update errors
- addToUpdateContainerErrors(allocateResponse, updateContainerErrors);
+ addToUpdateContainerErrors(allocateResponse, updateErrors);
// update the response with the deltas of node status changes
List<RMNode> updatedNodes = new ArrayList<RMNode>();
@@ -630,15 +630,7 @@ public class ApplicationMasterService extends AbstractService implements
.pullJustFinishedContainers());
allocateResponse.setAvailableResources(allocation.getResourceLimit());
- // Handling increased containers
- addToUpdatedContainers(
- allocateResponse, ContainerUpdateType.INCREASE_RESOURCE,
- allocation.getIncreasedContainers());
-
- // Handling decreased containers
- addToUpdatedContainers(
- allocateResponse, ContainerUpdateType.DECREASE_RESOURCE,
- allocation.getDecreasedContainers());
+ addToContainerUpdates(appAttemptId, allocateResponse, allocation);
allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
@@ -658,6 +650,33 @@ public class ApplicationMasterService extends AbstractService implements
.getApplicationPriority());
}
+ private void addToContainerUpdates(ApplicationAttemptId appAttemptId,
+ AllocateResponse allocateResponse, Allocation allocation) {
+ // Handling increased containers
+ addToUpdatedContainers(
+ allocateResponse, ContainerUpdateType.INCREASE_RESOURCE,
+ allocation.getIncreasedContainers());
+
+ // Handling decreased containers
+ addToUpdatedContainers(
+ allocateResponse, ContainerUpdateType.DECREASE_RESOURCE,
+ allocation.getDecreasedContainers());
+
+ // Handling promoted containers
+ addToUpdatedContainers(
+ allocateResponse, ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ allocation.getPromotedContainers());
+
+ // Handling demoted containers
+ addToUpdatedContainers(
+ allocateResponse, ContainerUpdateType.DEMOTE_EXECUTION_TYPE,
+ allocation.getDemotedContainers());
+
+ addToUpdateContainerErrors(allocateResponse,
+ ((AbstractYarnScheduler)rScheduler)
+ .getApplicationAttempt(appAttemptId).pullUpdateContainerErrors());
+ }
+
protected void addToUpdateContainerErrors(AllocateResponse allocateResponse,
List<UpdateContainerError> updateContainerErrors) {
if (!updateContainerErrors.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index 9d4c092..8f3a888 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocol;
import org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl;
-
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateRequest;
@@ -48,7 +47,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.proto.ApplicationMasterProtocol.ApplicationMasterProtocolService;
-
import org.apache.hadoop.yarn.server.api.protocolrecords.RemoteNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
@@ -57,6 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.NodeQueueLoadMonitor;
@@ -69,9 +68,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
-
import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator;
import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
import java.io.IOException;
@@ -251,6 +250,7 @@ public class OpportunisticContainerAllocatorAMService
// Allocate GUARANTEED containers.
request.setAskList(partitionedAsks.getGuaranteed());
+
super.allocateInternal(appAttemptId, request, allocateResponse);
}
@@ -298,15 +298,9 @@ public class OpportunisticContainerAllocatorAMService
boolean isRemotelyAllocated) {
for (Container container : allocContainers) {
// Create RMContainer
- SchedulerApplicationAttempt appAttempt =
- ((AbstractYarnScheduler) rmContext.getScheduler())
- .getCurrentAttemptForContainer(container.getId());
- RMContainer rmContainer = new RMContainerImpl(container,
- appAttempt.getApplicationAttemptId(), container.getNodeId(),
- appAttempt.getUser(), rmContext, isRemotelyAllocated);
- appAttempt.addRMContainer(container.getId(), rmContainer);
- ((AbstractYarnScheduler) rmContext.getScheduler()).getNode(
- container.getNodeId()).allocateContainer(rmContainer);
+ RMContainer rmContainer =
+ SchedulerUtils.createOpportunisticRmContainer(
+ rmContext, container, isRemotelyAllocated);
rmContainer.handle(
new RMContainerEvent(container.getId(),
RMContainerEventType.ACQUIRED));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 74898ca..ebbeb0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -63,6 +65,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt
.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler
.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler
@@ -80,7 +83,7 @@ import org.apache.hadoop.yarn.util.resource.Resources;
*/
public class RMServerUtils {
- private static final String UPDATE_OUTSTANDING_ERROR =
+ public static final String UPDATE_OUTSTANDING_ERROR =
"UPDATE_OUTSTANDING_ERROR";
private static final String INCORRECT_CONTAINER_VERSION_ERROR =
"INCORRECT_CONTAINER_VERSION_ERROR";
@@ -124,74 +127,105 @@ public class RMServerUtils {
/**
* Check if we have:
- * - Request for same containerId and different target resource
- * - If targetResources violates maximum/minimumAllocation
- * @param rmContext RM context
- * @param request Allocate Request
- * @param maximumAllocation Maximum Allocation
- * @param increaseResourceReqs Increase Resource Request
- * @param decreaseResourceReqs Decrease Resource Request
- * @return List of container Errors
+ * - Request for same containerId and different target resource.
+ * - If targetResources violates maximum/minimumAllocation.
+ * @param rmContext RM context.
+ * @param request Allocate Request.
+ * @param maximumAllocation Maximum Allocation.
+ * @param updateErrors Container update errors.
+ * @return ContainerUpdateRequests.
*/
- public static List<UpdateContainerError>
+ public static ContainerUpdates
validateAndSplitUpdateResourceRequests(RMContext rmContext,
AllocateRequest request, Resource maximumAllocation,
- List<UpdateContainerRequest> increaseResourceReqs,
- List<UpdateContainerRequest> decreaseResourceReqs) {
- List<UpdateContainerError> errors = new ArrayList<>();
+ List<UpdateContainerError> updateErrors) {
+ ContainerUpdates updateRequests =
+ new ContainerUpdates();
Set<ContainerId> outstandingUpdate = new HashSet<>();
for (UpdateContainerRequest updateReq : request.getUpdateRequests()) {
RMContainer rmContainer = rmContext.getScheduler().getRMContainer(
updateReq.getContainerId());
- String msg = null;
- if (rmContainer == null) {
- msg = INVALID_CONTAINER_ID;
- }
- // Only allow updates if the requested version matches the current
- // version
- if (msg == null && updateReq.getContainerVersion() !=
- rmContainer.getContainer().getVersion()) {
- msg = INCORRECT_CONTAINER_VERSION_ERROR + "|"
- + updateReq.getContainerVersion() + "|"
- + rmContainer.getContainer().getVersion();
- }
- // No more than 1 container update per request.
- if (msg == null &&
- outstandingUpdate.contains(updateReq.getContainerId())) {
- msg = UPDATE_OUTSTANDING_ERROR;
- }
+ String msg = validateContainerIdAndVersion(outstandingUpdate,
+ updateReq, rmContainer);
+ ContainerUpdateType updateType = updateReq.getContainerUpdateType();
if (msg == null) {
- Resource original = rmContainer.getContainer().getResource();
- Resource target = updateReq.getCapability();
- if (Resources.fitsIn(target, original)) {
- // This is a decrease request
- if (validateIncreaseDecreaseRequest(rmContext, updateReq,
- maximumAllocation, false)) {
- decreaseResourceReqs.add(updateReq);
- outstandingUpdate.add(updateReq.getContainerId());
+ if ((updateType != ContainerUpdateType.PROMOTE_EXECUTION_TYPE) &&
+ (updateType !=ContainerUpdateType.DEMOTE_EXECUTION_TYPE)) {
+ Resource original = rmContainer.getContainer().getResource();
+ Resource target = updateReq.getCapability();
+ if (Resources.fitsIn(target, original)) {
+ // This is a decrease request
+ if (validateIncreaseDecreaseRequest(rmContext, updateReq,
+ maximumAllocation, false)) {
+ updateRequests.getDecreaseRequests().add(updateReq);
+ outstandingUpdate.add(updateReq.getContainerId());
+ } else {
+ msg = RESOURCE_OUTSIDE_ALLOWED_RANGE;
+ }
} else {
- msg = RESOURCE_OUTSIDE_ALLOWED_RANGE;
+ // This is an increase request
+ if (validateIncreaseDecreaseRequest(rmContext, updateReq,
+ maximumAllocation, true)) {
+ updateRequests.getIncreaseRequests().add(updateReq);
+ outstandingUpdate.add(updateReq.getContainerId());
+ } else {
+ msg = RESOURCE_OUTSIDE_ALLOWED_RANGE;
+ }
}
} else {
- // This is an increase request
- if (validateIncreaseDecreaseRequest(rmContext, updateReq,
- maximumAllocation, true)) {
- increaseResourceReqs.add(updateReq);
- outstandingUpdate.add(updateReq.getContainerId());
- } else {
- msg = RESOURCE_OUTSIDE_ALLOWED_RANGE;
+ ExecutionType original = rmContainer.getExecutionType();
+ ExecutionType target = updateReq.getExecutionType();
+ if (target != original) {
+ if (target == ExecutionType.GUARANTEED &&
+ original == ExecutionType.OPPORTUNISTIC) {
+ updateRequests.getPromotionRequests().add(updateReq);
+ outstandingUpdate.add(updateReq.getContainerId());
+ } else if (target == ExecutionType.OPPORTUNISTIC &&
+ original == ExecutionType.GUARANTEED) {
+ updateRequests.getDemotionRequests().add(updateReq);
+ outstandingUpdate.add(updateReq.getContainerId());
+ }
}
}
}
- if (msg != null) {
- UpdateContainerError updateError = RECORD_FACTORY
- .newRecordInstance(UpdateContainerError.class);
- updateError.setReason(msg);
- updateError.setUpdateContainerRequest(updateReq);
- errors.add(updateError);
- }
+ checkAndcreateUpdateError(updateErrors, updateReq, msg);
+ }
+ return updateRequests;
+ }
+
+ private static void checkAndcreateUpdateError(
+ List<UpdateContainerError> errors, UpdateContainerRequest updateReq,
+ String msg) {
+ if (msg != null) {
+ UpdateContainerError updateError = RECORD_FACTORY
+ .newRecordInstance(UpdateContainerError.class);
+ updateError.setReason(msg);
+ updateError.setUpdateContainerRequest(updateReq);
+ errors.add(updateError);
+ }
+ }
+
+ private static String validateContainerIdAndVersion(
+ Set<ContainerId> outstandingUpdate, UpdateContainerRequest updateReq,
+ RMContainer rmContainer) {
+ String msg = null;
+ if (rmContainer == null) {
+ msg = INVALID_CONTAINER_ID;
+ }
+ // Only allow updates if the requested version matches the current
+ // version
+ if (msg == null && updateReq.getContainerVersion() !=
+ rmContainer.getContainer().getVersion()) {
+ msg = INCORRECT_CONTAINER_VERSION_ERROR + "|"
+ + updateReq.getContainerVersion() + "|"
+ + rmContainer.getContainer().getVersion();
+ }
+ // No more than 1 container update per request.
+ if (msg == null &&
+ outstandingUpdate.contains(updateReq.getContainerId())) {
+ msg = UPDATE_OUTSTANDING_ERROR;
}
- return errors;
+ return msg;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index f7ae488..ab84985 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -95,6 +95,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
@@ -1072,7 +1073,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
Collections.singletonList(appAttempt.amReq),
EMPTY_CONTAINER_RELEASE_LIST,
amBlacklist.getBlacklistAdditions(),
- amBlacklist.getBlacklistRemovals(), null, null);
+ amBlacklist.getBlacklistRemovals(),
+ new ContainerUpdates());
if (amContainerAllocation != null
&& amContainerAllocation.getContainers() != null) {
assert (amContainerAllocation.getContainers().size() == 0);
@@ -1096,7 +1098,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
Allocation amContainerAllocation =
appAttempt.scheduler.allocate(appAttempt.applicationAttemptId,
EMPTY_CONTAINER_REQUEST_LIST, EMPTY_CONTAINER_RELEASE_LIST, null,
- null, null, null);
+ null, new ContainerUpdates());
// There must be at least one container allocated, because a
// CONTAINER_ALLOCATED is emitted after an RMContainer is constructed,
// and is put in SchedulerApplication#newlyAllocatedContainers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index dbc6169..79709a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -108,6 +108,8 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
// Transitions from ACQUIRED state
.addTransition(RMContainerState.ACQUIRED, RMContainerState.RUNNING,
RMContainerEventType.LAUNCHED)
+ .addTransition(RMContainerState.ACQUIRED, RMContainerState.ACQUIRED,
+ RMContainerEventType.ACQUIRED)
.addTransition(RMContainerState.ACQUIRED, RMContainerState.COMPLETED,
RMContainerEventType.FINISHED, new FinishedTransition())
.addTransition(RMContainerState.ACQUIRED, RMContainerState.RELEASED,
@@ -125,6 +127,8 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
.addTransition(RMContainerState.RUNNING, RMContainerState.RELEASED,
RMContainerEventType.RELEASED, new KillTransition())
.addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
+ RMContainerEventType.ACQUIRED)
+ .addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
RMContainerEventType.RESERVED, new ContainerReservedTransition())
.addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
RMContainerEventType.CHANGE_RESOURCE, new ChangeResourceTransition())
@@ -163,13 +167,13 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
private final WriteLock writeLock;
private final ApplicationAttemptId appAttemptId;
private final NodeId nodeId;
- private final Container container;
private final RMContext rmContext;
private final EventHandler eventHandler;
private final ContainerAllocationExpirer containerAllocationExpirer;
private final String user;
private final String nodeLabelExpression;
+ private volatile Container container;
private Resource reservedResource;
private NodeId reservedNode;
private SchedulerRequestKey reservedSchedulerKey;
@@ -188,44 +192,44 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
private boolean isExternallyAllocated;
private SchedulerRequestKey allocatedSchedulerKey;
- public RMContainerImpl(Container container,
+ public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext) {
- this(container, appAttemptId, nodeId, user, rmContext, System
+ this(container, schedulerKey, appAttemptId, nodeId, user, rmContext, System
.currentTimeMillis(), "");
}
- public RMContainerImpl(Container container,
+ public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext, boolean isExternallyAllocated) {
- this(container, appAttemptId, nodeId, user, rmContext, System
+ this(container, schedulerKey, appAttemptId, nodeId, user, rmContext, System
.currentTimeMillis(), "", isExternallyAllocated);
}
private boolean saveNonAMContainerMetaInfo;
- public RMContainerImpl(Container container,
+ public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext, String nodeLabelExpression) {
- this(container, appAttemptId, nodeId, user, rmContext, System
+ this(container, schedulerKey, appAttemptId, nodeId, user, rmContext, System
.currentTimeMillis(), nodeLabelExpression);
}
- public RMContainerImpl(Container container,
+ public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext, long creationTime, String nodeLabelExpression) {
- this(container, appAttemptId, nodeId, user, rmContext, creationTime,
- nodeLabelExpression, false);
+ this(container, schedulerKey, appAttemptId, nodeId, user, rmContext,
+ creationTime, nodeLabelExpression, false);
}
- public RMContainerImpl(Container container,
+ public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext, long creationTime, String nodeLabelExpression,
boolean isExternallyAllocated) {
this.stateMachine = stateMachineFactory.make(this);
this.nodeId = nodeId;
this.container = container;
- this.allocatedSchedulerKey = SchedulerRequestKey.extractFrom(container);
+ this.allocatedSchedulerKey = schedulerKey;
this.appAttemptId = appAttemptId;
this.user = user;
this.creationTime = creationTime;
@@ -276,6 +280,10 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
return this.container;
}
+ public void setContainer(Container container) {
+ this.container = container;
+ }
+
@Override
public RMContainerState getState() {
this.readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index c1a985d..acfcde8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
@@ -62,6 +63,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
@@ -81,6 +83,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdate
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+
+
+import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
@@ -504,9 +511,11 @@ public abstract class AbstractYarnScheduler
ApplicationAttemptId attemptId =
container.getId().getApplicationAttemptId();
RMContainer rmContainer =
- new RMContainerImpl(container, attemptId, node.getNodeID(),
- applications.get(attemptId.getApplicationId()).getUser(), rmContext,
- status.getCreationTime(), status.getNodeLabelExpression());
+ new RMContainerImpl(container,
+ SchedulerRequestKey.extractFrom(container), attemptId,
+ node.getNodeID(), applications.get(
+ attemptId.getApplicationId()).getUser(), rmContext,
+ status.getCreationTime(), status.getNodeLabelExpression());
return rmContainer;
}
@@ -1053,4 +1062,93 @@ public abstract class AbstractYarnScheduler
normalizeRequest(ask);
}
}
+
+ protected void handleExecutionTypeUpdates(
+ SchedulerApplicationAttempt appAttempt,
+ List<UpdateContainerRequest> promotionRequests,
+ List<UpdateContainerRequest> demotionRequests) {
+ if (promotionRequests != null && !promotionRequests.isEmpty()) {
+ LOG.info("Promotion Update requests : " + promotionRequests);
+ handlePromotionRequests(appAttempt, promotionRequests);
+ }
+ if (demotionRequests != null && !demotionRequests.isEmpty()) {
+ LOG.info("Demotion Update requests : " + demotionRequests);
+ handleDemotionRequests(appAttempt, demotionRequests);
+ }
+ }
+
+ private void handlePromotionRequests(
+ SchedulerApplicationAttempt applicationAttempt,
+ List<UpdateContainerRequest> updateContainerRequests) {
+ for (UpdateContainerRequest uReq : updateContainerRequests) {
+ RMContainer rmContainer =
+ rmContext.getScheduler().getRMContainer(uReq.getContainerId());
+ // Check if this is a container update
+ // And not in the middle of a Demotion
+ if (rmContainer != null) {
+ // Check if this is an executionType change request
+ // If so, fix the rr to make it look like a normal rr
+ // with relaxLocality=false and numContainers=1
+ SchedulerNode schedulerNode = rmContext.getScheduler()
+ .getSchedulerNode(rmContainer.getContainer().getNodeId());
+
+ // Add only if no outstanding promote requests exist.
+ if (!applicationAttempt.getUpdateContext()
+ .checkAndAddToOutstandingIncreases(
+ rmContainer, schedulerNode, uReq)) {
+ applicationAttempt.addToUpdateContainerErrors(
+ UpdateContainerError.newInstance(
+ RMServerUtils.UPDATE_OUTSTANDING_ERROR, uReq));
+ }
+ } else {
+ LOG.warn("Cannot promote non-existent (or completed) Container ["
+ + uReq.getContainerId() + "]");
+ }
+ }
+ }
+
+ private void handleDemotionRequests(SchedulerApplicationAttempt appAttempt,
+ List<UpdateContainerRequest> demotionRequests) {
+ OpportunisticContainerContext oppCntxt =
+ appAttempt.getOpportunisticContainerContext();
+ for (UpdateContainerRequest uReq : demotionRequests) {
+ RMContainer rmContainer =
+ rmContext.getScheduler().getRMContainer(uReq.getContainerId());
+ if (rmContainer != null) {
+ if (appAttempt.getUpdateContext().checkAndAddToOutstandingDecreases(
+ rmContainer.getContainer())) {
+ RMContainer demotedRMContainer =
+ createDemotedRMContainer(appAttempt, oppCntxt, rmContainer);
+ appAttempt.addToNewlyDemotedContainers(
+ uReq.getContainerId(), demotedRMContainer);
+ } else {
+ appAttempt.addToUpdateContainerErrors(
+ UpdateContainerError.newInstance(
+ RMServerUtils.UPDATE_OUTSTANDING_ERROR, uReq));
+ }
+ } else {
+ LOG.warn("Cannot demote non-existent (or completed) Container ["
+ + uReq.getContainerId() + "]");
+ }
+ }
+ }
+
+ private RMContainer createDemotedRMContainer(
+ SchedulerApplicationAttempt appAttempt,
+ OpportunisticContainerContext oppCntxt,
+ RMContainer rmContainer) {
+ SchedulerRequestKey sk =
+ SchedulerRequestKey.extractFrom(rmContainer.getContainer());
+ Container demotedContainer = BuilderUtils.newContainer(
+ ContainerId.newContainerId(appAttempt.getApplicationAttemptId(),
+ oppCntxt.getContainerIdGenerator().generateContainerId()),
+ rmContainer.getContainer().getNodeId(),
+ rmContainer.getContainer().getNodeHttpAddress(),
+ rmContainer.getContainer().getResource(),
+ sk.getPriority(), null, ExecutionType.OPPORTUNISTIC,
+ sk.getAllocationRequestId());
+ demotedContainer.setVersion(rmContainer.getContainer().getVersion());
+ return SchedulerUtils.createOpportunisticRmContainer(
+ rmContext, demotedContainer, false);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Allocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Allocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Allocation.java
index b81da2b..43eadab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Allocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Allocation.java
@@ -36,6 +36,8 @@ public class Allocation {
final List<NMToken> nmTokens;
final List<Container> increasedContainers;
final List<Container> decreasedContainers;
+ final List<Container> promotedContainers;
+ final List<Container> demotedContainers;
private Resource resourceLimit;
@@ -50,13 +52,23 @@ public class Allocation {
Set<ContainerId> strictContainers, Set<ContainerId> fungibleContainers,
List<ResourceRequest> fungibleResources, List<NMToken> nmTokens) {
this(containers, resourceLimit,strictContainers, fungibleContainers,
- fungibleResources, nmTokens, null, null);
+ fungibleResources, nmTokens, null, null, null, null);
}
-
+
public Allocation(List<Container> containers, Resource resourceLimit,
Set<ContainerId> strictContainers, Set<ContainerId> fungibleContainers,
List<ResourceRequest> fungibleResources, List<NMToken> nmTokens,
List<Container> increasedContainers, List<Container> decreasedContainer) {
+ this(containers, resourceLimit,strictContainers, fungibleContainers,
+ fungibleResources, nmTokens, increasedContainers, decreasedContainer,
+ null, null);
+ }
+
+ public Allocation(List<Container> containers, Resource resourceLimit,
+ Set<ContainerId> strictContainers, Set<ContainerId> fungibleContainers,
+ List<ResourceRequest> fungibleResources, List<NMToken> nmTokens,
+ List<Container> increasedContainers, List<Container> decreasedContainer,
+ List<Container> promotedContainers, List<Container> demotedContainer) {
this.containers = containers;
this.resourceLimit = resourceLimit;
this.strictContainers = strictContainers;
@@ -65,6 +77,8 @@ public class Allocation {
this.nmTokens = nmTokens;
this.increasedContainers = increasedContainers;
this.decreasedContainers = decreasedContainer;
+ this.promotedContainers = promotedContainers;
+ this.demotedContainers = demotedContainer;
}
public List<Container> getContainers() {
@@ -99,6 +113,14 @@ public class Allocation {
return decreasedContainers;
}
+ public List<Container> getPromotedContainers() {
+ return promotedContainers;
+ }
+
+ public List<Container> getDemotedContainers() {
+ return demotedContainers;
+ }
+
@VisibleForTesting
public void setResourceLimit(Resource resource) {
this.resourceLimit = resource;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 0551df1..d901d90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -55,7 +55,6 @@ import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-
/**
* This class keeps track of all the consumption of an application. This also
* keeps track of current running/completed containers for the application.
@@ -92,10 +91,11 @@ public class AppSchedulingInfo {
final Map<NodeId, Map<SchedulerRequestKey, Map<ContainerId,
SchedContainerChangeRequest>>> containerIncreaseRequestMap =
new ConcurrentHashMap<>();
-
private final ReentrantReadWriteLock.ReadLock readLock;
private final ReentrantReadWriteLock.WriteLock writeLock;
+ public final ContainerUpdateContext updateContext;
+
public AppSchedulingInfo(ApplicationAttemptId appAttemptId,
String user, Queue queue, ActiveUsersManager activeUsersManager,
long epoch, ResourceUsage appResourceUsage) {
@@ -109,6 +109,7 @@ public class AppSchedulingInfo {
this.appResourceUsage = appResourceUsage;
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ updateContext = new ContainerUpdateContext(this);
readLock = lock.readLock();
writeLock = lock.writeLock();
}
@@ -376,6 +377,10 @@ public class AppSchedulingInfo {
}
}
+ public ContainerUpdateContext getUpdateContext() {
+ return updateContext;
+ }
+
/**
* The ApplicationMaster is updating resource requirements for the
* application, by asking for more resources and releasing resources acquired
@@ -413,29 +418,9 @@ public class AppSchedulingInfo {
}
// Update scheduling placement set
- for (Map.Entry<SchedulerRequestKey, Map<String, ResourceRequest>> entry : dedupRequests.entrySet()) {
- SchedulerRequestKey schedulerRequestKey = entry.getKey();
-
- if (!schedulerKeyToPlacementSets.containsKey(schedulerRequestKey)) {
- schedulerKeyToPlacementSets.put(schedulerRequestKey,
- new LocalitySchedulingPlacementSet<>(this));
- }
-
- // Update placement set
- ResourceRequestUpdateResult pendingAmountChanges =
- schedulerKeyToPlacementSets.get(schedulerRequestKey)
- .updateResourceRequests(
- entry.getValue().values(),
- recoverPreemptedRequestForAContainer);
-
- if (null != pendingAmountChanges) {
- updatePendingResources(
- pendingAmountChanges.getLastAnyResourceRequest(),
- pendingAmountChanges.getNewResourceRequest(), schedulerRequestKey,
- queue.getMetrics());
- offswitchResourcesUpdated = true;
- }
- }
+ offswitchResourcesUpdated =
+ addToPlacementSets(
+ recoverPreemptedRequestForAContainer, dedupRequests);
return offswitchResourcesUpdated;
} finally {
@@ -443,6 +428,37 @@ public class AppSchedulingInfo {
}
}
+ boolean addToPlacementSets(
+ boolean recoverPreemptedRequestForAContainer,
+ Map<SchedulerRequestKey, Map<String, ResourceRequest>> dedupRequests) {
+ boolean offswitchResourcesUpdated = false;
+ for (Map.Entry<SchedulerRequestKey, Map<String, ResourceRequest>> entry :
+ dedupRequests.entrySet()) {
+ SchedulerRequestKey schedulerRequestKey = entry.getKey();
+
+ if (!schedulerKeyToPlacementSets.containsKey(schedulerRequestKey)) {
+ schedulerKeyToPlacementSets.put(schedulerRequestKey,
+ new LocalitySchedulingPlacementSet<>(this));
+ }
+
+ // Update placement set
+ ResourceRequestUpdateResult pendingAmountChanges =
+ schedulerKeyToPlacementSets.get(schedulerRequestKey)
+ .updateResourceRequests(
+ entry.getValue().values(),
+ recoverPreemptedRequestForAContainer);
+
+ if (null != pendingAmountChanges) {
+ updatePendingResources(
+ pendingAmountChanges.getLastAnyResourceRequest(),
+ pendingAmountChanges.getNewResourceRequest(), schedulerRequestKey,
+ queue.getMetrics());
+ offswitchResourcesUpdated = true;
+ }
+ }
+ return offswitchResourcesUpdated;
+ }
+
private void updatePendingResources(ResourceRequest lastRequest,
ResourceRequest request, SchedulerRequestKey schedulerKey,
QueueMetrics metrics) {
@@ -717,8 +733,8 @@ public class AppSchedulingInfo {
updateMetricsForAllocatedContainer(type, containerAllocated);
}
- return schedulerKeyToPlacementSets.get(schedulerKey).allocate(type, node,
- request);
+ return schedulerKeyToPlacementSets.get(schedulerKey)
+ .allocate(schedulerKey, type, node, request);
} finally {
writeLock.unlock();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-5416.
TestRMRestart#testRMRestartWaitForPreviousAMToFinish failed intermittently
due to not wait SchedulerApplicationAttempt to be stopped. Contributed by
Junping Du
Posted by st...@apache.org.
YARN-5416. TestRMRestart#testRMRestartWaitForPreviousAMToFinish failed intermittently due to not wait SchedulerApplicationAttempt to be stopped. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/357eab95
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/357eab95
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/357eab95
Branch: refs/heads/HADOOP-13345
Commit: 357eab95668dbc419239857ac5ce763d76fd40e7
Parents: a5ec1e3
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jan 11 17:13:37 2017 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Jan 11 17:13:37 2017 +0000
----------------------------------------------------------------------
.../yarn/server/resourcemanager/TestRMRestart.java | 14 +++++++++++---
.../resourcemanager/scheduler/TestSchedulerUtils.java | 8 ++++----
2 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/357eab95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 9223ef3..30a7ef6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -107,6 +107,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
@@ -510,10 +511,12 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
// start RM
final MockRM rm1 = createMockRM(conf, memStore);
rm1.start();
+ AbstractYarnScheduler ys =
+ (AbstractYarnScheduler)rm1.getResourceScheduler();
MockNM nm1 =
new MockNM("127.0.0.1:1234" , 16382, rm1.getResourceTrackerService());
nm1.registerNode();
-
+
// submitting app
RMApp app1 = rm1.submitApp(200);
rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
@@ -521,10 +524,11 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
// Fail first AM.
rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
-
+ TestSchedulerUtils.waitSchedulerApplicationAttemptStopped(ys,
+ am1.getApplicationAttemptId());
// launch another AM.
MockAM am2 = launchAM(app1, rm1, nm1);
-
+
Assert.assertEquals(1, rmAppState.size());
Assert.assertEquals(app1.getState(), RMAppState.RUNNING);
Assert.assertEquals(app1.getAppAttempts()
@@ -562,6 +566,10 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
nm1.registerNode(Arrays.asList(status), null);
rm2.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED);
+ ys = (AbstractYarnScheduler) rm2.getResourceScheduler();
+ TestSchedulerUtils.waitSchedulerApplicationAttemptStopped(ys,
+ am2.getApplicationAttemptId());
+
launchAM(rmApp, rm2, nm1);
Assert.assertEquals(3, rmApp.getAppAttempts().size());
rm2.waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/357eab95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index 98e1e1e..854a4f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -75,8 +75,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
@@ -777,9 +775,11 @@ public class TestSchedulerUtils {
}
}
- public static void waitSchedulerApplicationAttemptStopped(CapacityScheduler cs,
+ public static void waitSchedulerApplicationAttemptStopped(
+ AbstractYarnScheduler ys,
ApplicationAttemptId attemptId) throws InterruptedException {
- FiCaSchedulerApp schedulerApp = cs.getApplicationAttempt(attemptId);
+ SchedulerApplicationAttempt schedulerApp =
+ ys.getApplicationAttempt(attemptId);
if (null == schedulerApp) {
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-6068. Log aggregation get
failed when NM restart even with recovery (Junping Du via Varun Saxena)
Posted by st...@apache.org.
YARN-6068. Log aggregation get failed when NM restart even with recovery (Junping Du via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f59e36b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f59e36b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f59e36b4
Branch: refs/heads/HADOOP-13345
Commit: f59e36b4ce71d3019ab91b136b6d7646316954e7
Parents: 94f1d06
Author: Varun Saxena <va...@apache.org>
Authored: Mon Jan 9 11:17:19 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Mon Jan 9 11:17:19 2017 +0530
----------------------------------------------------------------------
.../containermanager/logaggregation/AppLogAggregatorImpl.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59e36b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 07275b1..065854e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -498,8 +498,8 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
+ appId, e);
doAppLogAggregationPostCleanUp();
} finally {
- if (!this.appAggregationFinished.get()) {
- LOG.warn("Aggregation did not complete for application " + appId);
+ if (!this.appAggregationFinished.get() && !this.aborted.get()) {
+ LOG.warn("Log aggregation did not complete for application " + appId);
this.dispatcher.getEventHandler().handle(
new ApplicationEvent(this.appId,
ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HADOOP-13903. Improvements to KMS
logging to help debug authorization errors. (Tristan Stevens via asuresh)
Posted by st...@apache.org.
HADOOP-13903. Improvements to KMS logging to help debug authorization errors. (Tristan Stevens via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be529dad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be529dad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be529dad
Branch: refs/heads/HADOOP-13345
Commit: be529dade182dd2f3718fc52133f43e83dce191f
Parents: 467f5f1
Author: Arun Suresh <as...@apache.org>
Authored: Wed Jan 11 00:26:02 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 11 00:26:02 2017 -0800
----------------------------------------------------------------------
.../hadoop/crypto/key/kms/server/KMSACLs.java | 36 ++++++++++++++++++--
.../hadoop/crypto/key/kms/server/KMSAudit.java | 30 ++++++++++++++--
.../crypto/key/kms/server/KMSAuditLogger.java | 21 ++++++++++--
3 files changed, 80 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be529dad/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
index c36fcf8..096f756 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
@@ -236,9 +236,26 @@ public class KMSACLs implements Runnable, KeyACLs {
*/
public boolean hasAccess(Type type, UserGroupInformation ugi) {
boolean access = acls.get(type).isUserAllowed(ugi);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Checking user [{}] for: {} {} ", ugi.getShortUserName(),
+ type.toString(), acls.get(type).getAclString());
+ }
if (access) {
AccessControlList blacklist = blacklistedAcls.get(type);
access = (blacklist == null) || !blacklist.isUserInList(ugi);
+ if (LOG.isDebugEnabled()) {
+ if (blacklist == null) {
+ LOG.debug("No blacklist for {}", type.toString());
+ } else if (access) {
+ LOG.debug("user is in {}" , blacklist.getAclString());
+ } else {
+ LOG.debug("user is not in {}" , blacklist.getAclString());
+ }
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("User: [{}], Type: {} Result: {}", ugi.getShortUserName(),
+ type.toString(), access);
}
return access;
}
@@ -259,8 +276,12 @@ public class KMSACLs implements Runnable, KeyACLs {
@Override
public boolean hasAccessToKey(String keyName, UserGroupInformation ugi,
KeyOpType opType) {
- return checkKeyAccess(keyName, ugi, opType)
+ boolean access = checkKeyAccess(keyName, ugi, opType)
|| checkKeyAccess(whitelistKeyAcls, ugi, opType);
+ if (!access) {
+ KMSWebApp.getKMSAudit().unauthorized(ugi, opType, keyName);
+ }
+ return access;
}
private boolean checkKeyAccess(String keyName, UserGroupInformation ugi,
@@ -269,9 +290,15 @@ public class KMSACLs implements Runnable, KeyACLs {
if (keyAcl == null) {
// If No key acl defined for this key, check to see if
// there are key defaults configured for this operation
+ LOG.debug("Key: {} has no ACLs defined, using defaults.", keyName);
keyAcl = defaultKeyAcls;
}
- return checkKeyAccess(keyAcl, ugi, opType);
+ boolean access = checkKeyAccess(keyAcl, ugi, opType);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("User: [{}], OpType: {}, KeyName: {} Result: {}",
+ ugi.getShortUserName(), opType.toString(), keyName, access);
+ }
+ return access;
}
private boolean checkKeyAccess(Map<KeyOpType, AccessControlList> keyAcl,
@@ -280,8 +307,13 @@ public class KMSACLs implements Runnable, KeyACLs {
if (acl == null) {
// If no acl is specified for this operation,
// deny access
+ LOG.debug("No ACL available for key, denying access for {}", opType);
return false;
} else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Checking user [{}] for: {}: {}" + ugi.getShortUserName(),
+ opType.toString(), acl.getAclString());
+ }
return acl.isUserAllowed(ugi);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be529dad/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
index 7fdfc34..13a2d5c 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
@@ -23,6 +23,8 @@ import static org.apache.hadoop.crypto.key.kms.server.KMSAuditLogger.OpStatus;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.server.KMSACLs.Type;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
@@ -168,7 +170,25 @@ public class KMSAudit {
}
}
- private void op(final OpStatus opStatus, final KMS.KMSOp op,
+ /**
+ * Logs to the audit service a single operation on the KMS or on a key.
+ *
+ * @param opStatus
+ * The outcome of the audited event
+ * @param op
+ * The operation being audited (either {@link KMS.KMSOp} or
+ * {@link Type} N.B this is passed as an {@link Object} to allow
+ * either enum to be passed in.
+ * @param ugi
+ * The user's security context
+ * @param key
+ * The String name of the key if applicable
+ * @param remoteHost
+ * The hostname of the requesting service
+ * @param extraMsg
+ * Any extra details for auditing
+ */
+ private void op(final OpStatus opStatus, final Object op,
final UserGroupInformation ugi, final String key, final String remoteHost,
final String extraMsg) {
final String user = ugi == null ? null: ugi.getShortUserName();
@@ -215,6 +235,12 @@ public class KMSAudit {
op(OpStatus.UNAUTHORIZED, op, user, key, "Unknown", "");
}
+ public void unauthorized(UserGroupInformation user, KeyOpType op,
+ String key) {
+ op(OpStatus.UNAUTHORIZED, op, user, key, "Unknown", "");
+
+ }
+
public void error(UserGroupInformation user, String method, String url,
String extraMsg) {
op(OpStatus.ERROR, null, user, null, "Unknown", "Method:'" + method
@@ -228,7 +254,7 @@ public class KMSAudit {
+ " URL:" + url + " ErrorMsg:'" + extraMsg + "'");
}
- private static String createCacheKey(String user, String key, KMS.KMSOp op) {
+ private static String createCacheKey(String user, String key, Object op) {
return user + "#" + key + "#" + op;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be529dad/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuditLogger.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuditLogger.java
index a539724..2e2ba1d 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuditLogger.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuditLogger.java
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.server.KMSACLs.Type;
import org.apache.hadoop.security.UserGroupInformation;
/**
@@ -46,7 +47,7 @@ interface KMSAuditLogger {
*/
class AuditEvent {
private final AtomicLong accessCount = new AtomicLong(-1);
- private final KMS.KMSOp op;
+ private final Object op;
private final String keyName;
private final String user;
private final String impersonator;
@@ -55,7 +56,21 @@ interface KMSAuditLogger {
private final long startTime = System.currentTimeMillis();
private long endTime = startTime;
- AuditEvent(KMS.KMSOp op, UserGroupInformation ugi, String keyName,
+ /**
+ * @param op
+ * The operation being audited (either {@link KMS.KMSOp} or
+ * {@link Type} N.B this is passed as an {@link Object} to allow
+ * either enum to be passed in.
+ * @param ugi
+ * The user's security context
+ * @param keyName
+ * The String name of the key if applicable
+ * @param remoteHost
+ * The hostname of the requesting service
+ * @param msg
+ * Any extra details for auditing
+ */
+ AuditEvent(Object op, UserGroupInformation ugi, String keyName,
String remoteHost, String msg) {
this.keyName = keyName;
if (ugi == null) {
@@ -79,7 +94,7 @@ interface KMSAuditLogger {
return accessCount;
}
- public KMS.KMSOp getOp() {
+ public Object getOp() {
return op;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDFS-11292. log lastWrittenTxId
etc info in logSyncAll. Contributed by Yongjun Zhang.
Posted by st...@apache.org.
HDFS-11292. log lastWrittenTxId etc info in logSyncAll. Contributed by Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/603cbcd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/603cbcd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/603cbcd5
Branch: refs/heads/HADOOP-13345
Commit: 603cbcd513a74c29e0e4ec9dc181ff08887d64a4
Parents: 0a212a4
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Mon Jan 9 15:44:42 2017 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Mon Jan 9 15:44:42 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/FSEditLog.java | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/603cbcd5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index c9ee32b..8454a46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -596,7 +596,15 @@ public class FSEditLog implements LogsPurgeable {
*/
void logSyncAll() {
// Make sure we're synced up to the most recent transaction ID.
- logSync(getLastWrittenTxId());
+ long lastWrittenTxId = getLastWrittenTxId();
+ LOG.info("logSyncAll toSyncToTxId=" + lastWrittenTxId
+ + " lastSyncedTxid=" + synctxid
+ + " mostRecentTxid=" + txid);
+ logSync(lastWrittenTxId);
+ lastWrittenTxId = getLastWrittenTxId();
+ LOG.info("Done logSyncAll lastWrittenTxId=" + lastWrittenTxId
+ + " lastSyncedTxid=" + synctxid
+ + " mostRecentTxid=" + txid);
}
/**
@@ -1358,7 +1366,8 @@ public class FSEditLog implements LogsPurgeable {
* Transitions from IN_SEGMENT state to BETWEEN_LOG_SEGMENTS state.
*/
public synchronized void endCurrentLogSegment(boolean writeEndTxn) {
- LOG.info("Ending log segment " + curSegmentTxId);
+ LOG.info("Ending log segment " + curSegmentTxId +
+ ", " + getLastWrittenTxId());
Preconditions.checkState(isSegmentOpen(),
"Bad state: %s", state);
@@ -1372,7 +1381,10 @@ public class FSEditLog implements LogsPurgeable {
printStatistics(true);
final long lastTxId = getLastWrittenTxId();
-
+ final long lastSyncedTxId = getSyncTxId();
+ Preconditions.checkArgument(lastTxId == lastSyncedTxId,
+ "LastWrittenTxId %s is expected to be the same as lastSyncedTxId %s",
+ lastTxId, lastSyncedTxId);
try {
journalSet.finalizeLogSegment(curSegmentTxId, lastTxId);
editLogStream = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HDFS-11280. Allow WebHDFS to reuse
HTTP connections to NN. Contributed by Zheng Shao.
Posted by st...@apache.org.
HDFS-11280. Allow WebHDFS to reuse HTTP connections to NN. Contributed by Zheng Shao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a605ff36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a605ff36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a605ff36
Branch: refs/heads/HADOOP-13345
Commit: a605ff36a53a3d1283c3f6d81eb073e4a2942143
Parents: 5ed63e3
Author: Haohui Mai <wh...@apache.org>
Authored: Wed Jan 4 21:01:23 2017 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Wed Jan 4 21:01:23 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a605ff36/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index d4fa009..135eef7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -659,6 +659,8 @@ public class WebHdfsFileSystem extends FileSystem
url = new URL(conn.getHeaderField("Location"));
redirectHost = url.getHost() + ":" + url.getPort();
} finally {
+ // TODO: consider not calling conn.disconnect() to allow connection reuse
+ // See http://tinyurl.com/java7-http-keepalive
conn.disconnect();
}
}
@@ -891,7 +893,9 @@ public class WebHdfsFileSystem extends FileSystem
LOG.debug("Response decoding failure.", e);
throw ioe;
} finally {
- conn.disconnect();
+ // Don't call conn.disconnect() to allow connection reuse
+ // See http://tinyurl.com/java7-http-keepalive
+ conn.getInputStream().close();
}
}
@@ -938,6 +942,9 @@ public class WebHdfsFileSystem extends FileSystem
try {
validateResponse(op, conn, true);
} finally {
+ // This is a connection to DataNode. Let's disconnect since
+ // there is little chance that the connection will be reused
+ // any time soonl
conn.disconnect();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-5959. RM changes to support
change of container ExecutionType. (Arun Suresh via wangda)
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
new file mode 100644
index 0000000..7381250
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerError;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Class encapsulates all outstanding container increase and decrease
+ * requests for an application.
+ */
+public class ContainerUpdateContext {
+
+ public static final ContainerId UNDEFINED =
+ ContainerId.newContainerId(ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(-1, -1), -1), -1);
+ protected static final RecordFactory RECORD_FACTORY =
+ RecordFactoryProvider.getRecordFactory(null);
+
+ // Keep track of containers that are undergoing promotion
+ private final Map<SchedulerRequestKey, Map<Resource,
+ Map<NodeId, Set<ContainerId>>>> outstandingIncreases = new HashMap<>();
+
+ private final Set<ContainerId> outstandingDecreases = new HashSet<>();
+ private final AppSchedulingInfo appSchedulingInfo;
+
+ ContainerUpdateContext(AppSchedulingInfo appSchedulingInfo) {
+ this.appSchedulingInfo = appSchedulingInfo;
+ }
+
+ private synchronized boolean isBeingIncreased(Container container) {
+ Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap =
+ outstandingIncreases.get(
+ new SchedulerRequestKey(container.getPriority(),
+ container.getAllocationRequestId(), container.getId()));
+ if (resourceMap != null) {
+ Map<NodeId, Set<ContainerId>> locationMap =
+ resourceMap.get(container.getResource());
+ if (locationMap != null) {
+ Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
+ if (containerIds != null && !containerIds.isEmpty()) {
+ return containerIds.contains(container.getId());
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Add the container to outstanding decreases.
+ * @param container Container.
+ * @return true if updated to outstanding decreases was successful.
+ */
+ public synchronized boolean checkAndAddToOutstandingDecreases(
+ Container container) {
+ if (isBeingIncreased(container)
+ || outstandingDecreases.contains(container.getId())) {
+ return false;
+ }
+ outstandingDecreases.add(container.getId());
+ return true;
+ }
+
+ /**
+ * Add the container to outstanding increases.
+ * @param rmContainer RMContainer.
+ * @param schedulerNode SchedulerNode.
+ * @param updateRequest UpdateContainerRequest.
+ * @return true if updated to outstanding increases was successful.
+ */
+ public synchronized boolean checkAndAddToOutstandingIncreases(
+ RMContainer rmContainer, SchedulerNode schedulerNode,
+ UpdateContainerRequest updateRequest) {
+ Container container = rmContainer.getContainer();
+ SchedulerRequestKey schedulerKey =
+ SchedulerRequestKey.create(updateRequest,
+ rmContainer.getAllocatedSchedulerKey());
+ Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap =
+ outstandingIncreases.get(schedulerKey);
+ if (resourceMap == null) {
+ resourceMap = new HashMap<>();
+ outstandingIncreases.put(schedulerKey, resourceMap);
+ }
+ Map<NodeId, Set<ContainerId>> locationMap =
+ resourceMap.get(container.getResource());
+ if (locationMap == null) {
+ locationMap = new HashMap<>();
+ resourceMap.put(container.getResource(), locationMap);
+ }
+ Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
+ if (containerIds == null) {
+ containerIds = new HashSet<>();
+ locationMap.put(container.getNodeId(), containerIds);
+ }
+ if (containerIds.contains(container.getId())
+ || outstandingDecreases.contains(container.getId())) {
+ return false;
+ }
+ containerIds.add(container.getId());
+
+ Map<SchedulerRequestKey, Map<String, ResourceRequest>> updateResReqs =
+ new HashMap<>();
+ Resource resToIncrease = getResourceToIncrease(updateRequest, rmContainer);
+ Map<String, ResourceRequest> resMap =
+ createResourceRequests(rmContainer, schedulerNode,
+ schedulerKey, resToIncrease);
+ updateResReqs.put(schedulerKey, resMap);
+ appSchedulingInfo.addToPlacementSets(false, updateResReqs);
+ return true;
+ }
+
+ private Map<String, ResourceRequest> createResourceRequests(
+ RMContainer rmContainer, SchedulerNode schedulerNode,
+ SchedulerRequestKey schedulerKey, Resource resToIncrease) {
+ Map<String, ResourceRequest> resMap = new HashMap<>();
+ resMap.put(rmContainer.getContainer().getNodeId().getHost(),
+ createResourceReqForIncrease(schedulerKey, resToIncrease,
+ RECORD_FACTORY.newRecordInstance(ResourceRequest.class),
+ rmContainer, rmContainer.getContainer().getNodeId().getHost()));
+ resMap.put(schedulerNode.getRackName(),
+ createResourceReqForIncrease(schedulerKey, resToIncrease,
+ RECORD_FACTORY.newRecordInstance(ResourceRequest.class),
+ rmContainer, schedulerNode.getRackName()));
+ resMap.put(ResourceRequest.ANY,
+ createResourceReqForIncrease(schedulerKey, resToIncrease,
+ RECORD_FACTORY.newRecordInstance(ResourceRequest.class),
+ rmContainer, ResourceRequest.ANY));
+ return resMap;
+ }
+
+ private Resource getResourceToIncrease(UpdateContainerRequest updateReq,
+ RMContainer rmContainer) {
+ if (updateReq.getContainerUpdateType() ==
+ ContainerUpdateType.PROMOTE_EXECUTION_TYPE) {
+ return rmContainer.getContainer().getResource();
+ }
+ // TODO: Fix this for container increase..
+ // This has to equal the Resources in excess of fitsIn()
+ // for container increase and is equal to the container total
+ // resource for Promotion.
+ return null;
+ }
+
+ private static ResourceRequest createResourceReqForIncrease(
+ SchedulerRequestKey schedulerRequestKey, Resource resToIncrease,
+ ResourceRequest rr, RMContainer rmContainer, String resourceName) {
+ rr.setResourceName(resourceName);
+ rr.setNumContainers(1);
+ rr.setRelaxLocality(false);
+ rr.setPriority(rmContainer.getContainer().getPriority());
+ rr.setAllocationRequestId(schedulerRequestKey.getAllocationRequestId());
+ rr.setCapability(resToIncrease);
+ rr.setNodeLabelExpression(rmContainer.getNodeLabelExpression());
+ rr.setExecutionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true));
+ return rr;
+ }
+
+ /**
+ * Remove Container from outstanding increases / decreases. Calling this
+ * method essentially completes the update process.
+ * @param schedulerKey SchedulerRequestKey.
+ * @param container Container.
+ */
+ public synchronized void removeFromOutstandingUpdate(
+ SchedulerRequestKey schedulerKey, Container container) {
+ Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap =
+ outstandingIncreases.get(schedulerKey);
+ if (resourceMap != null) {
+ Map<NodeId, Set<ContainerId>> locationMap =
+ resourceMap.get(container.getResource());
+ if (locationMap != null) {
+ Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
+ if (containerIds != null && !containerIds.isEmpty()) {
+ containerIds.remove(container.getId());
+ if (containerIds.isEmpty()) {
+ locationMap.remove(container.getNodeId());
+ }
+ }
+ if (locationMap.isEmpty()) {
+ resourceMap.remove(container.getResource());
+ }
+ }
+ if (resourceMap.isEmpty()) {
+ outstandingIncreases.remove(schedulerKey);
+ }
+ }
+ outstandingDecreases.remove(container.getId());
+ }
+
+ /**
+ * Check if a new container is to be matched up against an outstanding
+ * Container increase request.
+ * @param schedulerKey SchedulerRequestKey.
+ * @param rmContainer RMContainer.
+ * @return ContainerId.
+ */
+ public ContainerId matchContainerToOutstandingIncreaseReq(
+ SchedulerNode node, SchedulerRequestKey schedulerKey,
+ RMContainer rmContainer) {
+ ContainerId retVal = null;
+ Container container = rmContainer.getContainer();
+ Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap =
+ outstandingIncreases.get(schedulerKey);
+ if (resourceMap != null) {
+ Map<NodeId, Set<ContainerId>> locationMap =
+ resourceMap.get(container.getResource());
+ if (locationMap != null) {
+ Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
+ if (containerIds != null && !containerIds.isEmpty()) {
+ retVal = containerIds.iterator().next();
+ }
+ }
+ }
+ // Allocation happened on NM on the same host, but not on the NM
+ // we need.. We need to signal that this container has to be released.
+ // We also need to add these requests back.. to be reallocated.
+ if (resourceMap != null && retVal == null) {
+ Map<SchedulerRequestKey, Map<String, ResourceRequest>> reqsToUpdate =
+ new HashMap<>();
+ Map<String, ResourceRequest> resMap = createResourceRequests
+ (rmContainer, node, schedulerKey,
+ rmContainer.getContainer().getResource());
+ reqsToUpdate.put(schedulerKey, resMap);
+ appSchedulingInfo.addToPlacementSets(true, reqsToUpdate);
+ return UNDEFINED;
+ }
+ return retVal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdates.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdates.java
new file mode 100644
index 0000000..77b1545
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdates.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Holder class that maintains list of container update requests
+ */
+public class ContainerUpdates {
+
+ final List<UpdateContainerRequest> increaseRequests = new ArrayList<>();
+ final List<UpdateContainerRequest> decreaseRequests = new ArrayList<>();
+ final List<UpdateContainerRequest> promotionRequests = new ArrayList<>();
+ final List<UpdateContainerRequest> demotionRequests = new ArrayList<>();
+
+ /**
+ * Returns Container Increase Requests.
+ * @return Container Increase Requests.
+ */
+ public List<UpdateContainerRequest> getIncreaseRequests() {
+ return increaseRequests;
+ }
+
+ /**
+ * Returns Container Decrease Requests.
+ * @return Container Decrease Requests.
+ */
+ public List<UpdateContainerRequest> getDecreaseRequests() {
+ return decreaseRequests;
+ }
+
+ /**
+ * Returns Container Promotion Requests.
+ * @return Container Promotion Requests.
+ */
+ public List<UpdateContainerRequest> getPromotionRequests() {
+ return promotionRequests;
+ }
+
+ /**
+ * Returns Container Demotion Requests.
+ * @return Container Demotion Requests.
+ */
+ public List<UpdateContainerRequest> getDemotionRequests() {
+ return demotionRequests;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index e94d800..4a8b2da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -47,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NMToken;
@@ -54,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -133,10 +136,15 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private AtomicLong firstContainerAllocatedTime = new AtomicLong(0);
protected List<RMContainer> newlyAllocatedContainers = new ArrayList<>();
+ protected Map<ContainerId, RMContainer> newlyPromotedContainers = new HashMap<>();
+ protected Map<ContainerId, RMContainer> newlyDemotedContainers = new HashMap<>();
+ protected List<RMContainer> tempContainerToKill = new ArrayList<>();
protected Map<ContainerId, RMContainer> newlyDecreasedContainers = new HashMap<>();
protected Map<ContainerId, RMContainer> newlyIncreasedContainers = new HashMap<>();
protected Set<NMToken> updatedNMTokens = new HashSet<>();
+ protected List<UpdateContainerError> updateContainerErrors = new ArrayList<>();
+
// This pendingRelease is used in work-preserving recovery scenario to keep
// track of the AM's outstanding release requests. RM on recovery could
// receive the release request form AM before it receives the container status
@@ -247,6 +255,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return this.appSchedulingInfo;
}
+ public ContainerUpdateContext getUpdateContext() {
+ return this.appSchedulingInfo.getUpdateContext();
+ }
+
/**
* Is this application pending?
* @return true if it is else false.
@@ -537,8 +549,9 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
writeLock.lock();
// Create RMContainer if necessary
if (rmContainer == null) {
- rmContainer = new RMContainerImpl(container, getApplicationAttemptId(),
- node.getNodeID(), appSchedulingInfo.getUser(), rmContext);
+ rmContainer = new RMContainerImpl(container, schedulerKey,
+ getApplicationAttemptId(), node.getNodeID(),
+ appSchedulingInfo.getUser(), rmContext);
}
if (rmContainer.getState() == RMContainerState.NEW) {
attemptResourceUsage.incReserved(node.getPartition(),
@@ -635,10 +648,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
}
private Container updateContainerAndNMToken(RMContainer rmContainer,
- boolean newContainer, boolean increasedContainer) {
+ ContainerUpdateType updateType) {
Container container = rmContainer.getContainer();
ContainerType containerType = ContainerType.TASK;
- if (!newContainer) {
+ if (updateType != null) {
container.setVersion(container.getVersion() + 1);
}
// The working knowledge is that masterContainer for AM is null as it
@@ -662,12 +675,15 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return null;
}
- if (newContainer) {
+ if (updateType == null ||
+ ContainerUpdateType.PROMOTE_EXECUTION_TYPE == updateType ||
+ ContainerUpdateType.DEMOTE_EXECUTION_TYPE == updateType) {
rmContainer.handle(new RMContainerEvent(
rmContainer.getContainerId(), RMContainerEventType.ACQUIRED));
} else {
rmContainer.handle(new RMContainerUpdatesAcquiredEvent(
- rmContainer.getContainerId(), increasedContainer));
+ rmContainer.getContainerId(),
+ ContainerUpdateType.INCREASE_RESOURCE == updateType));
}
return container;
}
@@ -699,8 +715,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
Iterator<RMContainer> i = newlyAllocatedContainers.iterator();
while (i.hasNext()) {
RMContainer rmContainer = i.next();
- Container updatedContainer = updateContainerAndNMToken(rmContainer,
- true, false);
+ Container updatedContainer =
+ updateContainerAndNMToken(rmContainer, null);
// Only add container to return list when it's not null.
// updatedContainer could be null when generate token failed, it can be
// caused by DNS resolving failed.
@@ -713,9 +729,142 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
} finally {
writeLock.unlock();
}
+ }
+ public void addToNewlyDemotedContainers(ContainerId containerId,
+ RMContainer rmContainer) {
+ newlyDemotedContainers.put(containerId, rmContainer);
}
-
+
+ protected synchronized void addToUpdateContainerErrors(
+ UpdateContainerError error) {
+ updateContainerErrors.add(error);
+ }
+
+ protected synchronized void addToNewlyAllocatedContainers(
+ SchedulerNode node, RMContainer rmContainer) {
+ if (oppContainerContext == null) {
+ newlyAllocatedContainers.add(rmContainer);
+ return;
+ }
+ ContainerId matchedContainerId =
+ getUpdateContext().matchContainerToOutstandingIncreaseReq(
+ node, rmContainer.getAllocatedSchedulerKey(), rmContainer);
+ if (matchedContainerId != null) {
+ if (ContainerUpdateContext.UNDEFINED == matchedContainerId) {
+ // This is a spurious allocation (relaxLocality = false
+ // resulted in the Container being allocated on an NM on the same host
+ // but not on the NM running the container to be updated. Can
+ // happen if more than one NM exists on the same host.. usually
+ // occurs when using MiniYARNCluster to test).
+ tempContainerToKill.add(rmContainer);
+ } else {
+ newlyPromotedContainers.put(matchedContainerId, rmContainer);
+ }
+ } else {
+ newlyAllocatedContainers.add(rmContainer);
+ }
+ }
+
+ public List<Container> pullNewlyPromotedContainers() {
+ return pullContainersWithUpdatedExecType(newlyPromotedContainers,
+ ContainerUpdateType.PROMOTE_EXECUTION_TYPE);
+ }
+
+ public List<Container> pullNewlyDemotedContainers() {
+ return pullContainersWithUpdatedExecType(newlyDemotedContainers,
+ ContainerUpdateType.DEMOTE_EXECUTION_TYPE);
+ }
+
+ public List<UpdateContainerError> pullUpdateContainerErrors() {
+ List<UpdateContainerError> errors =
+ new ArrayList<>(updateContainerErrors);
+ updateContainerErrors.clear();
+ return errors;
+ }
+
+ /**
+ * A container is promoted if its executionType is changed from
+ * OPPORTUNISTIC to GUARANTEED. It id demoted if the change is from
+ * GUARANTEED to OPPORTUNISTIC.
+ * @return Newly Promoted and Demoted containers
+ */
+ private List<Container> pullContainersWithUpdatedExecType(
+ Map<ContainerId, RMContainer> newlyUpdatedContainers,
+ ContainerUpdateType updateTpe) {
+ List<Container> updatedContainers = new ArrayList<>();
+ if (oppContainerContext == null) {
+ return updatedContainers;
+ }
+ try {
+ writeLock.lock();
+ Iterator<Map.Entry<ContainerId, RMContainer>> i =
+ newlyUpdatedContainers.entrySet().iterator();
+ while (i.hasNext()) {
+ Map.Entry<ContainerId, RMContainer> entry = i.next();
+ ContainerId matchedContainerId = entry.getKey();
+ RMContainer rmContainer = entry.getValue();
+
+ // swap containers
+ RMContainer existingRMContainer = swapContainer(
+ rmContainer, matchedContainerId);
+ getUpdateContext().removeFromOutstandingUpdate(
+ rmContainer.getAllocatedSchedulerKey(),
+ existingRMContainer.getContainer());
+ Container updatedContainer = updateContainerAndNMToken(
+ existingRMContainer, updateTpe);
+ updatedContainers.add(updatedContainer);
+
+ tempContainerToKill.add(rmContainer);
+ i.remove();
+ }
+ // Release all temporary containers
+ Iterator<RMContainer> tempIter = tempContainerToKill.iterator();
+ while (tempIter.hasNext()) {
+ RMContainer c = tempIter.next();
+ // Mark container for release (set RRs to null, so RM does not think
+ // it is a recoverable container)
+ ((RMContainerImpl) c).setResourceRequests(null);
+ ((AbstractYarnScheduler) rmContext.getScheduler()).completedContainer(c,
+ SchedulerUtils.createAbnormalContainerStatus(c.getContainerId(),
+ SchedulerUtils.UPDATED_CONTAINER),
+ RMContainerEventType.KILL);
+ tempIter.remove();
+ }
+ return updatedContainers;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private RMContainer swapContainer(RMContainer rmContainer, ContainerId
+ matchedContainerId) {
+ RMContainer existingRMContainer =
+ getRMContainer(matchedContainerId);
+ if (existingRMContainer != null) {
+ // Swap updated container with the existing container
+ Container updatedContainer = rmContainer.getContainer();
+
+ Container newContainer = Container.newInstance(matchedContainerId,
+ existingRMContainer.getContainer().getNodeId(),
+ existingRMContainer.getContainer().getNodeHttpAddress(),
+ updatedContainer.getResource(),
+ existingRMContainer.getContainer().getPriority(), null,
+ updatedContainer.getExecutionType());
+ newContainer.setAllocationRequestId(
+ existingRMContainer.getContainer().getAllocationRequestId());
+ newContainer.setVersion(existingRMContainer.getContainer().getVersion());
+
+ rmContainer.getContainer().setResource(
+ existingRMContainer.getContainer().getResource());
+ rmContainer.getContainer().setExecutionType(
+ existingRMContainer.getContainer().getExecutionType());
+
+ ((RMContainerImpl)existingRMContainer).setContainer(newContainer);
+ }
+ return existingRMContainer;
+ }
+
private List<Container> pullNewlyUpdatedContainers(
Map<ContainerId, RMContainer> updatedContainerMap, boolean increase) {
try {
@@ -728,7 +877,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
while (i.hasNext()) {
RMContainer rmContainer = i.next().getValue();
Container updatedContainer = updateContainerAndNMToken(rmContainer,
- false, increase);
+ increase ? ContainerUpdateType.INCREASE_RESOURCE :
+ ContainerUpdateType.DECREASE_RESOURCE);
if (updatedContainer != null) {
returnContainerList.add(updatedContainer);
i.remove();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index b227523..5360665 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -41,7 +42,10 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -57,6 +61,9 @@ public class SchedulerUtils {
public static final String RELEASED_CONTAINER =
"Container released by application";
+
+ public static final String UPDATED_CONTAINER =
+ "Temporary container killed by application for ExeutionType update";
public static final String LOST_CONTAINER =
"Container released on a *lost* node";
@@ -376,4 +383,19 @@ public class SchedulerUtils {
}
return hasPendingResourceRequest(rc, usage, partitionToLookAt, cluster);
}
+
+ public static RMContainer createOpportunisticRmContainer(RMContext rmContext,
+ Container container, boolean isRemotelyAllocated) {
+ SchedulerApplicationAttempt appAttempt =
+ ((AbstractYarnScheduler) rmContext.getScheduler())
+ .getCurrentAttemptForContainer(container.getId());
+ RMContainer rmContainer = new RMContainerImpl(container,
+ SchedulerRequestKey.extractFrom(container),
+ appAttempt.getApplicationAttemptId(), container.getNodeId(),
+ appAttempt.getUser(), rmContext, isRemotelyAllocated);
+ appAttempt.addRMContainer(container.getId(), rmContainer);
+ ((AbstractYarnScheduler) rmContext.getScheduler()).getNode(
+ container.getNodeId()).allocateContainer(rmContainer);
+ return rmContainer;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index ea1ae60..3185dc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -42,8 +43,6 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -137,8 +136,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
* @param release
* @param blacklistAdditions
* @param blacklistRemovals
- * @param increaseRequests
- * @param decreaseRequests
+ * @param updateRequests
* @return the {@link Allocation} for the application
*/
@Public
@@ -146,8 +144,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
Allocation allocate(ApplicationAttemptId appAttemptId,
List<ResourceRequest> ask, List<ContainerId> release,
List<String> blacklistAdditions, List<String> blacklistRemovals,
- List<UpdateContainerRequest> increaseRequests,
- List<UpdateContainerRequest> decreaseRequests);
+ ContainerUpdates updateRequests);
/**
* Get node resource usage report.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index b862850..55ffe25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -90,6 +90,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeDecreaseContai
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
@@ -921,22 +924,27 @@ public class CapacityScheduler extends
public Allocation allocate(ApplicationAttemptId applicationAttemptId,
List<ResourceRequest> ask, List<ContainerId> release,
List<String> blacklistAdditions, List<String> blacklistRemovals,
- List<UpdateContainerRequest> increaseRequests,
- List<UpdateContainerRequest> decreaseRequests) {
+ ContainerUpdates updateRequests) {
FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId);
if (application == null) {
return EMPTY_ALLOCATION;
}
+ // Handle promotions and demotions
+ handleExecutionTypeUpdates(
+ application, updateRequests.getPromotionRequests(),
+ updateRequests.getDemotionRequests());
+
// Release containers
releaseContainers(release, application);
// update increase requests
- LeafQueue updateDemandForQueue = updateIncreaseRequests(increaseRequests,
+ LeafQueue updateDemandForQueue =
+ updateIncreaseRequests(updateRequests.getIncreaseRequests(),
application);
// Decrease containers
- decreaseContainers(decreaseRequests, application);
+ decreaseContainers(updateRequests.getDecreaseRequests(), application);
// Sanity check for new allocation requests
normalizeRequests(ask);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index 1eb48bb..eeb0815 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -746,7 +746,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
// When reserving container
RMContainer updatedContainer = reservedContainer;
if (updatedContainer == null) {
- updatedContainer = new RMContainerImpl(container,
+ updatedContainer = new RMContainerImpl(container, schedulerKey,
application.getApplicationAttemptId(), node.getNodeID(),
application.getAppSchedulingInfo().getUser(), rmContext);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index b14bc20..809446f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -222,7 +222,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
}
// Create RMContainer
- RMContainer rmContainer = new RMContainerImpl(container,
+ RMContainer rmContainer = new RMContainerImpl(container, schedulerKey,
this.getApplicationAttemptId(), node.getNodeID(),
appSchedulingInfo.getUser(), this.rmContext,
request.getNodeLabelExpression());
@@ -554,12 +554,14 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
// Update this application for the allocated container
if (!allocation.isIncreasedAllocation()) {
// Allocate a new container
- newlyAllocatedContainers.add(rmContainer);
+ addToNewlyAllocatedContainers(
+ schedulerContainer.getSchedulerNode(), rmContainer);
liveContainers.put(containerId, rmContainer);
// Deduct pending resource requests
List<ResourceRequest> requests = appSchedulingInfo.allocate(
- allocation.getAllocationLocalityType(), schedulerContainer.getSchedulerNode(),
+ allocation.getAllocationLocalityType(),
+ schedulerContainer.getSchedulerNode(),
schedulerContainer.getSchedulerRequestKey(),
schedulerContainer.getRmContainer().getContainer());
((RMContainerImpl) rmContainer).setResourceRequests(requests);
@@ -751,12 +753,15 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
List<Container> newlyAllocatedContainers = pullNewlyAllocatedContainers();
List<Container> newlyIncreasedContainers = pullNewlyIncreasedContainers();
List<Container> newlyDecreasedContainers = pullNewlyDecreasedContainers();
+ List<Container> newlyPromotedContainers = pullNewlyPromotedContainers();
+ List<Container> newlyDemotedContainers = pullNewlyDemotedContainers();
List<NMToken> updatedNMTokens = pullUpdatedNMTokens();
Resource headroom = getHeadroom();
setApplicationHeadroomForMetrics(headroom);
return new Allocation(newlyAllocatedContainers, headroom, null,
currentContPreemption, Collections.singletonList(rr), updatedNMTokens,
- newlyIncreasedContainers, newlyDecreasedContainers);
+ newlyIncreasedContainers, newlyDecreasedContainers,
+ newlyPromotedContainers, newlyDemotedContainers);
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index a9591a5..0daa8a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -448,13 +448,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
// Create RMContainer
- rmContainer = new RMContainerImpl(container,
+ rmContainer = new RMContainerImpl(container, schedulerKey,
getApplicationAttemptId(), node.getNodeID(),
appSchedulingInfo.getUser(), rmContext);
((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
// Add it to allContainers list.
- newlyAllocatedContainers.add(rmContainer);
+ addToNewlyAllocatedContainers(node, rmContainer);
liveContainers.put(container.getId(), rmContainer);
// Update consumption and track allocations
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index e3af150..78dfe61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -18,18 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
@@ -51,7 +41,6 @@ import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -80,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
@@ -102,8 +92,17 @@ import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
/**
* A scheduler that schedules resources between a set of queues. The scheduler
@@ -812,8 +811,7 @@ public class FairScheduler extends
public Allocation allocate(ApplicationAttemptId appAttemptId,
List<ResourceRequest> ask, List<ContainerId> release,
List<String> blacklistAdditions, List<String> blacklistRemovals,
- List<UpdateContainerRequest> increaseRequests,
- List<UpdateContainerRequest> decreaseRequests) {
+ ContainerUpdates updateRequests) {
// Make sure this application exists
FSAppAttempt application = getSchedulerApp(appAttemptId);
@@ -823,6 +821,11 @@ public class FairScheduler extends
return EMPTY_ALLOCATION;
}
+ // Handle promotions and demotions
+ handleExecutionTypeUpdates(
+ application, updateRequests.getPromotionRequests(),
+ updateRequests.getDemotionRequests());
+
// Sanity check
normalizeRequests(ask);
@@ -879,7 +882,9 @@ public class FairScheduler extends
application.setApplicationHeadroomForMetrics(headroom);
return new Allocation(newlyAllocatedContainers, headroom,
preemptionContainerIds, null, null,
- application.pullUpdatedNMTokens());
+ application.pullUpdatedNMTokens(), null, null,
+ application.pullNewlyPromotedContainers(),
+ application.pullNewlyDemotedContainers());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
index e60f70e..fa61710 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
@@ -68,7 +68,7 @@ public class FifoAppAttempt extends FiCaSchedulerApp {
// Create RMContainer
RMContainer rmContainer = new RMContainerImpl(container,
- this.getApplicationAttemptId(), node.getNodeID(),
+ schedulerKey, this.getApplicationAttemptId(), node.getNodeID(),
appSchedulingInfo.getUser(), this.rmContext,
request.getNodeLabelExpression());
((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
@@ -76,7 +76,7 @@ public class FifoAppAttempt extends FiCaSchedulerApp {
updateAMContainerDiagnostics(AMState.ASSIGNED, null);
// Add it to allContainers list.
- newlyAllocatedContainers.add(rmContainer);
+ addToNewlyAllocatedContainers(node, rmContainer);
ContainerId containerId = container.getId();
liveContainers.put(containerId, rmContainer);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 643199c..f4ab9c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -71,6 +70,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
@@ -326,8 +326,7 @@ public class FifoScheduler extends
public Allocation allocate(ApplicationAttemptId applicationAttemptId,
List<ResourceRequest> ask, List<ContainerId> release,
List<String> blacklistAdditions, List<String> blacklistRemovals,
- List<UpdateContainerRequest> increaseRequests,
- List<UpdateContainerRequest> decreaseRequests) {
+ ContainerUpdates updateRequests) {
FifoAppAttempt application = getApplicationAttempt(applicationAttemptId);
if (application == null) {
LOG.error("Calling allocate on removed " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
index 9dbf024..157518e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
@@ -157,7 +157,8 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
return resourceRequestMap.get(resourceName);
}
- private void decrementOutstanding(ResourceRequest offSwitchRequest) {
+ private void decrementOutstanding(SchedulerRequestKey schedulerRequestKey,
+ ResourceRequest offSwitchRequest) {
int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;
// Do not remove ANY
@@ -166,8 +167,6 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
// Do we have any outstanding requests?
// If there is nothing, we need to deactivate this application
if (numOffSwitchContainers == 0) {
- SchedulerRequestKey schedulerRequestKey = SchedulerRequestKey.create(
- offSwitchRequest);
appSchedulingInfo.decrementSchedulerKeyReference(schedulerRequestKey);
appSchedulingInfo.checkForDeactivation();
}
@@ -177,11 +176,15 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
offSwitchRequest.getCapability());
}
- private ResourceRequest cloneResourceRequest(ResourceRequest request) {
- ResourceRequest newRequest =
- ResourceRequest.newInstance(request.getPriority(),
- request.getResourceName(), request.getCapability(), 1,
- request.getRelaxLocality(), request.getNodeLabelExpression());
+ public ResourceRequest cloneResourceRequest(ResourceRequest request) {
+ ResourceRequest newRequest = ResourceRequest.newBuilder()
+ .priority(request.getPriority())
+ .allocationRequestId(request.getAllocationRequestId())
+ .resourceName(request.getResourceName())
+ .capability(request.getCapability())
+ .numContainers(1)
+ .relaxLocality(request.getRelaxLocality())
+ .nodeLabelExpression(request.getNodeLabelExpression()).build();
return newRequest;
}
@@ -189,15 +192,15 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
* The {@link ResourceScheduler} is allocating data-local resources to the
* application.
*/
- private void allocateRackLocal(SchedulerNode node,
- ResourceRequest rackLocalRequest,
+ private void allocateRackLocal(SchedulerRequestKey schedulerKey,
+ SchedulerNode node, ResourceRequest rackLocalRequest,
List<ResourceRequest> resourceRequests) {
// Update future requirements
decResourceRequest(node.getRackName(), rackLocalRequest);
ResourceRequest offRackRequest = resourceRequestMap.get(
ResourceRequest.ANY);
- decrementOutstanding(offRackRequest);
+ decrementOutstanding(schedulerKey, offRackRequest);
// Update cloned RackLocal and OffRack requests for recovery
resourceRequests.add(cloneResourceRequest(rackLocalRequest));
@@ -208,10 +211,11 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
* The {@link ResourceScheduler} is allocating data-local resources to the
* application.
*/
- private void allocateOffSwitch(ResourceRequest offSwitchRequest,
+ private void allocateOffSwitch(SchedulerRequestKey schedulerKey,
+ ResourceRequest offSwitchRequest,
List<ResourceRequest> resourceRequests) {
// Update future requirements
- decrementOutstanding(offSwitchRequest);
+ decrementOutstanding(schedulerKey, offSwitchRequest);
// Update cloned OffRack requests for recovery
resourceRequests.add(cloneResourceRequest(offSwitchRequest));
}
@@ -221,8 +225,8 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
* The {@link ResourceScheduler} is allocating data-local resources to the
* application.
*/
- private void allocateNodeLocal(SchedulerNode node,
- ResourceRequest nodeLocalRequest,
+ private void allocateNodeLocal(SchedulerRequestKey schedulerKey,
+ SchedulerNode node, ResourceRequest nodeLocalRequest,
List<ResourceRequest> resourceRequests) {
// Update future requirements
decResourceRequest(node.getNodeName(), nodeLocalRequest);
@@ -233,7 +237,7 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
ResourceRequest offRackRequest = resourceRequestMap.get(
ResourceRequest.ANY);
- decrementOutstanding(offRackRequest);
+ decrementOutstanding(schedulerKey, offRackRequest);
// Update cloned NodeLocal, RackLocal and OffRack requests for recovery
resourceRequests.add(cloneResourceRequest(nodeLocalRequest));
@@ -278,8 +282,8 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
}
@Override
- public List<ResourceRequest> allocate(NodeType type, SchedulerNode node,
- ResourceRequest request) {
+ public List<ResourceRequest> allocate(SchedulerRequestKey schedulerKey,
+ NodeType type, SchedulerNode node, ResourceRequest request) {
try {
writeLock.lock();
@@ -296,11 +300,11 @@ public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
}
if (type == NodeType.NODE_LOCAL) {
- allocateNodeLocal(node, request, resourceRequests);
+ allocateNodeLocal(schedulerKey, node, request, resourceRequests);
} else if (type == NodeType.RACK_LOCAL) {
- allocateRackLocal(node, request, resourceRequests);
+ allocateRackLocal(schedulerKey, node, request, resourceRequests);
} else{
- allocateOffSwitch(request, resourceRequests);
+ allocateOffSwitch(schedulerKey, request, resourceRequests);
}
return resourceRequests;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
index cdb7c04..3cf5fa2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
@@ -78,13 +78,14 @@ public interface SchedulingPlacementSet<N extends SchedulerNode> {
/**
* Notify container allocated.
+ * @param schedulerKey SchedulerRequestKey for this ResourceRequest
* @param type Type of the allocation
* @param node Which node this container allocated on
* @param request Which resource request to allocate
* @return list of ResourceRequests deducted
*/
- List<ResourceRequest> allocate(NodeType type, SchedulerNode node,
- ResourceRequest request);
+ List<ResourceRequest> allocate(SchedulerRequestKey schedulerKey,
+ NodeType type, SchedulerNode node, ResourceRequest request);
/**
* We can still have pending requirement for a given NodeType and node
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
index 3288d39..da1bc2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -331,7 +332,8 @@ public class Application {
// Get resources from the ResourceManager
Allocation allocation = resourceManager.getResourceScheduler().allocate(
applicationAttemptId, new ArrayList<ResourceRequest>(ask),
- new ArrayList<ContainerId>(), null, null, null, null);
+ new ArrayList<ContainerId>(), null, null,
+ new ContainerUpdates());
if (LOG.isInfoEnabled()) {
LOG.info("-=======" + applicationAttemptId + System.lineSeparator() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
index 593de08..fbeca7b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
@@ -251,6 +251,13 @@ public class MockAM {
return allocate(req);
}
+ public AllocateResponse sendContainerUpdateRequest(
+ List<UpdateContainerRequest> updateRequests) throws Exception {
+ final AllocateRequest req = AllocateRequest.newInstance(0, 0F, null, null,
+ null, updateRequests);
+ return allocate(req);
+ }
+
public AllocateResponse allocate(AllocateRequest allocateRequest)
throws Exception {
UserGroupInformation ugi =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index 32cdb1b..2d76127 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -195,6 +195,12 @@ public class MockNM {
isHealthy, resId);
}
+ public NodeHeartbeatResponse nodeHeartbeat(
+ List<ContainerStatus> updatedStats, boolean isHealthy) throws Exception {
+ return nodeHeartbeat(updatedStats, Collections.<Container>emptyList(),
+ isHealthy, ++responseId);
+ }
+
public NodeHeartbeatResponse nodeHeartbeat(List<ContainerStatus> updatedStats,
List<Container> increasedConts, boolean isHealthy, int resId)
throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index cb57f39..c135384 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -142,6 +142,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
+
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
@@ -1072,7 +1074,8 @@ public class TestClientRMService {
Container container = Container.newInstance(
ContainerId.newContainerId(attemptId, 1), null, "", null, null, null);
RMContainerImpl containerimpl = spy(new RMContainerImpl(container,
- attemptId, null, "", rmContext));
+ SchedulerRequestKey.extractFrom(container), attemptId, null, "",
+ rmContext));
Map<ApplicationAttemptId, RMAppAttempt> attempts =
new HashMap<ApplicationAttemptId, RMAppAttempt>();
attempts.put(attemptId, rmAppAttemptImpl);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: HADOOP-13336 S3A to support
per-bucket configuration. Contributed by Steve Loughran
Posted by st...@apache.org.
HADOOP-13336 S3A to support per-bucket configuration. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e648b6e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e648b6e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e648b6e1
Branch: refs/heads/HADOOP-13345
Commit: e648b6e1382336af69434dfbf9161bced3caa244
Parents: 357eab9
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jan 11 17:13:05 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jan 11 17:25:15 2017 +0000
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 9 ++
.../org/apache/hadoop/fs/s3a/Constants.java | 14 ++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 16 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 97 +++++++++++-
.../src/site/markdown/tools/hadoop-aws/index.md | 153 ++++++++++++++++---
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 1 -
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 130 ++++++++++++++++
.../apache/hadoop/fs/s3a/S3ATestConstants.java | 12 --
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 63 ++++++--
.../scale/ITestS3AInputStreamPerformance.java | 10 +-
.../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 8 +-
11 files changed, 442 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 9355a3c..1941bec 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -955,6 +955,15 @@
</property>
<property>
+ <name>fs.s3a.security.credential.provider.path</name>
+ <value />
+ <description>
+ Optional comma separated list of credential providers, a list
+ which is prepended to that set in hadoop.security.credential.provider.path
+ </description>
+</property>
+
+<property>
<name>fs.s3a.connection.maximum</name>
<value>15</value>
<description>Controls the maximum number of simultaneous connections to S3.</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 65df0bf..45d974c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -48,6 +48,14 @@ public final class Constants {
public static final String AWS_CREDENTIALS_PROVIDER =
"fs.s3a.aws.credentials.provider";
+ /**
+ * Extra set of security credentials which will be prepended to that
+ * set in {@code "hadoop.security.credential.provider.path"}.
+ * This extra option allows for per-bucket overrides.
+ */
+ public static final String S3A_SECURITY_CREDENTIAL_PROVIDER_PATH =
+ "fs.s3a.security.credential.provider.path";
+
// session token for when using TemporaryAWSCredentialsProvider
public static final String SESSION_TOKEN = "fs.s3a.session.token";
@@ -226,6 +234,12 @@ public final class Constants {
public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size";
public static final String FS_S3A = "s3a";
+ /** Prefix for all S3A properties: {@value}. */
+ public static final String FS_S3A_PREFIX = "fs.s3a.";
+
+ /** Prefix for S3A bucket-specific properties: {@value}. */
+ public static final String FS_S3A_BUCKET_PREFIX = "fs.s3a.bucket.";
+
public static final int S3A_DEFAULT_PORT = -1;
public static final String USER_AGENT_PREFIX = "fs.s3a.user.agent.prefix";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index f1cde4e..8152bf8 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -29,7 +29,6 @@ import java.util.Date;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ExecutorService;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -153,21 +152,28 @@ public class S3AFileSystem extends FileSystem {
/** Called after a new FileSystem instance is constructed.
* @param name a uri whose authority section names the host, port, etc.
* for this FileSystem
- * @param conf the configuration
+ * @param originalConf the configuration to use for the FS. The
+ * bucket-specific options are patched over the base ones before any use is
+ * made of the config.
*/
- public void initialize(URI name, Configuration conf) throws IOException {
+ public void initialize(URI name, Configuration originalConf)
+ throws IOException {
+ uri = S3xLoginHelper.buildFSURI(name);
+ // get the host; this is guaranteed to be non-null, non-empty
+ bucket = name.getHost();
+ // clone the configuration into one with propagated bucket options
+ Configuration conf = propagateBucketOptions(originalConf, bucket);
+ patchSecurityCredentialProviders(conf);
super.initialize(name, conf);
setConf(conf);
try {
instrumentation = new S3AInstrumentation(name);
- uri = S3xLoginHelper.buildFSURI(name);
// Username is the current user at the time the FS was instantiated.
username = UserGroupInformation.getCurrentUser().getShortUserName();
workingDir = new Path("/user", username)
.makeQualified(this.uri, this.getWorkingDirectory());
- bucket = name.getHost();
Class<? extends S3ClientFactory> s3ClientFactoryClass = conf.getClass(
S3_CLIENT_FACTORY_IMPL, DEFAULT_S3_CLIENT_FACTORY_IMPL,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index aeb8403..c4ff638 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.security.ProviderUtils;
+
+import com.google.common.collect.Lists;
import org.slf4j.Logger;
import java.io.EOFException;
@@ -46,15 +48,13 @@ import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.URI;
import java.nio.file.AccessDeniedException;
+import java.util.Collection;
import java.util.Date;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
-import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
-import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
-import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_MIN_SIZE;
-import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.*;
/**
* Utility methods for S3A code.
@@ -74,6 +74,13 @@ public final class S3AUtils {
"is abstract and therefore cannot be created";
static final String ENDPOINT_KEY = "Endpoint";
+ /**
+ * Core property for provider path. Duplicated here for consistent
+ * code across Hadoop version: {@value}.
+ */
+ static final String CREDENTIAL_PROVIDER_PATH =
+ "hadoop.security.credential.provider.path";
+
private S3AUtils() {
}
@@ -636,4 +643,84 @@ public final class S3AUtils {
return null;
}
}
+
+ /**
+ * Propagates bucket-specific settings into generic S3A configuration keys.
+ * This is done by propagating the values of the form
+ * {@code fs.s3a.bucket.${bucket}.key} to
+ * {@code fs.s3a.key}, for all values of "key" other than a small set
+ * of unmodifiable values.
+ *
+ * The source of the updated property is set to the key name of the bucket
+ * property, to aid in diagnostics of where things came from.
+ *
+ * Returns a new configuration. Why the clone?
+ * You can use the same conf for different filesystems, and the original
+ * values are not updated.
+ *
+ * The {@code fs.s3a.impl} property cannot be set, nor can
+ * any with the prefix {@code fs.s3a.bucket}.
+ *
+ * This method does not propagate security provider path information from
+ * the S3A property into the Hadoop common provider: callers must call
+ * {@link #patchSecurityCredentialProviders(Configuration)} explicitly.
+ * @param source Source Configuration object.
+ * @param bucket bucket name. Must not be empty.
+ * @return a (potentially) patched clone of the original.
+ */
+ public static Configuration propagateBucketOptions(Configuration source,
+ String bucket) {
+
+ Preconditions.checkArgument(StringUtils.isNotEmpty(bucket), "bucket");
+ final String bucketPrefix = FS_S3A_BUCKET_PREFIX + bucket +'.';
+ LOG.debug("Propagating entries under {}", bucketPrefix);
+ final Configuration dest = new Configuration(source);
+ for (Map.Entry<String, String> entry : source) {
+ final String key = entry.getKey();
+ // get the (unexpanded) value.
+ final String value = entry.getValue();
+ if (!key.startsWith(bucketPrefix) || bucketPrefix.equals(key)) {
+ continue;
+ }
+ // there's a bucket prefix, so strip it
+ final String stripped = key.substring(bucketPrefix.length());
+ if (stripped.startsWith("bucket.") || "impl".equals(stripped)) {
+ //tell user off
+ LOG.debug("Ignoring bucket option {}", key);
+ } else {
+ // propagate the value, building a new origin field.
+ // to track overwrites, the generic key is overwritten even if
+ // already matches the new one.
+ final String generic = FS_S3A_PREFIX + stripped;
+ LOG.debug("Updating {}", generic);
+ dest.set(generic, value, key);
+ }
+ }
+ return dest;
+ }
+
+ /**
+ * Patch the security credential provider information in
+ * {@link #CREDENTIAL_PROVIDER_PATH}
+ * with the providers listed in
+ * {@link Constants#S3A_SECURITY_CREDENTIAL_PROVIDER_PATH}.
+ *
+ * This allows different buckets to use different credential files.
+ * @param conf configuration to patch
+ */
+ static void patchSecurityCredentialProviders(Configuration conf) {
+ Collection<String> customCredentials = conf.getStringCollection(
+ S3A_SECURITY_CREDENTIAL_PROVIDER_PATH);
+ Collection<String> hadoopCredentials = conf.getStringCollection(
+ CREDENTIAL_PROVIDER_PATH);
+ if (!customCredentials.isEmpty()) {
+ List<String> all = Lists.newArrayList(customCredentials);
+ all.addAll(hadoopCredentials);
+ String joined = StringUtils.join(all, ',');
+ LOG.debug("Setting {} to {}", CREDENTIAL_PROVIDER_PATH,
+ joined);
+ conf.set(CREDENTIAL_PROVIDER_PATH, joined,
+ "patch of " + S3A_SECURITY_CREDENTIAL_PROVIDER_PATH);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 54a4ba9..d804a59 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -255,6 +255,7 @@ properties, the Hadoop key management store and IAM roles.
* Test suites includes distcp and suites in downstream projects.
* Available since Hadoop 2.6; considered production ready in Hadoop 2.7.
* Actively maintained.
+* Supports per-bucket configuration.
S3A is now the recommended client for working with S3 objects. It is also the
one where patches for functionality and performance are very welcome.
@@ -609,6 +610,29 @@ in XML configuration files.
Because this property only supplies the path to the secrets file, the configuration
option itself is no longer a sensitive item.
+The property `hadoop.security.credential.provider.path` is global to all
+filesystems and secrets.
+There is another property, `fs.s3a.security.credential.provider.path`
+which only lists credential providers for S3A filesystems.
+The two properties are combined into one, with the list of providers in the
+`fs.s3a.` property taking precedence
+over that of the `hadoop.security` list (i.e. they are prepended to the common list).
+
+```xml
+<property>
+ <name>fs.s3a.security.credential.provider.path</name>
+ <value />
+ <description>
+ Optional comma separated list of credential providers, a list
+ which is prepended to that set in hadoop.security.credential.provider.path
+ </description>
+</property>
+```
+
+Supporting a separate list in an `fs.s3a.` prefix permits per-bucket configuration
+of credential files.
+
+
###### Using the credentials
Once the provider is set in the Hadoop configuration, hadoop commands
@@ -631,7 +655,7 @@ hadoop distcp \
hdfs://nn1.example.com:9001/user/backup/007020615 s3a://glacier1/
hadoop fs \
- -D hadoop.security.credential.provider.path=jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks \
+ -D fs.s3a.security.credential.provider.path=jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks \
-ls s3a://glacier1/
```
@@ -869,6 +893,78 @@ from placing its declaration on the command line.
any call to setReadahead() is made to an open stream.</description>
</property>
+### Configurations different S3 buckets
+
+Different S3 buckets can be accessed with different S3A client configurations.
+This allows for different endpoints, data read and write strategies, as well
+as login details.
+
+1. All `fs.s3a` options other than a small set of unmodifiable values
+ (currently `fs.s3a.impl`) can be set on a per bucket basis.
+1. The bucket specific option is set by replacing the `fs.s3a.` prefix on an option
+with `fs.s3a.bucket.BUCKETNAME.`, where `BUCKETNAME` is the name of the bucket.
+1. When connecting to a bucket, all options explicitly set will override
+the base `fs.s3a.` values.
+
+As an example, a configuration could have a base configuration to use the IAM
+role information available when deployed in Amazon EC2.
+
+```xml
+<property>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <value>org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider</value>
+</property>
+```
+
+This will be the default authentication mechanism for S3A buckets.
+
+A bucket `s3a://nightly/` used for nightly data uses a session key:
+
+```xml
+<property>
+ <name>fs.s3a.bucket.nightly.access.key</name>
+ <value>AKAACCESSKEY-2</value>
+</property>
+
+<property>
+ <name>fs.s3a.bucket.nightly.secret.key</name>
+ <value>SESSIONSECRETKEY</value>
+</property>
+
+<property>
+ <name>fs.s3a.bucket.nightly.session.token</name>
+ <value>Short-lived-session-token</value>
+</property>
+
+<property>
+ <name>fs.s3a.bucket.nightly.aws.credentials.provider</name>
+ <value>org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider</value>
+</property>
+```
+
+Finally, the public `s3a://landsat-pds/` bucket is accessed anonymously:
+
+```xml
+<property>
+ <name>fs.s3a.bucket.landsat-pds.aws.credentials.provider</name>
+ <value>org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider</value>
+</property>
+```
+
+**Customizing S3A secrets held in credential files**
+
+Although most properties are automatically propagated from their
+`fs.s3a.bucket.`-prefixed custom entry to that of the base `fs.s3a.` option
+supporting secrets kept in Hadoop credential files is slightly more complex.
+This is because the property values are kept in these files, and cannot be
+dynamically patched.
+
+Instead, callers need to create different configuration files for each
+bucket, setting the base secrets (`fs.s3a.bucket.nightly.access.key`, etc),
+then declare the path to the appropriate credential file in
+a bucket-specific version of the property `fs.s3a.security.credential.provider.path`.
+
+
### Working with buckets in different regions
S3 Buckets are hosted in different regions, the default being US-East.
@@ -924,6 +1020,16 @@ If the wrong endpoint is used, the request may fail. This may be reported as a 3
or as a 400 Bad Request.
+If you are trying to mix endpoints for different buckets, use a per-bucket endpoint
+declaration. For example:
+
+```xml
+<property>
+ <name>fs.s3a.bucket.landsat-pds.endpoint</name>
+ <value>s3.amazonaws.com</value>
+ <description>The endpoint for s3a://landsat-pds URLs</description>
+</property>
+```
### <a name="s3a_fast_upload"></a>Stabilizing: S3A Fast Upload
@@ -1603,15 +1709,15 @@ org.apache.hadoop.fs.s3a.AWSS3IOException: Received permanent redirect response
1. If not using "V4" authentication (see above), the original S3 endpoint
can be used:
-```
- <property>
- <name>fs.s3a.endpoint</name>
- <value>s3.amazonaws.com</value>
- </property>
+```xml
+<property>
+ <name>fs.s3a.endpoint</name>
+ <value>s3.amazonaws.com</value>
+</property>
```
-Using the explicit endpoint for the region is recommended for speed and the
-ability to use the V4 signing API.
+Using the explicit endpoint for the region is recommended for speed and
+to use the V4 signing API.
### "Timeout waiting for connection from pool" when writing to S3A
@@ -2163,32 +2269,33 @@ is hosted in Amazon's US-east datacenter.
1. If the property is set to a different path, then that data must be readable
and "sufficiently" large.
-To test on different S3 endpoints, or alternate infrastructures supporting
-the same APIs, the option `fs.s3a.scale.test.csvfile` must either be
-set to " ", or an object of at least 10MB is uploaded to the object store, and
-the `fs.s3a.scale.test.csvfile` option set to its path.
+(the reason the space or newline is needed is to add "an empty entry"; an empty
+`<value/>` would be considered undefined and pick up the default)
+
+Of using a test file in an S3 region requiring a different endpoint value
+set in `fs.s3a.endpoint`, a bucket-specific endpoint must be defined.
+For the default test dataset, hosted in the `landsat-pds` bucket, this is:
```xml
<property>
- <name>fs.s3a.scale.test.csvfile</name>
- <value> </value>
+ <name>fs.s3a.bucket.landsat-pds.endpoint</name>
+ <value>s3.amazonaws.com</value>
+ <description>The endpoint for s3a://landsat-pds URLs</description>
</property>
```
-(the reason the space or newline is needed is to add "an empty entry"; an empty
-`<value/>` would be considered undefined and pick up the default)
-
-*Note:* if using a test file in an S3 region requiring a different endpoint value
-set in `fs.s3a.endpoint`, define it in `fs.s3a.scale.test.csvfile.endpoint`.
-If the default CSV file is used, the tests will automatically use the us-east
-endpoint:
+To test on alternate infrastructures supporting
+the same APIs, the option `fs.s3a.scale.test.csvfile` must either be
+set to " ", or an object of at least 10MB is uploaded to the object store, and
+the `fs.s3a.scale.test.csvfile` option set to its path.
```xml
<property>
- <name>fs.s3a.scale.test.csvfile.endpoint</name>
- <value>s3.amazonaws.com</value>
+ <name>fs.s3a.scale.test.csvfile</name>
+ <value> </value>
</property>
```
+
### Viewing Integration Test Reports
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
index d7f3411..99454f4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
@@ -129,7 +129,6 @@ public class ITestS3AAWSCredentialsProvider {
AnonymousAWSCredentialsProvider.class.getName());
Path testFile = new Path(
conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
- S3ATestUtils.useCSVDataEndpoint(conf);
FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
assertNotNull(fs);
assertTrue(fs instanceof S3AFileSystem);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index f7d77f4..dd75cb6 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -35,6 +35,7 @@ import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.TEST_FS_S3A_NAME;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
@@ -45,6 +46,7 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -54,6 +56,10 @@ import org.apache.hadoop.util.VersionInfo;
import org.apache.http.HttpStatus;
import org.junit.rules.TemporaryFolder;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+
/**
* S3A tests for configuration.
*/
@@ -505,4 +511,128 @@ public class ITestS3AConfiguration {
fieldType.isAssignableFrom(obj.getClass()));
return fieldType.cast(obj);
}
+
+ @Test
+ public void testBucketConfigurationPropagation() throws Throwable {
+ Configuration config = new Configuration(false);
+ setBucketOption(config, "b", "base", "1024");
+ String basekey = "fs.s3a.base";
+ assertOptionEquals(config, basekey, null);
+ String bucketKey = "fs.s3a.bucket.b.base";
+ assertOptionEquals(config, bucketKey, "1024");
+ Configuration updated = propagateBucketOptions(config, "b");
+ assertOptionEquals(updated, basekey, "1024");
+ // original conf is not updated
+ assertOptionEquals(config, basekey, null);
+
+ String[] sources = updated.getPropertySources(basekey);
+ assertEquals(1, sources.length);
+ String sourceInfo = sources[0];
+ assertTrue("Wrong source " + sourceInfo, sourceInfo.contains(bucketKey));
+ }
+
+ @Test
+ public void testBucketConfigurationPropagationResolution() throws Throwable {
+ Configuration config = new Configuration(false);
+ String basekey = "fs.s3a.base";
+ String baseref = "fs.s3a.baseref";
+ String baseref2 = "fs.s3a.baseref2";
+ config.set(basekey, "orig");
+ config.set(baseref2, "${fs.s3a.base}");
+ setBucketOption(config, "b", basekey, "1024");
+ setBucketOption(config, "b", baseref, "${fs.s3a.base}");
+ Configuration updated = propagateBucketOptions(config, "b");
+ assertOptionEquals(updated, basekey, "1024");
+ assertOptionEquals(updated, baseref, "1024");
+ assertOptionEquals(updated, baseref2, "1024");
+ }
+
+ @Test
+ public void testMultipleBucketConfigurations() throws Throwable {
+ Configuration config = new Configuration(false);
+ setBucketOption(config, "b", USER_AGENT_PREFIX, "UA-b");
+ setBucketOption(config, "c", USER_AGENT_PREFIX, "UA-c");
+ config.set(USER_AGENT_PREFIX, "UA-orig");
+ Configuration updated = propagateBucketOptions(config, "c");
+ assertOptionEquals(updated, USER_AGENT_PREFIX, "UA-c");
+ }
+
+ @Test
+ public void testBucketConfigurationSkipsUnmodifiable() throws Throwable {
+ Configuration config = new Configuration(false);
+ String impl = "fs.s3a.impl";
+ config.set(impl, "orig");
+ setBucketOption(config, "b", impl, "b");
+ String metastoreImpl = "fs.s3a.metadatastore.impl";
+ String ddb = "org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore";
+ setBucketOption(config, "b", metastoreImpl, ddb);
+ setBucketOption(config, "b", "impl2", "b2");
+ setBucketOption(config, "b", "bucket.b.loop", "b3");
+ assertOptionEquals(config, "fs.s3a.bucket.b.impl", "b");
+
+ Configuration updated = propagateBucketOptions(config, "b");
+ assertOptionEquals(updated, impl, "orig");
+ assertOptionEquals(updated, "fs.s3a.impl2", "b2");
+ assertOptionEquals(updated, metastoreImpl, ddb);
+ assertOptionEquals(updated, "fs.s3a.bucket.b.loop", null);
+ }
+
+ @Test
+ public void testConfOptionPropagationToFS() throws Exception {
+ Configuration config = new Configuration();
+ String testFSName = config.getTrimmed(TEST_FS_S3A_NAME, "");
+ String bucket = new URI(testFSName).getHost();
+ setBucketOption(config, bucket, "propagation", "propagated");
+ fs = S3ATestUtils.createTestFileSystem(config);
+ Configuration updated = fs.getConf();
+ assertOptionEquals(updated, "fs.s3a.propagation", "propagated");
+ }
+
+ @Test
+ public void testSecurityCredentialPropagationNoOverride() throws Exception {
+ Configuration config = new Configuration();
+ config.set(CREDENTIAL_PROVIDER_PATH, "base");
+ patchSecurityCredentialProviders(config);
+ assertOptionEquals(config, CREDENTIAL_PROVIDER_PATH,
+ "base");
+ }
+
+ @Test
+ public void testSecurityCredentialPropagationOverrideNoBase()
+ throws Exception {
+ Configuration config = new Configuration();
+ config.unset(CREDENTIAL_PROVIDER_PATH);
+ config.set(S3A_SECURITY_CREDENTIAL_PROVIDER_PATH, "override");
+ patchSecurityCredentialProviders(config);
+ assertOptionEquals(config, CREDENTIAL_PROVIDER_PATH,
+ "override");
+ }
+
+ @Test
+ public void testSecurityCredentialPropagationOverride() throws Exception {
+ Configuration config = new Configuration();
+ config.set(CREDENTIAL_PROVIDER_PATH, "base");
+ config.set(S3A_SECURITY_CREDENTIAL_PROVIDER_PATH, "override");
+ patchSecurityCredentialProviders(config);
+ assertOptionEquals(config, CREDENTIAL_PROVIDER_PATH,
+ "override,base");
+ Collection<String> all = config.getStringCollection(
+ CREDENTIAL_PROVIDER_PATH);
+ assertTrue(all.contains("override"));
+ assertTrue(all.contains("base"));
+ }
+
+ @Test
+ public void testSecurityCredentialPropagationEndToEnd() throws Exception {
+ Configuration config = new Configuration();
+ config.set(CREDENTIAL_PROVIDER_PATH, "base");
+ setBucketOption(config, "b", S3A_SECURITY_CREDENTIAL_PROVIDER_PATH,
+ "override");
+ Configuration updated = propagateBucketOptions(config, "b");
+
+ patchSecurityCredentialProviders(updated);
+ assertOptionEquals(updated, CREDENTIAL_PROVIDER_PATH,
+ "override,base");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
index 8c22f47..acbe610 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
@@ -86,18 +86,6 @@ public interface S3ATestConstants {
String DEFAULT_CSVTEST_FILE = "s3a://landsat-pds/scene_list.gz";
/**
- * Endpoint for the S3 CSV/scale tests. This defaults to
- * being us-east.
- */
- String KEY_CSVTEST_ENDPOINT = S3A_SCALE_TEST + "csvfile.endpoint";
-
- /**
- * Endpoint for the S3 CSV/scale tests. This defaults to
- * being us-east.
- */
- String DEFAULT_CSVTEST_ENDPOINT = "s3.amazonaws.com";
-
- /**
* Name of the property to define the timeout for scale tests: {@value}.
* Measured in seconds.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 462914c..567bacb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -22,10 +22,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.internal.AssumptionViolatedException;
import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
@@ -34,11 +35,14 @@ import java.util.List;
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.junit.Assert.*;
/**
* Utilities for the S3A tests.
*/
public final class S3ATestUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ S3ATestUtils.class);
/**
* Value to set a system property to (in maven) to declare that
@@ -137,20 +141,6 @@ public final class S3ATestUtils {
}
/**
- * patch the endpoint option so that irrespective of where other tests
- * are working, the IO performance tests can work with the landsat
- * images.
- * @param conf configuration to patch
- */
- public static void useCSVDataEndpoint(Configuration conf) {
- String endpoint = conf.getTrimmed(S3AScaleTestBase.KEY_CSVTEST_ENDPOINT,
- S3AScaleTestBase.DEFAULT_CSVTEST_ENDPOINT);
- if (!endpoint.isEmpty()) {
- conf.set(ENDPOINT, endpoint);
- }
- }
-
- /**
* Get a long test property.
* <ol>
* <li>Look up configuration value (which can pick up core-default.xml),
@@ -511,4 +501,47 @@ public final class S3ATestUtils {
*/
private S3ATestUtils() {
}
+
+ /**
+ * Set a bucket specific property to a particular value.
+ * If the generic key passed in has an {@code fs.s3a. prefix},
+ * that's stripped off, so that when the the bucket properties are propagated
+ * down to the generic values, that value gets copied down.
+ * @param conf configuration to set
+ * @param bucket bucket name
+ * @param genericKey key; can start with "fs.s3a."
+ * @param value value to set
+ */
+ public static void setBucketOption(Configuration conf, String bucket,
+ String genericKey, String value) {
+ final String baseKey = genericKey.startsWith(FS_S3A_PREFIX) ?
+ genericKey.substring(FS_S3A_PREFIX.length())
+ : genericKey;
+ conf.set(FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey, value);
+ }
+
+ /**
+ * Assert that a configuration option matches the expected value.
+ * @param conf configuration
+ * @param key option key
+ * @param expected expected value
+ */
+ public static void assertOptionEquals(Configuration conf,
+ String key,
+ String expected) {
+ assertEquals("Value of " + key, expected, conf.get(key));
+ }
+
+ /**
+ * Assume that a condition is met. If not: log at WARN and
+ * then throw an {@link AssumptionViolatedException}.
+ * @param message
+ * @param condition
+ */
+ public static void assume(String message, boolean condition) {
+ if (!condition) {
+ LOG.warn(message);
+ }
+ Assume.assumeTrue(message, condition);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
index e36d086..236ffcd 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AInputPolicy;
import org.apache.hadoop.fs.s3a.S3AInputStream;
import org.apache.hadoop.fs.s3a.S3AInstrumentation;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -36,7 +35,6 @@ import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.util.LineReader;
import org.junit.After;
import org.junit.Assert;
-import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@@ -47,6 +45,7 @@ import java.io.IOException;
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
/**
* Look at the performance of S3a operations.
@@ -79,10 +78,11 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
if (testFile.isEmpty()) {
assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;
+ LOG.warn(assumptionMessage);
testDataAvailable = false;
} else {
- S3ATestUtils.useCSVDataEndpoint(conf);
testData = new Path(testFile);
+ LOG.info("Using {} as input stream source", testData);
Path path = this.testData;
bindS3aFS(path);
try {
@@ -113,7 +113,7 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
* Declare that the test requires the CSV test dataset.
*/
private void requireCSVTestData() {
- Assume.assumeTrue(assumptionMessage, testDataAvailable);
+ assume(assumptionMessage, testDataAvailable);
}
/**
@@ -146,7 +146,7 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
/**
* Open a test file with the read buffer specified in the setting
- * {@link #KEY_READ_BUFFER_SIZE}.
+ * {@link org.apache.hadoop.fs.s3a.S3ATestConstants#KEY_READ_BUFFER_SIZE}.
*
* @param path path to open
* @param inputPolicy input policy to use
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648b6e1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
index 9da621f..0f844b1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.s3a.S3ATestConstants;
import org.apache.hadoop.fs.s3a.Statistic;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
-import org.junit.Assume;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,14 +90,13 @@ public class S3AScaleTestBase extends AbstractS3ATestBase {
super.setup();
testPath = path("/tests3ascale");
LOG.debug("Scale test operation count = {}", getOperationCount());
- // multipart purges are disabled on the scale tests
- // check for the test being enabled
enabled = getTestPropertyBool(
getConf(),
KEY_SCALE_TESTS_ENABLED,
DEFAULT_SCALE_TESTS_ENABLED);
- Assume.assumeTrue("Scale test disabled: to enable set property " +
- KEY_SCALE_TESTS_ENABLED, isEnabled());
+ assume("Scale test disabled: to enable set property " +
+ KEY_SCALE_TESTS_ENABLED,
+ isEnabled());
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-6073. Misuse of format
specifier in Preconditions.checkArgument (Contributed by Yuanbo Liu via
Daniel Templeton)
Posted by st...@apache.org.
YARN-6073. Misuse of format specifier in Preconditions.checkArgument (Contributed by Yuanbo Liu via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6332a318
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6332a318
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6332a318
Branch: refs/heads/HADOOP-13345
Commit: 6332a318bc1e2e9d73d7159eab26347bb3f1f9b3
Parents: 945db55
Author: Daniel Templeton <te...@apache.org>
Authored: Mon Jan 9 22:12:58 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Mon Jan 9 22:12:58 2017 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6332a318/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 2508c3e..4826013 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -685,7 +685,7 @@ public class RMAdminCLI extends HAAdmin {
}
int nLabels = map.get(nodeId).size();
- Preconditions.checkArgument(nLabels <= 1, "%d labels specified on host=%s"
+ Preconditions.checkArgument(nLabels <= 1, "%s labels specified on host=%s"
+ ", please note that we do not support specifying multiple"
+ " labels on a single host for now.", nLabels, nodeIdStr);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-6066. Opportunistic
containers Minor fixes : API annotations, parameter name changes,
checkstyles. (asuresh)
Posted by st...@apache.org.
YARN-6066. Opportunistic containers Minor fixes : API annotations, parameter name changes, checkstyles. (asuresh)
(cherry picked from commit 4985217de453a04ddffd7b52644bdc8d153f753c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85826f6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85826f6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85826f6c
Branch: refs/heads/HADOOP-13345
Commit: 85826f6ca5a6d06b711a6805f7a1a6788852db05
Parents: 679478d
Author: Arun Suresh <as...@apache.org>
Authored: Fri Dec 30 08:46:10 2016 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Sun Jan 8 09:04:20 2017 -0800
----------------------------------------------------------------------
.../v2/app/rm/RMContainerAllocator.java | 20 ++++++++++----------
.../apache/hadoop/mapreduce/MRJobConfig.java | 6 +++---
.../hadoop/mapred/TestMROpportunisticMaps.java | 2 +-
.../hadoop/yarn/api/records/ContainerState.java | 2 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 9 +++++++++
.../site/markdown/OpportunisticContainers.md | 4 ++--
6 files changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index 2b63b9e..31bc380 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -237,10 +237,10 @@ public class RMContainerAllocator extends RMContainerRequestor
// Init startTime to current time. If all goes well, it will be reset after
// first attempt to contact RM.
retrystartTime = System.currentTimeMillis();
- this.scheduledRequests.setNumOpportunisticMapsPer100(
- conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE,
- MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE));
- LOG.info(this.scheduledRequests.getNumOpportunisticMapsPer100() +
+ this.scheduledRequests.setNumOpportunisticMapsPercent(
+ conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENT,
+ MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT));
+ LOG.info(this.scheduledRequests.getNumOpportunisticMapsPercent() +
"% of the mappers will be scheduled using OPPORTUNISTIC containers");
}
@@ -1056,14 +1056,14 @@ public class RMContainerAllocator extends RMContainerRequestor
final Map<TaskAttemptId, ContainerRequest> maps =
new LinkedHashMap<TaskAttemptId, ContainerRequest>();
int mapsMod100 = 0;
- int numOpportunisticMapsPer100 = 0;
+ int numOpportunisticMapsPercent = 0;
- void setNumOpportunisticMapsPer100(int numMaps) {
- this.numOpportunisticMapsPer100 = numMaps;
+ void setNumOpportunisticMapsPercent(int numMaps) {
+ this.numOpportunisticMapsPercent = numMaps;
}
- int getNumOpportunisticMapsPer100() {
- return this.numOpportunisticMapsPer100;
+ int getNumOpportunisticMapsPercent() {
+ return this.numOpportunisticMapsPercent;
}
@VisibleForTesting
@@ -1110,7 +1110,7 @@ public class RMContainerAllocator extends RMContainerRequestor
maps.put(event.getAttemptID(), request);
addContainerReq(request);
} else {
- if (mapsMod100 < numOpportunisticMapsPer100) {
+ if (mapsMod100 < numOpportunisticMapsPercent) {
request =
new ContainerRequest(event, PRIORITY_OPPORTUNISTIC_MAP,
mapNodeLabelExpression);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 32bbe96..7add7df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -1005,9 +1005,9 @@ public interface MRJobConfig {
* requested by the AM will be opportunistic. If the total number of maps
* for the job is less than 'x', then ALL maps will be OPPORTUNISTIC
*/
- public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE =
- "mapreduce.job.num-opportunistic-maps-percentage";
- public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE = 0;
+ public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENT =
+ "mapreduce.job.num-opportunistic-maps-percent";
+ public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = 0;
/**
* A comma-separated list of properties whose value will be redacted.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
index 462ff04..eed731f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
@@ -145,7 +145,7 @@ public class TestMROpportunisticMaps {
job.setNumReduceTasks(numReducers);
// All OPPORTUNISTIC
- job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE, percent);
+ job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENT, percent);
job.setInt("mapreduce.map.maxattempts", 1);
job.setInt("mapreduce.reduce.maxattempts", 1);
job.setInt("mapred.test.num_lines", numLines);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
index 4efd8c1..696fe06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Stable;
/**
@@ -37,5 +38,6 @@ public enum ContainerState {
COMPLETE,
/** Scheduled (awaiting resources) at the NM. */
+ @InterfaceStability.Unstable
SCHEDULED
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d0ade22..7dd5ce3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -314,6 +314,7 @@ public class YarnConfiguration extends Configuration {
/** Setting that controls whether opportunistic container allocation
* is enabled or not. */
+ @Unstable
public static final String OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED =
RM_PREFIX + "opportunistic-container-allocation.enabled";
public static final boolean
@@ -321,12 +322,14 @@ public class YarnConfiguration extends Configuration {
/** Number of nodes to be used by the Opportunistic Container allocator for
* dispatching containers during container allocation. */
+ @Unstable
public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
RM_PREFIX + "opportunistic-container-allocation.nodes-used";
public static final int DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
10;
/** Frequency for computing least loaded NMs. */
+ @Unstable
public static final String NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS =
RM_PREFIX + "nm-container-queuing.sorting-nodes-interval-ms";
public static final long
@@ -334,6 +337,7 @@ public class YarnConfiguration extends Configuration {
/** Comparator for determining node load for scheduling of opportunistic
* containers. */
+ @Unstable
public static final String NM_CONTAINER_QUEUING_LOAD_COMPARATOR =
RM_PREFIX + "nm-container-queuing.load-comparator";
public static final String DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR =
@@ -341,6 +345,7 @@ public class YarnConfiguration extends Configuration {
/** Value of standard deviation used for calculation of queue limit
* thresholds. */
+ @Unstable
public static final String NM_CONTAINER_QUEUING_LIMIT_STDEV =
RM_PREFIX + "nm-container-queuing.queue-limit-stdev";
public static final float DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV =
@@ -349,6 +354,7 @@ public class YarnConfiguration extends Configuration {
/** Min length of container queue at NodeManager. This is a cluster-wide
* configuration that acts as the lower-bound of optimal queue length
* calculated by the NodeQueueLoadMonitor */
+ @Unstable
public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH =
RM_PREFIX + "nm-container-queuing.min-queue-length";
public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH = 5;
@@ -356,17 +362,20 @@ public class YarnConfiguration extends Configuration {
/** Max length of container queue at NodeManager. This is a cluster-wide
* configuration that acts as the upper-bound of optimal queue length
* calculated by the NodeQueueLoadMonitor */
+ @Unstable
public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH =
RM_PREFIX + "nm-container-queuing.max-queue-length";
public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH = 15;
/** Min queue wait time for a container at a NodeManager. */
+ @Unstable
public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS =
RM_PREFIX + "nm-container-queuing.min-queue-wait-time-ms";
public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS =
10;
/** Max queue wait time for a container queue at a NodeManager. */
+ @Unstable
public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS =
RM_PREFIX + "nm-container-queuing.max-queue-wait-time-ms";
public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
index ac26d88..223930e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md
@@ -72,10 +72,10 @@ By default, allocation of opportunistic containers is performed centrally throug
The following command can be used to run a sample pi map-reduce job, executing 40% of mappers using opportunistic containers (substitute `3.0.0-alpha2-SNAPSHOT` below with the version of Hadoop you are using):
```
-$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar pi -Dmapreduce.job.num-opportunistic-maps-percentage="40" 50 100
+$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar pi -Dmapreduce.job.num-opportunistic-maps-percent="40" 50 100
```
-By changing the value of `mapreduce.job.num-opportunistic-maps-percentage` in the above command, we can specify the percentage of mappers that can be executed through opportunistic containers.
+By changing the value of `mapreduce.job.num-opportunistic-maps-percent` in the above command, we can specify the percentage of mappers that can be executed through opportunistic containers.
###<a name="Opportunistic_Containers_in_Web_UI"></a>Opportunistic Containers in Web UI
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HADOOP-13896. Invoke assembly
plugin after building jars.
Posted by st...@apache.org.
HADOOP-13896. Invoke assembly plugin after building jars.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02766b6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02766b6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02766b6c
Branch: refs/heads/HADOOP-13345
Commit: 02766b6c224bc163a6fad37f146ade063e7fb422
Parents: 0a55bd8
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Jan 5 13:40:43 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Jan 5 13:40:43 2017 -0800
----------------------------------------------------------------------
hadoop-project-dist/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02766b6c/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 730ad05..2a03699 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -412,7 +412,7 @@
<executions>
<execution>
<id>dist</id>
- <phase>prepare-package</phase>
+ <phase>package</phase>
<goals>
<goal>single</goal>
</goals>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDFS-11259. Update fsck to display
maintenance state info. (Manoj Govindassamy via lei)
Posted by st...@apache.org.
HDFS-11259. Update fsck to display maintenance state info. (Manoj Govindassamy via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c18590fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c18590fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c18590fc
Branch: refs/heads/HADOOP-13345
Commit: c18590fce283378edb09acd4e764706a9a4a8b5f
Parents: 4c431a6
Author: Lei Xu <le...@apache.org>
Authored: Tue Jan 10 19:37:58 2017 +0800
Committer: Lei Xu <le...@apache.org>
Committed: Tue Jan 10 19:37:58 2017 +0800
----------------------------------------------------------------------
.../hdfs/server/namenode/NamenodeFsck.java | 61 +++-
.../org/apache/hadoop/hdfs/tools/DFSck.java | 4 +
.../hadoop/hdfs/server/namenode/TestFsck.java | 340 +++++++++++++++++--
3 files changed, 368 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c18590fc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 756a2f6..3941378 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -117,6 +117,9 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
public static final String HEALTHY_STATUS = "is HEALTHY";
public static final String DECOMMISSIONING_STATUS = "is DECOMMISSIONING";
public static final String DECOMMISSIONED_STATUS = "is DECOMMISSIONED";
+ public static final String ENTERING_MAINTENANCE_STATUS =
+ "is ENTERING MAINTENANCE";
+ public static final String IN_MAINTENANCE_STATUS = "is IN MAINTENANCE";
public static final String NONEXISTENT_STATUS = "does not exist";
public static final String FAILURE_STATUS = "FAILED";
public static final String UNDEFINED = "undefined";
@@ -280,6 +283,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
+ numberReplicas.decommissioned());
out.println("No. of decommissioning Replica: "
+ numberReplicas.decommissioning());
+ out.println("No. of entering maintenance Replica: "
+ + numberReplicas.liveEnteringMaintenanceReplicas());
+ out.println("No. of in maintenance Replica: "
+ + numberReplicas.maintenanceNotForReadReplicas());
out.println("No. of corrupted Replica: " +
numberReplicas.corruptReplicas());
//record datanodes that have corrupted block replica
@@ -300,6 +307,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
out.print(DECOMMISSIONED_STATUS);
} else if (dn.isDecommissionInProgress()) {
out.print(DECOMMISSIONING_STATUS);
+ } else if (dn.isEnteringMaintenance()) {
+ out.print(ENTERING_MAINTENANCE_STATUS);
+ } else if (dn.isInMaintenance()) {
+ out.print(IN_MAINTENANCE_STATUS);
} else {
out.print(HEALTHY_STATUS);
}
@@ -598,6 +609,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
sb.append("DECOMMISSIONED)");
} else if (dnDesc.isDecommissionInProgress()) {
sb.append("DECOMMISSIONING)");
+ } else if (dnDesc.isEnteringMaintenance()) {
+ sb.append("ENTERING MAINTENANCE)");
+ } else if (dnDesc.isInMaintenance()) {
+ sb.append("IN MAINTENANCE)");
} else if (corruptReplicas != null
&& corruptReplicas.contains(dnDesc)) {
sb.append("CORRUPT)");
@@ -620,7 +635,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
}
private void collectBlocksSummary(String parent, HdfsFileStatus file,
- Result res, LocatedBlocks blocks) throws IOException {
+ Result res, LocatedBlocks blocks) throws IOException {
String path = file.getFullName(parent);
boolean isOpen = blocks.isUnderConstruction();
if (isOpen && !showOpenFiles) {
@@ -651,13 +666,21 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
NumberReplicas numberReplicas = blockManager.countNodes(storedBlock);
int decommissionedReplicas = numberReplicas.decommissioned();
int decommissioningReplicas = numberReplicas.decommissioning();
+ int enteringMaintenanceReplicas =
+ numberReplicas.liveEnteringMaintenanceReplicas();
+ int inMaintenanceReplicas =
+ numberReplicas.maintenanceNotForReadReplicas();
res.decommissionedReplicas += decommissionedReplicas;
res.decommissioningReplicas += decommissioningReplicas;
+ res.enteringMaintenanceReplicas += enteringMaintenanceReplicas;
+ res.inMaintenanceReplicas += inMaintenanceReplicas;
// count total replicas
int liveReplicas = numberReplicas.liveReplicas();
- int totalReplicasPerBlock = liveReplicas + decommissionedReplicas +
- decommissioningReplicas;
+ int totalReplicasPerBlock = liveReplicas + decommissionedReplicas
+ + decommissioningReplicas
+ + enteringMaintenanceReplicas
+ + inMaintenanceReplicas;
res.totalReplicas += totalReplicasPerBlock;
boolean isMissing;
@@ -711,12 +734,14 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
if (!showFiles) {
out.print("\n" + path + ": ");
}
- out.println(" Under replicated " + block +
- ". Target Replicas is " +
- targetFileReplication + " but found " +
- liveReplicas + " live replica(s), " +
- decommissionedReplicas + " decommissioned replica(s) and " +
- decommissioningReplicas + " decommissioning replica(s).");
+ out.println(" Under replicated " + block + ". Target Replicas is "
+ + targetFileReplication + " but found "
+ + liveReplicas+ " live replica(s), "
+ + decommissionedReplicas + " decommissioned replica(s), "
+ + decommissioningReplicas + " decommissioning replica(s), "
+ + enteringMaintenanceReplicas
+ + " entering maintenance replica(s) and "
+ + inMaintenanceReplicas + " in maintenance replica(s).");
}
// count mis replicated blocks
@@ -1095,6 +1120,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
long missingReplicas = 0L;
long decommissionedReplicas = 0L;
long decommissioningReplicas = 0L;
+ long enteringMaintenanceReplicas = 0L;
+ long inMaintenanceReplicas = 0L;
long numUnderMinReplicatedBlocks = 0L;
long numOverReplicatedBlocks = 0L;
long numUnderReplicatedBlocks = 0L;
@@ -1243,6 +1270,14 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
res.append("\n DecommissioningReplicas:\t").append(
decommissioningReplicas);
}
+ if (enteringMaintenanceReplicas > 0) {
+ res.append("\n EnteringMaintenanceReplicas:\t").append(
+ enteringMaintenanceReplicas);
+ }
+ if (inMaintenanceReplicas > 0) {
+ res.append("\n InMaintenanceReplicas:\t").append(
+ inMaintenanceReplicas);
+ }
return res.toString();
}
}
@@ -1349,6 +1384,14 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
res.append("\n Decommissioning internal blocks:\t").append(
decommissioningReplicas);
}
+ if (enteringMaintenanceReplicas > 0) {
+ res.append("\n EnteringMaintenanceReplicas:\t").append(
+ enteringMaintenanceReplicas);
+ }
+ if (inMaintenanceReplicas > 0) {
+ res.append("\n InMaintenanceReplicas:\t").append(
+ inMaintenanceReplicas);
+ }
return res.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c18590fc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 9cf234a..0fd6122 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -372,6 +372,10 @@ public class DFSck extends Configured implements Tool {
errCode = 2;
} else if (lastLine.endsWith(NamenodeFsck.DECOMMISSIONING_STATUS)) {
errCode = 3;
+ } else if (lastLine.endsWith(NamenodeFsck.IN_MAINTENANCE_STATUS)) {
+ errCode = 4;
+ } else if (lastLine.endsWith(NamenodeFsck.ENTERING_MAINTENANCE_STATUS)) {
+ errCode = 5;
}
return errCode;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c18590fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 81a25fd..0956ed2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -54,6 +54,7 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -168,11 +169,11 @@ public class TestFsck {
PrintStream out = new PrintStream(bStream, true);
GenericTestUtils.setLogLevel(FSPermissionChecker.LOG, Level.ALL);
int errCode = ToolRunner.run(new DFSck(conf, out), path);
+ LOG.info("OUTPUT = " + bStream.toString());
if (checkErrorCode) {
assertEquals(expectedErrCode, errCode);
}
GenericTestUtils.setLogLevel(FSPermissionChecker.LOG, Level.INFO);
- LOG.info("OUTPUT = " + bStream.toString());
return bStream.toString();
}
@@ -878,14 +879,13 @@ public class TestFsck {
assertTrue(outStr.contains("dfs.namenode.replication.min:\t2"));
}
- @Test(timeout = 60000)
+ @Test(timeout = 90000)
public void testFsckReplicaDetails() throws Exception {
final short replFactor = 1;
short numDn = 1;
final long blockSize = 512;
final long fileSize = 1024;
- boolean checkDecommissionInProgress = false;
String[] racks = {"/rack1"};
String[] hosts = {"host1"};
@@ -910,49 +910,110 @@ public class TestFsck {
"-replicaDetails");
assertTrue(fsckOut.contains(NamenodeFsck.HEALTHY_STATUS));
assertTrue(fsckOut.contains("(LIVE)"));
+ assertTrue(!fsckOut.contains("(ENTERING MAINTENANCE)"));
+ assertTrue(!fsckOut.contains("(IN MAINTENANCE)"));
// decommission datanode
- ExtendedBlock eb = DFSTestUtil.getFirstBlock(dfs, path);
FSNamesystem fsn = cluster.getNameNode().getNamesystem();
BlockManager bm = fsn.getBlockManager();
- BlockCollection bc = null;
- try {
- fsn.writeLock();
- BlockInfo bi = bm.getStoredBlock(eb.getLocalBlock());
- bc = fsn.getBlockCollection(bi);
- } finally {
- fsn.writeUnlock();
- }
- DatanodeDescriptor dn = bc.getBlocks()[0]
- .getDatanode(0);
- bm.getDatanodeManager().getDecomManager().startDecommission(dn);
- String dnName = dn.getXferAddr();
+ final DatanodeManager dnm = bm.getDatanodeManager();
+ DatanodeDescriptor dnDesc0 = dnm.getDatanode(
+ cluster.getDataNodes().get(0).getDatanodeId());
+
+ bm.getDatanodeManager().getDecomManager().startDecommission(dnDesc0);
+ final String dn0Name = dnDesc0.getXferAddr();
// check the replica status while decommissioning
fsckOut = runFsck(conf, 0, true, testFile, "-files", "-blocks",
"-replicaDetails");
assertTrue(fsckOut.contains("(DECOMMISSIONING)"));
+ assertTrue(!fsckOut.contains("(ENTERING MAINTENANCE)"));
+ assertTrue(!fsckOut.contains("(IN MAINTENANCE)"));
- // Start 2nd Datanode and wait for decommission to start
- cluster.startDataNodes(conf, 1, true, null, null, null);
- DatanodeInfo datanodeInfo = null;
- do {
- Thread.sleep(2000);
- for (DatanodeInfo info : dfs.getDataNodeStats()) {
- if (dnName.equals(info.getXferAddr())) {
- datanodeInfo = info;
+ // Start 2nd DataNode
+ cluster.startDataNodes(conf, 1, true, null,
+ new String[] {"/rack2"}, new String[] {"host2"}, null, false);
+
+ // Wait for decommission to start
+ final AtomicBoolean checkDecommissionInProgress =
+ new AtomicBoolean(false);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ DatanodeInfo datanodeInfo = null;
+ try {
+ for (DatanodeInfo info : dfs.getDataNodeStats()) {
+ if (dn0Name.equals(info.getXferAddr())) {
+ datanodeInfo = info;
+ }
+ }
+ if (!checkDecommissionInProgress.get() && datanodeInfo != null
+ && datanodeInfo.isDecommissionInProgress()) {
+ checkDecommissionInProgress.set(true);
+ }
+ if (datanodeInfo != null && datanodeInfo.isDecommissioned()) {
+ return true;
+ }
+ } catch (Exception e) {
+ LOG.warn("Unexpected exception: " + e);
+ return false;
}
+ return false;
}
- if (!checkDecommissionInProgress && datanodeInfo != null
- && datanodeInfo.isDecommissionInProgress()) {
- checkDecommissionInProgress = true;
+ }, 500, 30000);
+
+ // check the replica status after decommission is done
+ fsckOut = runFsck(conf, 0, true, testFile, "-files", "-blocks",
+ "-replicaDetails");
+ assertTrue(fsckOut.contains("(DECOMMISSIONED)"));
+ assertTrue(!fsckOut.contains("(ENTERING MAINTENANCE)"));
+ assertTrue(!fsckOut.contains("(IN MAINTENANCE)"));
+
+ DatanodeDescriptor dnDesc1 = dnm.getDatanode(
+ cluster.getDataNodes().get(1).getDatanodeId());
+ final String dn1Name = dnDesc1.getXferAddr();
+
+ bm.getDatanodeManager().getDecomManager().startMaintenance(dnDesc1,
+ Long.MAX_VALUE);
+ // check the replica status while entering maintenance
+ fsckOut = runFsck(conf, 0, true, testFile, "-files", "-blocks",
+ "-replicaDetails");
+ assertTrue(fsckOut.contains("(DECOMMISSIONED)"));
+ assertTrue(fsckOut.contains("(ENTERING MAINTENANCE)"));
+ assertTrue(!fsckOut.contains("(IN MAINTENANCE)"));
+
+ // Start 3rd DataNode
+ cluster.startDataNodes(conf, 1, true, null,
+ new String[] {"/rack3"}, new String[] {"host3"}, null, false);
+
+ // Wait for the 2nd node to reach in maintenance state
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ DatanodeInfo dnInfo = null;
+ try {
+ for (DatanodeInfo info : dfs.getDataNodeStats()) {
+ if (dn1Name.equals(info.getXferAddr())) {
+ dnInfo = info;
+ }
+ }
+ if (dnInfo != null && dnInfo.isInMaintenance()) {
+ return true;
+ }
+ } catch (Exception e) {
+ LOG.warn("Unexpected exception: " + e);
+ return false;
+ }
+ return false;
}
- } while (datanodeInfo != null && !datanodeInfo.isDecommissioned());
+ }, 500, 30000);
// check the replica status after decommission is done
fsckOut = runFsck(conf, 0, true, testFile, "-files", "-blocks",
"-replicaDetails");
assertTrue(fsckOut.contains("(DECOMMISSIONED)"));
+ assertTrue(!fsckOut.contains("(ENTERING MAINTENANCE)"));
+ assertTrue(fsckOut.contains("(IN MAINTENANCE)"));
}
/** Test if fsck can return -1 in case of failure.
@@ -1460,6 +1521,118 @@ public class TestFsck {
}
/**
+ * Test for blockIdCK with datanode maintenance.
+ */
+ @Test (timeout = 90000)
+ public void testBlockIdCKMaintenance() throws Exception {
+ final short replFactor = 2;
+ short numDn = 2;
+ final long blockSize = 512;
+ String[] hosts = {"host1", "host2"};
+ String[] racks = {"/rack1", "/rack2"};
+
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replFactor);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, replFactor);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY,
+ replFactor);
+
+ DistributedFileSystem dfs;
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(numDn)
+ .hosts(hosts)
+ .racks(racks)
+ .build();
+
+ assertNotNull("Failed Cluster Creation", cluster);
+ cluster.waitClusterUp();
+ dfs = cluster.getFileSystem();
+ assertNotNull("Failed to get FileSystem", dfs);
+
+ DFSTestUtil util = new DFSTestUtil.Builder().
+ setName(getClass().getSimpleName()).setNumFiles(1).build();
+ //create files
+ final String pathString = new String("/testfile");
+ final Path path = new Path(pathString);
+ util.createFile(dfs, path, 1024, replFactor, 1000L);
+ util.waitReplication(dfs, path, replFactor);
+ StringBuilder sb = new StringBuilder();
+ for (LocatedBlock lb: util.getAllBlocks(dfs, path)){
+ sb.append(lb.getBlock().getLocalBlock().getBlockName()+" ");
+ }
+ String[] bIds = sb.toString().split(" ");
+
+ //make sure datanode that has replica is fine before maintenance
+ String outStr = runFsck(conf, 0, true, "/", "-blockId", bIds[0]);
+ System.out.println(outStr);
+ assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
+
+ FSNamesystem fsn = cluster.getNameNode().getNamesystem();
+ BlockManager bm = fsn.getBlockManager();
+ DatanodeManager dnm = bm.getDatanodeManager();
+ DatanodeDescriptor dn = dnm.getDatanode(cluster.getDataNodes().get(0)
+ .getDatanodeId());
+ bm.getDatanodeManager().getDecomManager().startMaintenance(dn,
+ Long.MAX_VALUE);
+ final String dnName = dn.getXferAddr();
+
+ //wait for the node to enter maintenance state
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ DatanodeInfo datanodeInfo = null;
+ try {
+ for (DatanodeInfo info : dfs.getDataNodeStats()) {
+ if (dnName.equals(info.getXferAddr())) {
+ datanodeInfo = info;
+ }
+ }
+ if (datanodeInfo != null && datanodeInfo.isEnteringMaintenance()) {
+ String fsckOut = runFsck(conf, 5, false, "/", "-blockId", bIds[0]);
+ assertTrue(fsckOut.contains(
+ NamenodeFsck.ENTERING_MAINTENANCE_STATUS));
+ return true;
+ }
+ } catch (Exception e) {
+ LOG.warn("Unexpected exception: " + e);
+ return false;
+ }
+ return false;
+ }
+ }, 500, 30000);
+
+ // Start 3rd DataNode
+ cluster.startDataNodes(conf, 1, true, null,
+ new String[] {"/rack3"}, new String[] {"host3"}, null, false);
+
+ // Wait for 1st node to reach in maintenance state
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ DatanodeInfo datanodeInfo = null;
+ for (DatanodeInfo info : dfs.getDataNodeStats()) {
+ if (dnName.equals(info.getXferAddr())) {
+ datanodeInfo = info;
+ }
+ }
+ if (datanodeInfo != null && datanodeInfo.isInMaintenance()) {
+ return true;
+ }
+ } catch (Exception e) {
+ LOG.warn("Unexpected exception: " + e);
+ return false;
+ }
+ return false;
+ }
+ }, 500, 30000);
+
+ //check in maintenance node
+ String fsckOut = runFsck(conf, 4, false, "/", "-blockId", bIds[0]);
+ assertTrue(fsckOut.contains(NamenodeFsck.IN_MAINTENANCE_STATUS));
+ }
+
+ /**
* Test for blockIdCK with block corruption.
*/
@Test
@@ -1655,6 +1828,117 @@ public class TestFsck {
String fsckOut = runFsck(conf, 0, true, testFile);
}
+ /**
+ * Test for blocks on maintenance hosts are not shown as missing.
+ */
+ @Test (timeout = 90000)
+ public void testFsckWithMaintenanceReplicas() throws Exception {
+ final short replFactor = 2;
+ short numDn = 2;
+ final long blockSize = 512;
+ String[] hosts = {"host1", "host2"};
+ String[] racks = {"/rack1", "/rack2"};
+
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replFactor);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, replFactor);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY,
+ replFactor);
+
+ DistributedFileSystem dfs;
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(numDn)
+ .hosts(hosts)
+ .racks(racks)
+ .build();
+
+ assertNotNull("Failed Cluster Creation", cluster);
+ cluster.waitClusterUp();
+ dfs = cluster.getFileSystem();
+ assertNotNull("Failed to get FileSystem", dfs);
+
+ DFSTestUtil util = new DFSTestUtil.Builder().
+ setName(getClass().getSimpleName()).setNumFiles(1).build();
+ //create files
+ final String testFile = new String("/testfile");
+ final Path path = new Path(testFile);
+ util.createFile(dfs, path, 1024, replFactor, 1000L);
+ util.waitReplication(dfs, path, replFactor);
+ StringBuilder sb = new StringBuilder();
+ for (LocatedBlock lb: util.getAllBlocks(dfs, path)){
+ sb.append(lb.getBlock().getLocalBlock().getBlockName()+" ");
+ }
+ String[] bIds = sb.toString().split(" ");
+
+ //make sure datanode that has replica is fine before maintenance
+ String outStr = runFsck(conf, 0, true, testFile);
+ System.out.println(outStr);
+ assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
+
+ FSNamesystem fsn = cluster.getNameNode().getNamesystem();
+ BlockManager bm = fsn.getBlockManager();
+ DatanodeManager dnm = bm.getDatanodeManager();
+ DatanodeDescriptor dn = dnm.getDatanode(cluster.getDataNodes().get(0)
+ .getDatanodeId());
+ bm.getDatanodeManager().getDecomManager().startMaintenance(dn,
+ Long.MAX_VALUE);
+ final String dnName = dn.getXferAddr();
+
+ //wait for the node to enter maintenance state
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ DatanodeInfo datanodeInfo = null;
+ try {
+ for (DatanodeInfo info : dfs.getDataNodeStats()) {
+ if (dnName.equals(info.getXferAddr())) {
+ datanodeInfo = info;
+ }
+ }
+ if (datanodeInfo != null && datanodeInfo.isEnteringMaintenance()) {
+ // verify fsck returns Healthy status
+ String fsckOut = runFsck(conf, 0, true, testFile);
+ assertTrue(fsckOut.contains(NamenodeFsck.HEALTHY_STATUS));
+ return true;
+ }
+ } catch (Exception e) {
+ LOG.warn("Unexpected exception: " + e);
+ return false;
+ }
+ return false;
+ }
+ }, 500, 30000);
+
+ // Start 3rd DataNode and wait for node to reach in maintenance state
+ cluster.startDataNodes(conf, 1, true, null,
+ new String[] {"/rack3"}, new String[] {"host3"}, null, false);
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ DatanodeInfo datanodeInfo = null;
+ try {
+ for (DatanodeInfo info : dfs.getDataNodeStats()) {
+ if (dnName.equals(info.getXferAddr())) {
+ datanodeInfo = info;
+ }
+ }
+ if (datanodeInfo != null && datanodeInfo.isInMaintenance()) {
+ return true;
+ }
+ } catch (Exception e) {
+ LOG.warn("Unexpected exception: " + e);
+ return false;
+ }
+ return false;
+ }
+ }, 500, 30000);
+
+ // verify fsck returns Healthy status
+ String fsckOut = runFsck(conf, 0, true, testFile);
+ assertTrue(fsckOut.contains(NamenodeFsck.HEALTHY_STATUS));
+ }
+
@Test
public void testECFsck() throws Exception {
FileSystem fs = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: MAPREDUCE-6715. Fix Several Unsafe
Practices (Contributed by Yufei Gu via Daniel Templeton)
Posted by st...@apache.org.
MAPREDUCE-6715. Fix Several Unsafe Practices (Contributed by Yufei Gu via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b8a7c18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b8a7c18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b8a7c18
Branch: refs/heads/HADOOP-13345
Commit: 0b8a7c18ddbe73b356b3c9baf4460659ccaee095
Parents: 5d18294
Author: Daniel Templeton <te...@apache.org>
Authored: Thu Jan 5 17:55:05 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Thu Jan 5 17:56:08 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/mapred/CleanupQueue.java | 7 +++-
.../java/org/apache/hadoop/mapred/MapTask.java | 10 ++++-
.../mapreduce/lib/output/TextOutputFormat.java | 15 ++++----
.../task/reduce/ShuffleSchedulerImpl.java | 40 +++++++++++---------
.../hadoop/examples/dancing/Pentomino.java | 5 +++
.../hadoop/examples/terasort/TeraScheduler.java | 16 ++++----
6 files changed, 57 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b8a7c18/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
index 456ed7c..2282b54 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
@@ -136,7 +136,12 @@ class CleanupQueue {
LOG.debug("DELETED " + context.fullPath);
}
} catch (InterruptedException t) {
- LOG.warn("Interrupted deletion of " + context.fullPath);
+ if (context == null) {
+ LOG.warn("Interrupted deletion of an invalid path: Path deletion "
+ + "context is null.");
+ } else {
+ LOG.warn("Interrupted deletion of " + context.fullPath);
+ }
return;
} catch (Exception e) {
LOG.warn("Error deleting path " + context.fullPath + ": " + e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b8a7c18/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
index 3753fba..9ec0914 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
@@ -411,8 +411,14 @@ public class MapTask extends Task {
LOG.warn(msg, e);
}
}
- throw new IOException("Initialization of all the collectors failed. " +
- "Error in last collector was :" + lastException.getMessage(), lastException);
+
+ if (lastException != null) {
+ throw new IOException("Initialization of all the collectors failed. " +
+ "Error in last collector was:" + lastException.toString(),
+ lastException);
+ } else {
+ throw new IOException("Initialization of all the collectors failed.");
+ }
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b8a7c18/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
index 1c8ea72..2e49f68 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
@@ -113,19 +113,18 @@ public class TextOutputFormat<K, V> extends FileOutputFormat<K, V> {
if (isCompressed) {
Class<? extends CompressionCodec> codecClass =
getOutputCompressorClass(job, GzipCodec.class);
- codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
+ codec = ReflectionUtils.newInstance(codecClass, conf);
extension = codec.getDefaultExtension();
}
Path file = getDefaultWorkFile(job, extension);
FileSystem fs = file.getFileSystem(conf);
- if (!isCompressed) {
- FSDataOutputStream fileOut = fs.create(file, false);
- return new LineRecordWriter<K, V>(fileOut, keyValueSeparator);
+ FSDataOutputStream fileOut = fs.create(file, false);
+ if (isCompressed) {
+ return new LineRecordWriter<>(
+ new DataOutputStream(codec.createOutputStream(fileOut)),
+ keyValueSeparator);
} else {
- FSDataOutputStream fileOut = fs.create(file, false);
- return new LineRecordWriter<K, V>(new DataOutputStream
- (codec.createOutputStream(fileOut)),
- keyValueSeparator);
+ return new LineRecordWriter<>(fileOut, keyValueSeparator);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b8a7c18/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
index a819771..2b6dc57 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
@@ -433,25 +433,29 @@ public class ShuffleSchedulerImpl<K,V> implements ShuffleScheduler<K,V> {
public synchronized MapHost getHost() throws InterruptedException {
- while(pendingHosts.isEmpty()) {
- wait();
- }
+ while(pendingHosts.isEmpty()) {
+ wait();
+ }
- MapHost host = null;
- Iterator<MapHost> iter = pendingHosts.iterator();
- int numToPick = random.nextInt(pendingHosts.size());
- for (int i=0; i <= numToPick; ++i) {
- host = iter.next();
- }
+ Iterator<MapHost> iter = pendingHosts.iterator();
+ // Safe to take one because we know pendingHosts isn't empty
+ MapHost host = iter.next();
+ int numToPick = random.nextInt(pendingHosts.size());
+ for (int i = 0; i < numToPick; ++i) {
+ host = iter.next();
+ }
- pendingHosts.remove(host);
- host.markBusy();
+ pendingHosts.remove(host);
+ host.markBusy();
- LOG.debug("Assigning " + host + " with " + host.getNumKnownMapOutputs() +
- " to " + Thread.currentThread().getName());
- SHUFFLE_START.set(Time.monotonicNow());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Assigning " + host + " with " + host.getNumKnownMapOutputs() + " to "
+ + Thread.currentThread().getName());
+ }
+ SHUFFLE_START.set(Time.monotonicNow());
- return host;
+ return host;
}
public synchronized List<TaskAttemptID> getMapsForHost(MapHost host) {
@@ -477,8 +481,10 @@ public class ShuffleSchedulerImpl<K,V> implements ShuffleScheduler<K,V> {
host.addKnownMap(id);
}
}
- LOG.debug("assigned " + includedMaps + " of " + totalSize + " to " +
- host + " to " + Thread.currentThread().getName());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("assigned " + includedMaps + " of " + totalSize + " to " + host
+ + " to " + Thread.currentThread().getName());
+ }
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b8a7c18/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java
index 5e636b9..2485728 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java
@@ -153,6 +153,11 @@ public class Pentomino {
break;
}
}
+
+ if (piece == null) {
+ continue;
+ }
+
// for each point where the piece was placed, mark it with the piece name
for(ColumnName item: row) {
if (item instanceof Point) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b8a7c18/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java
index 30b50d8..3e12a3d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java
@@ -73,14 +73,14 @@ class TeraScheduler {
List<String> readFile(String filename) throws IOException {
List<String> result = new ArrayList<String>(10000);
- BufferedReader in = new BufferedReader(
- new InputStreamReader(new FileInputStream(filename), Charsets.UTF_8));
- String line = in.readLine();
- while (line != null) {
- result.add(line);
- line = in.readLine();
- }
- in.close();
+ try (BufferedReader in = new BufferedReader(
+ new InputStreamReader(new FileInputStream(filename), Charsets.UTF_8))) {
+ String line = in.readLine();
+ while (line != null) {
+ result.add(line);
+ line = in.readLine();
+ }
+ }
return result;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HADOOP-13578. Add Codec for
ZStandard Compression. Contributed by churro morales
Posted by st...@apache.org.
HADOOP-13578. Add Codec for ZStandard Compression. Contributed by churro morales
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0a27616
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0a27616
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0a27616
Branch: refs/heads/HADOOP-13345
Commit: a0a276162147e843a5a4e028abdca5b66f5118da
Parents: e49e0a6
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jan 4 14:46:25 2017 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Jan 4 14:46:25 2017 +0000
----------------------------------------------------------------------
BUILDING.txt | 25 +
dev-support/bin/dist-copynativelibs | 13 +
hadoop-common-project/hadoop-common/pom.xml | 21 +
.../hadoop-common/src/CMakeLists.txt | 29 ++
.../hadoop-common/src/config.h.cmake | 1 +
.../hadoop/fs/CommonConfigurationKeys.java | 16 +
.../apache/hadoop/io/compress/Decompressor.java | 2 +-
.../hadoop/io/compress/ZStandardCodec.java | 242 +++++++++
.../io/compress/zstd/ZStandardCompressor.java | 305 ++++++++++++
.../io/compress/zstd/ZStandardDecompressor.java | 323 ++++++++++++
.../hadoop/io/compress/zstd/package-info.java | 22 +
.../apache/hadoop/util/NativeCodeLoader.java | 5 +
.../hadoop/util/NativeLibraryChecker.java | 13 +-
.../io/compress/zstd/ZStandardCompressor.c | 259 ++++++++++
.../io/compress/zstd/ZStandardDecompressor.c | 218 +++++++++
.../zstd/org_apache_hadoop_io_compress_zstd.h | 34 ++
.../org/apache/hadoop/util/NativeCodeLoader.c | 11 +
...g.apache.hadoop.io.compress.CompressionCodec | 1 +
.../src/site/markdown/NativeLibraries.md.vm | 1 +
.../apache/hadoop/io/compress/TestCodec.java | 12 +
.../io/compress/TestCompressionStreamReuse.java | 8 +
.../TestZStandardCompressorDecompressor.java | 485 +++++++++++++++++++
.../src/test/resources/zstd/test_file.txt | 71 +++
.../src/test/resources/zstd/test_file.txt.zst | Bin 0 -> 3690 bytes
.../hadoop-mapreduce-client-nativetask/pom.xml | 8 +
hadoop-project-dist/pom.xml | 6 +
hadoop-project/pom.xml | 2 +
27 files changed, 2130 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 7afc3f0..a1721ba 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -83,6 +83,8 @@ Optional packages:
$ sudo apt-get install libjansson-dev
* Linux FUSE
$ sudo apt-get install fuse libfuse-dev
+* ZStandard compression
+ $ sudo apt-get install zstd
----------------------------------------------------------------------------------
Maven main modules:
@@ -155,6 +157,29 @@ Maven build goals:
and it ignores the -Dsnappy.prefix option. If -Dsnappy.lib isn't given, the
bundling and building will fail.
+
+ ZStandard build options:
+
+ ZStandard is a compression library that can be utilized by the native code.
+ It is currently an optional component, meaning that Hadoop can be built with
+ or without this dependency.
+
+ * Use -Drequire.zstd to fail the build if libzstd.so is not found.
+ If this option is not specified and the zstd library is missing.
+
+ * Use -Dzstd.prefix to specify a nonstandard location for the libzstd
+ header files and library files. You do not need this option if you have
+ installed zstandard using a package manager.
+
+ * Use -Dzstd.lib to specify a nonstandard location for the libzstd library
+ files. Similarly to zstd.prefix, you do not need this option if you have
+ installed using a package manager.
+
+ * Use -Dbundle.zstd to copy the contents of the zstd.lib directory into
+ the final tar file. This option requires that -Dzstd.lib is also given,
+ and it ignores the -Dzstd.prefix option. If -Dzstd.lib isn't given, the
+ bundling and building will fail.
+
OpenSSL build options:
OpenSSL includes a crypto library that can be utilized by the native code.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/dev-support/bin/dist-copynativelibs
----------------------------------------------------------------------
diff --git a/dev-support/bin/dist-copynativelibs b/dev-support/bin/dist-copynativelibs
index efe250b..67d2edf 100755
--- a/dev-support/bin/dist-copynativelibs
+++ b/dev-support/bin/dist-copynativelibs
@@ -114,6 +114,15 @@ for i in "$@"; do
--snappylibbundle=*)
SNAPPYLIBBUNDLE=${i#*=}
;;
+ --zstdbinbundle=*)
+ ZSTDBINBUNDLE=${i#*=}
+ ;;
+ --zstdlib=*)
+ ZSTDLIB=${i#*=}
+ ;;
+ --zstdlibbundle=*)
+ ZSTDLIBBUNDLE=${i#*=}
+ ;;
esac
done
@@ -139,6 +148,8 @@ if [[ -d "${LIB_DIR}" ]]; then
bundle_native_lib "${SNAPPYLIBBUNDLE}" "snappy.lib" "snappy" "${SNAPPYLIB}"
+ bundle_native_lib "${ZSTDLIBBUNDLE}" "zstd.lib" "zstd" "${ZSTDLIB}"
+
bundle_native_lib "${OPENSSLLIBBUNDLE}" "openssl.lib" "crypto" "${OPENSSLLIB}"
bundle_native_lib "${ISALBUNDLE}" "isal.lib" "isa" "${ISALLIB}"
@@ -159,6 +170,8 @@ if [[ -d "${BIN_DIR}" ]] ; then
bundle_native_bin "${SNAPPYBINBUNDLE}" "${SNAPPYLIBBUNDLE}" "snappy.lib" "snappy" "${SNAPPYLIB}"
+ bundle_native_bin "${ZSTDBINBUNDLE}" "${ZSTDLIBBUNDLE}" "zstd.lib" "zstd" "${ZSTDLIB}"
+
bundle_native_bin "${OPENSSLBINBUNDLE}" "${OPENSSLLIBBUNDLE}" "openssl.lib" "crypto" "${OPENSSLLIB}"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index c9b282f..b616074 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -595,6 +595,10 @@
<snappy.lib></snappy.lib>
<snappy.include></snappy.include>
<require.snappy>false</require.snappy>
+ <zstd.prefix></zstd.prefix>
+ <zstd.lib></zstd.lib>
+ <zstd.include></zstd.include>
+ <require.zstd>false</require.zstd>
<openssl.prefix></openssl.prefix>
<openssl.lib></openssl.lib>
<openssl.include></openssl.include>
@@ -652,6 +656,8 @@
<javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyCompressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
+ <javahClassName>org.apache.hadoop.io.compress.zstd.ZStandardCompressor</javahClassName>
+ <javahClassName>org.apache.hadoop.io.compress.zstd.ZStandardDecompressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
<javahClassName>org.apache.hadoop.io.erasurecode.ErasureCodeNative</javahClassName>
@@ -685,9 +691,13 @@
<JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
<REQUIRE_BZIP2>${require.bzip2}</REQUIRE_BZIP2>
<REQUIRE_SNAPPY>${require.snappy}</REQUIRE_SNAPPY>
+ <REQUIRE_ZSTD>${require.zstd}</REQUIRE_ZSTD>
<CUSTOM_SNAPPY_PREFIX>${snappy.prefix}</CUSTOM_SNAPPY_PREFIX>
<CUSTOM_SNAPPY_LIB>${snappy.lib} </CUSTOM_SNAPPY_LIB>
<CUSTOM_SNAPPY_INCLUDE>${snappy.include} </CUSTOM_SNAPPY_INCLUDE>
+ <CUSTOM_ZSTD_PREFIX>${zstd.prefix}</CUSTOM_ZSTD_PREFIX>
+ <CUSTOM_ZSTD_LIB>${zstd.lib} </CUSTOM_ZSTD_LIB>
+ <CUSTOM_ZSTD_INCLUDE>${zstd.include} </CUSTOM_ZSTD_INCLUDE>
<REQUIRE_ISAL>${require.isal} </REQUIRE_ISAL>
<CUSTOM_ISAL_PREFIX>${isal.prefix} </CUSTOM_ISAL_PREFIX>
<CUSTOM_ISAL_LIB>${isal.lib} </CUSTOM_ISAL_LIB>
@@ -745,6 +755,11 @@
<isal.lib></isal.lib>
<require.snappy>false</require.snappy>
<bundle.snappy.in.bin>true</bundle.snappy.in.bin>
+ <zstd.prefix></zstd.prefix>
+ <zstd.lib></zstd.lib>
+ <zstd.include></zstd.include>
+ <require.ztsd>false</require.ztsd>
+ <bundle.zstd.in.bin>true</bundle.zstd.in.bin>
<openssl.prefix></openssl.prefix>
<openssl.lib></openssl.lib>
<openssl.include></openssl.include>
@@ -794,6 +809,8 @@
<javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyCompressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
+ <javahClassName>org.apache.hadoop.io.compress.zstd.ZStandardCompressor</javahClassName>
+ <javahClassName>org.apache.hadoop.io.compress.zstd.ZStandardDecompressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
<javahClassName>org.apache.hadoop.io.erasurecode.ErasureCodeNative</javahClassName>
@@ -850,6 +867,10 @@
<argument>/p:CustomSnappyLib=${snappy.lib}</argument>
<argument>/p:CustomSnappyInclude=${snappy.include}</argument>
<argument>/p:RequireSnappy=${require.snappy}</argument>
+ <argument>/p:CustomZstdPrefix=${zstd.prefix}</argument>
+ <argument>/p:CustomZstdLib=${zstd.lib}</argument>
+ <argument>/p:CustomZstdInclude=${zstd.include}</argument>
+ <argument>/p:RequireZstd=${require.ztsd}</argument>
<argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
<argument>/p:CustomOpensslLib=${openssl.lib}</argument>
<argument>/p:CustomOpensslInclude=${openssl.include}</argument>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
index 8317a46..10b0f23 100644
--- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt
+++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
@@ -94,6 +94,33 @@ else()
endif()
endif()
+# Require zstandard
+SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+hadoop_set_find_shared_library_version("1")
+find_library(ZSTD_LIBRARY
+ NAMES zstd
+ PATHS ${CUSTOM_ZSTD_PREFIX} ${CUSTOM_ZSTD_PREFIX}/lib
+ ${CUSTOM_ZSTD_PREFIX}/lib64 ${CUSTOM_ZSTD_LIB})
+SET(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
+find_path(ZSTD_INCLUDE_DIR
+ NAMES zstd.h
+ PATHS ${CUSTOM_ZSTD_PREFIX} ${CUSTOM_ZSTD_PREFIX}/include
+ ${CUSTOM_ZSTD_INCLUDE})
+if (ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
+ GET_FILENAME_COMPONENT(HADOOP_ZSTD_LIBRARY ${ZSTD_LIBRARY} NAME)
+ set(ZSTD_SOURCE_FILES
+ "${SRC}/io/compress/zstd/ZStandardCompressor.c"
+ "${SRC}/io/compress/zstd/ZStandardDecompressor.c")
+ set(REQUIRE_ZSTD ${REQUIRE_ZSTD}) # Stop warning about unused variable.
+ message(STATUS "Found ZStandard: ${ZSTD_LIBRARY}")
+else ()
+ set(ZSTD_INCLUDE_DIR "")
+ set(ZSTD_SOURCE_FILES "")
+ IF(REQUIRE_ZSTD)
+ MESSAGE(FATAL_ERROR "Required zstandard library could not be found. ZSTD_LIBRARY=${ZSTD_LIBRARY}, ZSTD_INCLUDE_DIR=${ZSTD_INCLUDE_DIR}, CUSTOM_ZSTD_INCLUDE_DIR=${CUSTOM_ZSTD_INCLUDE_DIR}, CUSTOM_ZSTD_PREFIX=${CUSTOM_ZSTD_PREFIX}, CUSTOM_ZSTD_INCLUDE=${CUSTOM_ZSTD_INCLUDE}")
+ ENDIF(REQUIRE_ZSTD)
+endif ()
+
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
hadoop_set_find_shared_library_version("2")
find_library(ISAL_LIBRARY
@@ -208,6 +235,7 @@ include_directories(
${BZIP2_INCLUDE_DIR}
${SNAPPY_INCLUDE_DIR}
${ISAL_INCLUDE_DIR}
+ ${ZSTD_INCLUDE_DIR}
${OPENSSL_INCLUDE_DIR}
${SRC}/util
)
@@ -222,6 +250,7 @@ hadoop_add_dual_library(hadoop
${SRC}/io/compress/lz4/lz4hc.c
${ISAL_SOURCE_FILES}
${SNAPPY_SOURCE_FILES}
+ ${ZSTD_SOURCE_FILES}
${OPENSSL_SOURCE_FILES}
${SRC}/io/compress/zlib/ZlibCompressor.c
${SRC}/io/compress/zlib/ZlibDecompressor.c
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/config.h.cmake
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/config.h.cmake b/hadoop-common-project/hadoop-common/src/config.h.cmake
index 445cc33..40aa467 100644
--- a/hadoop-common-project/hadoop-common/src/config.h.cmake
+++ b/hadoop-common-project/hadoop-common/src/config.h.cmake
@@ -21,6 +21,7 @@
#cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@"
#cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@"
#cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@"
+#cmakedefine HADOOP_ZSTD_LIBRARY "@HADOOP_ZSTD_LIBRARY@"
#cmakedefine HADOOP_OPENSSL_LIBRARY "@HADOOP_OPENSSL_LIBRARY@"
#cmakedefine HADOOP_ISAL_LIBRARY "@HADOOP_ISAL_LIBRARY@"
#cmakedefine HAVE_SYNC_FILE_RANGE
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index fe522b3..b8a60d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -141,6 +141,22 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
public static final int IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT =
256 * 1024;
+ /** ZStandard compression level. */
+ public static final String IO_COMPRESSION_CODEC_ZSTD_LEVEL_KEY =
+ "io.compression.codec.zstd.level";
+
+ /** Default value for IO_COMPRESSION_CODEC_ZSTD_LEVEL_KEY. */
+ public static final int IO_COMPRESSION_CODEC_ZSTD_LEVEL_DEFAULT = 3;
+
+ /** ZStandard buffer size. */
+ public static final String IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_KEY =
+ "io.compression.codec.zstd.buffersize";
+
+ /** ZStandard buffer size a value of 0 means use the recommended zstd
+ * buffer size that the library recommends. */
+ public static final int
+ IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_DEFAULT = 0;
+
/** Internal buffer size for Lz4 compressor/decompressors */
public static final String IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY =
"io.compression.codec.lz4.buffersize";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java
index 8cb0b2a..3808003 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java
@@ -95,7 +95,7 @@ public interface Decompressor {
* @param b Buffer for the compressed data
* @param off Start offset of the data
* @param len Size of the buffer
- * @return The actual number of bytes of compressed data.
+ * @return The actual number of bytes of uncompressed data.
* @throws IOException
*/
public int decompress(byte[] b, int off, int len) throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java
new file mode 100644
index 0000000..c56bbba
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.compress.zstd.ZStandardCompressor;
+import org.apache.hadoop.io.compress.zstd.ZStandardDecompressor;
+import org.apache.hadoop.util.NativeCodeLoader;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_KEY;
+
+/**
+ * This class creates zstd compressors/decompressors.
+ */
+public class ZStandardCodec implements
+ Configurable, CompressionCodec, DirectDecompressionCodec {
+ private Configuration conf;
+
+ /**
+ * Set the configuration to be used by this object.
+ *
+ * @param conf the configuration object.
+ */
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Return the configuration used by this object.
+ *
+ * @return the configuration object used by this object.
+ */
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public static void checkNativeCodeLoaded() {
+ if (!NativeCodeLoader.isNativeCodeLoaded() ||
+ !NativeCodeLoader.buildSupportsZstd()) {
+ throw new RuntimeException("native zStandard library "
+ + "not available: this version of libhadoop was built "
+ + "without zstd support.");
+ }
+ if (!ZStandardCompressor.isNativeCodeLoaded()) {
+ throw new RuntimeException("native zStandard library not "
+ + "available: ZStandardCompressor has not been loaded.");
+ }
+ if (!ZStandardDecompressor.isNativeCodeLoaded()) {
+ throw new RuntimeException("native zStandard library not "
+ + "available: ZStandardDecompressor has not been loaded.");
+ }
+ }
+
+ public static boolean isNativeCodeLoaded() {
+ return ZStandardCompressor.isNativeCodeLoaded()
+ && ZStandardDecompressor.isNativeCodeLoaded();
+ }
+
+ public static String getLibraryName() {
+ return ZStandardCompressor.getLibraryName();
+ }
+
+ public static int getCompressionLevel(Configuration conf) {
+ return conf.getInt(
+ CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_LEVEL_KEY,
+ CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_LEVEL_DEFAULT);
+ }
+
+ public static int getCompressionBufferSize(Configuration conf) {
+ int bufferSize = getBufferSize(conf);
+ return bufferSize == 0 ?
+ ZStandardCompressor.getRecommendedBufferSize() :
+ bufferSize;
+ }
+
+ public static int getDecompressionBufferSize(Configuration conf) {
+ int bufferSize = getBufferSize(conf);
+ return bufferSize == 0 ?
+ ZStandardDecompressor.getRecommendedBufferSize() :
+ bufferSize;
+ }
+
+ private static int getBufferSize(Configuration conf) {
+ return conf.getInt(IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_KEY,
+ IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_DEFAULT);
+ }
+
+ /**
+ * Create a {@link CompressionOutputStream} that will write to the given
+ * {@link OutputStream}.
+ *
+ * @param out the location for the final output stream
+ * @return a stream the user can write uncompressed data to have compressed
+ * @throws IOException
+ */
+ @Override
+ public CompressionOutputStream createOutputStream(OutputStream out)
+ throws IOException {
+ return Util.
+ createOutputStreamWithCodecPool(this, conf, out);
+ }
+
+ /**
+ * Create a {@link CompressionOutputStream} that will write to the given
+ * {@link OutputStream} with the given {@link Compressor}.
+ *
+ * @param out the location for the final output stream
+ * @param compressor compressor to use
+ * @return a stream the user can write uncompressed data to have compressed
+ * @throws IOException
+ */
+ @Override
+ public CompressionOutputStream createOutputStream(OutputStream out,
+ Compressor compressor)
+ throws IOException {
+ checkNativeCodeLoaded();
+ return new CompressorStream(out, compressor,
+ getCompressionBufferSize(conf));
+ }
+
+ /**
+ * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+ *
+ * @return the type of compressor needed by this codec.
+ */
+ @Override
+ public Class<? extends Compressor> getCompressorType() {
+ checkNativeCodeLoaded();
+ return ZStandardCompressor.class;
+ }
+
+ /**
+ * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+ *
+ * @return a new compressor for use by this codec
+ */
+ @Override
+ public Compressor createCompressor() {
+ checkNativeCodeLoaded();
+ return new ZStandardCompressor(
+ getCompressionLevel(conf), getCompressionBufferSize(conf));
+ }
+
+
+ /**
+ * Create a {@link CompressionInputStream} that will read from the given
+ * input stream.
+ *
+ * @param in the stream to read compressed bytes from
+ * @return a stream to read uncompressed bytes from
+ * @throws IOException
+ */
+ @Override
+ public CompressionInputStream createInputStream(InputStream in)
+ throws IOException {
+ return Util.
+ createInputStreamWithCodecPool(this, conf, in);
+ }
+
+ /**
+ * Create a {@link CompressionInputStream} that will read from the given
+ * {@link InputStream} with the given {@link Decompressor}.
+ *
+ * @param in the stream to read compressed bytes from
+ * @param decompressor decompressor to use
+ * @return a stream to read uncompressed bytes from
+ * @throws IOException
+ */
+ @Override
+ public CompressionInputStream createInputStream(InputStream in,
+ Decompressor decompressor)
+ throws IOException {
+ checkNativeCodeLoaded();
+ return new DecompressorStream(in, decompressor,
+ getDecompressionBufferSize(conf));
+ }
+
+ /**
+ * Get the type of {@link Decompressor} needed by
+ * this {@link CompressionCodec}.
+ *
+ * @return the type of decompressor needed by this codec.
+ */
+ @Override
+ public Class<? extends Decompressor> getDecompressorType() {
+ checkNativeCodeLoaded();
+ return ZStandardDecompressor.class;
+ }
+
+ /**
+ * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+ *
+ * @return a new decompressor for use by this codec
+ */
+ @Override
+ public Decompressor createDecompressor() {
+ checkNativeCodeLoaded();
+ return new ZStandardDecompressor(getDecompressionBufferSize(conf));
+ }
+
+ /**
+ * Get the default filename extension for this kind of compression.
+ *
+ * @return <code>.zst</code>.
+ */
+ @Override
+ public String getDefaultExtension() {
+ return ".zst";
+ }
+
+ @Override
+ public DirectDecompressor createDirectDecompressor() {
+ return new ZStandardDecompressor.ZStandardDirectDecompressor(
+ getDecompressionBufferSize(conf)
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
new file mode 100644
index 0000000..eb2121a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress.zstd;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.ZStandardCodec;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * A {@link Compressor} based on the zStandard compression algorithm.
+ * https://github.com/facebook/zstd
+ */
+public class ZStandardCompressor implements Compressor {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ZStandardCompressor.class);
+
+ private long stream;
+ private int level;
+ private int directBufferSize;
+ private byte[] userBuf = null;
+ private int userBufOff = 0, userBufLen = 0;
+ private ByteBuffer uncompressedDirectBuf = null;
+ private int uncompressedDirectBufOff = 0, uncompressedDirectBufLen = 0;
+ private boolean keepUncompressedBuf = false;
+ private ByteBuffer compressedDirectBuf = null;
+ private boolean finish, finished;
+ private long bytesRead = 0;
+ private long bytesWritten = 0;
+
+ private static boolean nativeZStandardLoaded = false;
+
+ static {
+ if (NativeCodeLoader.isNativeCodeLoaded()) {
+ try {
+ // Initialize the native library
+ initIDs();
+ nativeZStandardLoaded = true;
+ } catch (Throwable t) {
+ LOG.warn("Error loading zstandard native libraries: " + t);
+ }
+ }
+ }
+
+ public static boolean isNativeCodeLoaded() {
+ return nativeZStandardLoaded;
+ }
+
+ public static int getRecommendedBufferSize() {
+ return getStreamSize();
+ }
+
+ @VisibleForTesting
+ ZStandardCompressor() {
+ this(CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_LEVEL_DEFAULT,
+ CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
+ }
+
+ /**
+ * Creates a new compressor with the default compression level.
+ * Compressed data will be generated in ZStandard format.
+ */
+ public ZStandardCompressor(int level, int bufferSize) {
+ this(level, bufferSize, bufferSize);
+ }
+
+ @VisibleForTesting
+ ZStandardCompressor(int level, int inputBufferSize, int outputBufferSize) {
+ this.level = level;
+ stream = create();
+ this.directBufferSize = outputBufferSize;
+ uncompressedDirectBuf = ByteBuffer.allocateDirect(inputBufferSize);
+ compressedDirectBuf = ByteBuffer.allocateDirect(outputBufferSize);
+ compressedDirectBuf.position(outputBufferSize);
+ reset();
+ }
+
+ /**
+ * Prepare the compressor to be used in a new stream with settings defined in
+ * the given Configuration. It will reset the compressor's compression level
+ * and compression strategy.
+ *
+ * @param conf Configuration storing new settings
+ */
+ @Override
+ public void reinit(Configuration conf) {
+ if (conf == null) {
+ return;
+ }
+ level = ZStandardCodec.getCompressionLevel(conf);
+ reset();
+ LOG.debug("Reinit compressor with new compression configuration");
+ }
+
+ @Override
+ public void setInput(byte[] b, int off, int len) {
+ if (b == null) {
+ throw new NullPointerException();
+ }
+ if (off < 0 || len < 0 || off > b.length - len) {
+ throw new ArrayIndexOutOfBoundsException();
+ }
+
+ this.userBuf = b;
+ this.userBufOff = off;
+ this.userBufLen = len;
+ uncompressedDirectBufOff = 0;
+ setInputFromSavedData();
+
+ compressedDirectBuf.limit(directBufferSize);
+ compressedDirectBuf.position(directBufferSize);
+ }
+
+ //copy enough data from userBuf to uncompressedDirectBuf
+ private void setInputFromSavedData() {
+ int len = Math.min(userBufLen, uncompressedDirectBuf.remaining());
+ uncompressedDirectBuf.put(userBuf, userBufOff, len);
+ userBufLen -= len;
+ userBufOff += len;
+ uncompressedDirectBufLen = uncompressedDirectBuf.position();
+ }
+
+ @Override
+ public void setDictionary(byte[] b, int off, int len) {
+ throw new UnsupportedOperationException(
+ "Dictionary support is not enabled");
+ }
+
+ @Override
+ public boolean needsInput() {
+ // Consume remaining compressed data?
+ if (compressedDirectBuf.remaining() > 0) {
+ return false;
+ }
+
+ // have we consumed all input
+ if (keepUncompressedBuf && uncompressedDirectBufLen > 0) {
+ return false;
+ }
+
+ if (uncompressedDirectBuf.remaining() > 0) {
+ // Check if we have consumed all user-input
+ if (userBufLen <= 0) {
+ return true;
+ } else {
+ // copy enough data from userBuf to uncompressedDirectBuf
+ setInputFromSavedData();
+ // uncompressedDirectBuf is not full
+ return uncompressedDirectBuf.remaining() > 0;
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public void finish() {
+ finish = true;
+ }
+
+ @Override
+ public boolean finished() {
+ // Check if 'zstd' says its 'finished' and all compressed
+ // data has been consumed
+ return (finished && compressedDirectBuf.remaining() == 0);
+ }
+
+ @Override
+ public int compress(byte[] b, int off, int len) throws IOException {
+ checkStream();
+ if (b == null) {
+ throw new NullPointerException();
+ }
+ if (off < 0 || len < 0 || off > b.length - len) {
+ throw new ArrayIndexOutOfBoundsException();
+ }
+
+ // Check if there is compressed data
+ int n = compressedDirectBuf.remaining();
+ if (n > 0) {
+ n = Math.min(n, len);
+ compressedDirectBuf.get(b, off, n);
+ return n;
+ }
+
+ // Re-initialize the output direct buffer
+ compressedDirectBuf.rewind();
+ compressedDirectBuf.limit(directBufferSize);
+
+ // Compress data
+ n = deflateBytesDirect(
+ uncompressedDirectBuf,
+ uncompressedDirectBufOff,
+ uncompressedDirectBufLen,
+ compressedDirectBuf,
+ directBufferSize
+ );
+ compressedDirectBuf.limit(n);
+
+ // Check if we have consumed all input buffer
+ if (uncompressedDirectBufLen <= 0) {
+ // consumed all input buffer
+ keepUncompressedBuf = false;
+ uncompressedDirectBuf.clear();
+ uncompressedDirectBufOff = 0;
+ uncompressedDirectBufLen = 0;
+ } else {
+ // did not consume all input buffer
+ keepUncompressedBuf = true;
+ }
+
+ // Get at most 'len' bytes
+ n = Math.min(n, len);
+ compressedDirectBuf.get(b, off, n);
+ return n;
+ }
+
+ /**
+ * Returns the total number of compressed bytes output so far.
+ *
+ * @return the total (non-negative) number of compressed bytes output so far
+ */
+ @Override
+ public long getBytesWritten() {
+ checkStream();
+ return bytesWritten;
+ }
+
+ /**
+ * <p>Returns the total number of uncompressed bytes input so far.</p>
+ *
+ * @return the total (non-negative) number of uncompressed bytes input so far
+ */
+ @Override
+ public long getBytesRead() {
+ checkStream();
+ return bytesRead;
+ }
+
+ @Override
+ public void reset() {
+ checkStream();
+ init(level, stream);
+ finish = false;
+ finished = false;
+ bytesRead = 0;
+ bytesWritten = 0;
+ uncompressedDirectBuf.rewind();
+ uncompressedDirectBufOff = 0;
+ uncompressedDirectBufLen = 0;
+ keepUncompressedBuf = false;
+ compressedDirectBuf.limit(directBufferSize);
+ compressedDirectBuf.position(directBufferSize);
+ userBufOff = 0;
+ userBufLen = 0;
+ }
+
+ @Override
+ public void end() {
+ if (stream != 0) {
+ end(stream);
+ stream = 0;
+ }
+ }
+
+ private void checkStream() {
+ if (stream == 0) {
+ throw new NullPointerException();
+ }
+ }
+
+ private native static long create();
+ private native static void init(int level, long stream);
+ private native int deflateBytesDirect(ByteBuffer src, int srcOffset,
+ int srcLen, ByteBuffer dst, int dstLen);
+ private static native int getStreamSize();
+ private native static void end(long strm);
+ private native static void initIDs();
+ public native static String getLibraryName();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
new file mode 100644
index 0000000..73d73e1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress.zstd;
+
+import org.apache.hadoop.io.compress.Decompressor;
+import org.apache.hadoop.io.compress.DirectDecompressor;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * A {@link Decompressor} based on the zStandard compression algorithm.
+ * https://github.com/facebook/zstd
+ */
+public class ZStandardDecompressor implements Decompressor {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ZStandardDecompressor.class);
+
+ private long stream;
+ private int directBufferSize;
+ private ByteBuffer compressedDirectBuf = null;
+ private int compressedDirectBufOff, bytesInCompressedBuffer;
+ private ByteBuffer uncompressedDirectBuf = null;
+ private byte[] userBuf = null;
+ private int userBufOff = 0, userBufferBytesToConsume = 0;
+ private boolean finished;
+ private int remaining = 0;
+
+ private static boolean nativeZStandardLoaded = false;
+
+ static {
+ if (NativeCodeLoader.isNativeCodeLoaded()) {
+ try {
+ // Initialize the native library
+ initIDs();
+ nativeZStandardLoaded = true;
+ } catch (Throwable t) {
+ LOG.warn("Error loading zstandard native libraries: " + t);
+ }
+ }
+ }
+
+ public static boolean isNativeCodeLoaded() {
+ return nativeZStandardLoaded;
+ }
+
+ public static int getRecommendedBufferSize() {
+ return getStreamSize();
+ }
+
+ public ZStandardDecompressor() {
+ this(getStreamSize());
+ }
+
+ /**
+ * Creates a new decompressor.
+ */
+ public ZStandardDecompressor(int bufferSize) {
+ this.directBufferSize = bufferSize;
+ compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+ uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+ uncompressedDirectBuf.position(directBufferSize);
+ stream = create();
+ reset();
+ }
+
+ @Override
+ public void setInput(byte[] b, int off, int len) {
+ if (b == null) {
+ throw new NullPointerException();
+ }
+ if (off < 0 || len < 0 || off > b.length - len) {
+ throw new ArrayIndexOutOfBoundsException();
+ }
+
+ this.userBuf = b;
+ this.userBufOff = off;
+ this.userBufferBytesToConsume = len;
+
+ setInputFromSavedData();
+
+ uncompressedDirectBuf.limit(directBufferSize);
+ uncompressedDirectBuf.position(directBufferSize);
+ }
+
+ private void setInputFromSavedData() {
+ compressedDirectBufOff = 0;
+ bytesInCompressedBuffer = userBufferBytesToConsume;
+ if (bytesInCompressedBuffer > directBufferSize) {
+ bytesInCompressedBuffer = directBufferSize;
+ }
+
+ compressedDirectBuf.rewind();
+ compressedDirectBuf.put(
+ userBuf, userBufOff, bytesInCompressedBuffer);
+
+ userBufOff += bytesInCompressedBuffer;
+ userBufferBytesToConsume -= bytesInCompressedBuffer;
+ }
+
+ // dictionary is not supported
+ @Override
+ public void setDictionary(byte[] b, int off, int len) {
+ throw new UnsupportedOperationException(
+ "Dictionary support is not enabled");
+ }
+
+ @Override
+ public boolean needsInput() {
+ // Consume remaining compressed data?
+ if (uncompressedDirectBuf.remaining() > 0) {
+ return false;
+ }
+
+ // Check if we have consumed all input
+ if (bytesInCompressedBuffer - compressedDirectBufOff <= 0) {
+ // Check if we have consumed all user-input
+ if (userBufferBytesToConsume <= 0) {
+ return true;
+ } else {
+ setInputFromSavedData();
+ }
+ }
+ return false;
+ }
+
+ // dictionary is not supported.
+ @Override
+ public boolean needsDictionary() {
+ return false;
+ }
+
+ @Override
+ public boolean finished() {
+ // finished == true if ZSTD_decompressStream() returns 0
+ // also check we have nothing left in our buffer
+ return (finished && uncompressedDirectBuf.remaining() == 0);
+ }
+
+ @Override
+ public int decompress(byte[] b, int off, int len)
+ throws IOException {
+ checkStream();
+ if (b == null) {
+ throw new NullPointerException();
+ }
+ if (off < 0 || len < 0 || off > b.length - len) {
+ throw new ArrayIndexOutOfBoundsException();
+ }
+
+ // Check if there is uncompressed data
+ int n = uncompressedDirectBuf.remaining();
+ if (n > 0) {
+ return populateUncompressedBuffer(b, off, len, n);
+ }
+
+ // Re-initialize the output direct buffer
+ uncompressedDirectBuf.rewind();
+ uncompressedDirectBuf.limit(directBufferSize);
+
+ // Decompress data
+ n = inflateBytesDirect(
+ compressedDirectBuf,
+ compressedDirectBufOff,
+ bytesInCompressedBuffer,
+ uncompressedDirectBuf,
+ 0,
+ directBufferSize
+ );
+ uncompressedDirectBuf.limit(n);
+
+ // Get at most 'len' bytes
+ return populateUncompressedBuffer(b, off, len, n);
+ }
+
+ /**
+ * <p>Returns the number of bytes remaining in the input buffers;
+ * normally called when finished() is true to determine amount of post-stream
+ * data.</p>
+ *
+ * @return the total (non-negative) number of unprocessed bytes in input
+ */
+ @Override
+ public int getRemaining() {
+ checkStream();
+ // userBuf + compressedDirectBuf
+ return userBufferBytesToConsume + remaining;
+ }
+
+ /**
+ * Resets everything including the input buffers (user and direct).
+ */
+ @Override
+ public void reset() {
+ checkStream();
+ init(stream);
+ remaining = 0;
+ finished = false;
+ compressedDirectBufOff = 0;
+ bytesInCompressedBuffer = 0;
+ uncompressedDirectBuf.limit(directBufferSize);
+ uncompressedDirectBuf.position(directBufferSize);
+ userBufOff = 0;
+ userBufferBytesToConsume = 0;
+ }
+
+ @Override
+ public void end() {
+ if (stream != 0) {
+ free(stream);
+ stream = 0;
+ }
+ }
+
+ @Override
+ protected void finalize() {
+ reset();
+ }
+
+ private void checkStream() {
+ if (stream == 0) {
+ throw new NullPointerException("Stream not initialized");
+ }
+ }
+
+ private int populateUncompressedBuffer(byte[] b, int off, int len, int n) {
+ n = Math.min(n, len);
+ uncompressedDirectBuf.get(b, off, n);
+ return n;
+ }
+
+ private native static void initIDs();
+ private native static long create();
+ private native static void init(long stream);
+ private native int inflateBytesDirect(ByteBuffer src, int srcOffset,
+ int srcLen, ByteBuffer dst, int dstOffset, int dstLen);
+ private native static void free(long strm);
+ private native static int getStreamSize();
+
+ int inflateDirect(ByteBuffer src, ByteBuffer dst) throws IOException {
+ assert
+ (this instanceof ZStandardDecompressor.ZStandardDirectDecompressor);
+
+ int originalPosition = dst.position();
+ int n = inflateBytesDirect(
+ src, src.position(), src.remaining(), dst, dst.position(),
+ dst.remaining()
+ );
+ dst.position(originalPosition + n);
+ if (bytesInCompressedBuffer > 0) {
+ src.position(compressedDirectBufOff);
+ } else {
+ src.position(src.limit());
+ }
+ return n;
+ }
+
+ /**
+ * A {@link DirectDecompressor} for ZStandard
+ * https://github.com/facebook/zstd.
+ */
+ public static class ZStandardDirectDecompressor
+ extends ZStandardDecompressor implements DirectDecompressor {
+
+ public ZStandardDirectDecompressor(int directBufferSize) {
+ super(directBufferSize);
+ }
+
+ @Override
+ public boolean finished() {
+ return (endOfInput && super.finished());
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ endOfInput = true;
+ }
+
+ private boolean endOfInput;
+
+ @Override
+ public void decompress(ByteBuffer src, ByteBuffer dst)
+ throws IOException {
+ assert dst.isDirect() : "dst.isDirect()";
+ assert src.isDirect() : "src.isDirect()";
+ assert dst.remaining() > 0 : "dst.remaining() > 0";
+ this.inflateDirect(src, dst);
+ endOfInput = !src.hasRemaining();
+ }
+
+ @Override
+ public void setDictionary(byte[] b, int off, int len) {
+ throw new UnsupportedOperationException(
+ "byte[] arrays are not supported for DirectDecompressor");
+ }
+
+ @Override
+ public int decompress(byte[] b, int off, int len) {
+ throw new UnsupportedOperationException(
+ "byte[] arrays are not supported for DirectDecompressor");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/package-info.java
new file mode 100644
index 0000000..9069070
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.io.compress.zstd;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
index dd04a19..c381336 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
@@ -85,6 +85,11 @@ public final class NativeCodeLoader {
public static native boolean buildSupportsIsal();
/**
+ * Returns true only if this build was compiled with support for ZStandard.
+ */
+ public static native boolean buildSupportsZstd();
+
+ /**
* Returns true only if this build was compiled with support for openssl.
*/
public static native boolean buildSupportsOpenssl();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
index e166bec..776839c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.util;
+import org.apache.hadoop.io.compress.ZStandardCodec;
import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.OpensslCipher;
@@ -67,6 +68,7 @@ public class NativeLibraryChecker {
boolean zlibLoaded = false;
boolean snappyLoaded = false;
boolean isalLoaded = false;
+ boolean zStdLoaded = false;
// lz4 is linked within libhadoop
boolean lz4Loaded = nativeHadoopLoaded;
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
@@ -78,6 +80,7 @@ public class NativeLibraryChecker {
String zlibLibraryName = "";
String snappyLibraryName = "";
String isalDetail = "";
+ String zstdLibraryName = "";
String lz4LibraryName = "";
String bzip2LibraryName = "";
String winutilsPath = null;
@@ -88,7 +91,11 @@ public class NativeLibraryChecker {
if (zlibLoaded) {
zlibLibraryName = ZlibFactory.getLibraryName();
}
-
+ zStdLoaded = NativeCodeLoader.buildSupportsZstd() &&
+ ZStandardCodec.isNativeCodeLoaded();
+ if (zStdLoaded && NativeCodeLoader.buildSupportsZstd()) {
+ zstdLibraryName = ZStandardCodec.getLibraryName();
+ }
snappyLoaded = NativeCodeLoader.buildSupportsSnappy() &&
SnappyCodec.isNativeCodeLoaded();
if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
@@ -135,6 +142,7 @@ public class NativeLibraryChecker {
System.out.println("Native library checking:");
System.out.printf("hadoop: %b %s%n", nativeHadoopLoaded, hadoopLibraryName);
System.out.printf("zlib: %b %s%n", zlibLoaded, zlibLibraryName);
+ System.out.printf("zstd : %b %s%n", zStdLoaded, zstdLibraryName);
System.out.printf("snappy: %b %s%n", snappyLoaded, snappyLibraryName);
System.out.printf("lz4: %b %s%n", lz4Loaded, lz4LibraryName);
System.out.printf("bzip2: %b %s%n", bzip2Loaded, bzip2LibraryName);
@@ -146,7 +154,8 @@ public class NativeLibraryChecker {
}
if ((!nativeHadoopLoaded) || (Shell.WINDOWS && (!winutilsExists)) ||
- (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded && isalLoaded))) {
+ (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded
+ && bzip2Loaded && isalLoaded && zStdLoaded))) {
// return 1 to indicated check failed
ExitUtil.terminate(1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
new file mode 100644
index 0000000..04f2a3e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include "org_apache_hadoop_io_compress_zstd.h"
+
+#if defined HADOOP_ZSTD_LIBRARY
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef UNIX
+#include <dlfcn.h>
+#include "config.h"
+#endif
+
+#include "org_apache_hadoop_io_compress_zstd_ZStandardCompressor.h"
+
+static jfieldID ZStandardCompressor_stream;
+static jfieldID ZStandardCompressor_uncompressedDirectBufOff;
+static jfieldID ZStandardCompressor_uncompressedDirectBufLen;
+static jfieldID ZStandardCompressor_directBufferSize;
+static jfieldID ZStandardCompressor_finish;
+static jfieldID ZStandardCompressor_finished;
+static jfieldID ZStandardCompressor_bytesWritten;
+static jfieldID ZStandardCompressor_bytesRead;
+
+#ifdef UNIX
+static size_t (*dlsym_ZSTD_CStreamInSize)(void);
+static size_t (*dlsym_ZSTD_CStreamOutSize)(void);
+static ZSTD_CStream* (*dlsym_ZSTD_createCStream)(void);
+static size_t (*dlsym_ZSTD_initCStream)(ZSTD_CStream*, int);
+static size_t (*dlsym_ZSTD_freeCStream)(ZSTD_CStream*);
+static size_t (*dlsym_ZSTD_compressStream)(ZSTD_CStream*, ZSTD_outBuffer*, ZSTD_inBuffer*);
+static size_t (*dlsym_ZSTD_endStream)(ZSTD_CStream*, ZSTD_outBuffer*);
+static size_t (*dlsym_ZSTD_flushStream)(ZSTD_CStream*, ZSTD_outBuffer*);
+static unsigned (*dlsym_ZSTD_isError)(size_t);
+static const char * (*dlsym_ZSTD_getErrorName)(size_t);
+#endif
+
+#ifdef WINDOWS
+typedef size_t (__cdecl *__dlsym_ZSTD_CStreamInSize)(void);
+typedef size_t (__cdecl *__dlsym_ZSTD_CStreamOutSize)(void);
+typedef ZSTD_CStream* (__cdecl *__dlsym_ZSTD_createCStream)(void);
+typedef size_t (__cdecl *__dlsym_ZSTD_initCStream)(ZSTD_CStream*, int);
+typedef size_t (__cdecl *__dlsym_ZSTD_freeCStream)(ZSTD_CStream*);
+typedef size_t (__cdecl *__dlsym_ZSTD_compressStream)(ZSTD_CStream*, ZSTD_outBuffer*, ZSTD_inBuffer*);
+typedef size_t (__cdecl *__dlsym_ZSTD_endStream)(ZSTD_CStream*, ZSTD_outBuffer*);
+typedef size_t (__cdecl *__dlsym_ZSTD_flushStream)(ZSTD_CStream*, ZSTD_outBuffer*);
+typedef unsigned (__cdecl *__dlsym_ZSTD_isError)(size_t);
+typedef const char * (__cdecl *__dlsym_ZSTD_getErrorName)(size_t);
+
+static __dlsym_ZSTD_CStreamInSize dlsym_ZSTD_CStreamInSize;
+static __dlsym_ZSTD_CStreamOutSize dlsym_ZSTD_CStreamOutSize;
+static __dlsym_ZSTD_createCStream dlsym_ZSTD_createCStream;
+static __dlsym_ZSTD_initCStream dlsym_ZSTD_initCStream;
+static __dlsym_ZSTD_freeCStream dlsym_ZSTD_freeCStream;
+static __dlsym_ZSTD_compressStream dlsym_ZSTD_compressStream;
+static __dlsym_ZSTD_endStream dlsym_ZSTD_endStream;
+static __dlsym_ZSTD_flushStream dlsym_ZSTD_flushStream;
+static __dlsym_ZSTD_isError dlsym_ZSTD_isError;
+static __dlsym_ZSTD_getErrorName dlsym_ZSTD_getErrorName;
+#endif
+
+// Load the libztsd.so from disk
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_initIDs (JNIEnv *env, jclass clazz) {
+#ifdef UNIX
+ // Load libzstd.so
+ void *libzstd = dlopen(HADOOP_ZSTD_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
+ if (!libzstd) {
+ char* msg = (char*)malloc(10000);
+ snprintf(msg, 10000, "%s (%s)!", "Cannot load " HADOOP_ZSTD_LIBRARY, dlerror());
+ THROW(env, "java/lang/InternalError", msg);
+ return;
+ }
+#endif
+
+#ifdef WINDOWS
+ HMODULE libzstd = LoadLibrary(HADOOP_ZSTD_LIBRARY);
+ if (!libzstd) {
+ THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load zstd.dll");
+ return;
+ }
+#endif
+
+#ifdef UNIX
+ // load dynamic symbols
+ dlerror();
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_CStreamInSize, env, libzstd, "ZSTD_CStreamInSize");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_CStreamOutSize, env, libzstd, "ZSTD_CStreamOutSize");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_createCStream, env, libzstd, "ZSTD_createCStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_initCStream, env, libzstd, "ZSTD_initCStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_freeCStream, env, libzstd, "ZSTD_freeCStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_compressStream, env, libzstd, "ZSTD_compressStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_endStream, env, libzstd, "ZSTD_endStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_flushStream, env, libzstd, "ZSTD_flushStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_isError, env, libzstd, "ZSTD_isError");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_getErrorName, env, libzstd, "ZSTD_getErrorName");
+#endif
+
+#ifdef WINDOWS
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_CStreamInSize, dlsym_ZSTD_CStreamInSize, env, libzstd, "ZSTD_CStreamInSize");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_CStreamOutSize, dlsym_ZSTD_CStreamOutSize, env, libzstd, "ZSTD_CStreamOutSize");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_createCStream, dlsym_ZSTD_createCStream, env, libzstd, "ZSTD_createCStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_initCStream, dlsym_ZSTD_initCStream, env, libzstd, "ZSTD_initCStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_freeCStream, dlsym_ZSTD_freeCStream, env, libzstd, "ZSTD_freeCStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_compressStream, dlsym_ZSTD_compressStream, env, libzstd, "ZSTD_compressStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_endStream, dlsym_ZSTD_endStream, env, libzstd, "ZSTD_endStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_flushStream, dlsym_ZSTD_flushStream, env, libzstd, "ZSTD_flushStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_isError, dlsym_ZSTD_isError, env, libzstd, "ZSTD_isError");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_getErrorName, dlsym_ZSTD_getErrorName, env, libzstd, "ZSTD_getErrorName");
+#endif
+
+ // load fields
+ ZStandardCompressor_stream = (*env)->GetFieldID(env, clazz, "stream", "J");
+ ZStandardCompressor_finish = (*env)->GetFieldID(env, clazz, "finish", "Z");
+ ZStandardCompressor_finished = (*env)->GetFieldID(env, clazz, "finished", "Z");
+ ZStandardCompressor_uncompressedDirectBufOff = (*env)->GetFieldID(env, clazz, "uncompressedDirectBufOff", "I");
+ ZStandardCompressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, clazz, "uncompressedDirectBufLen", "I");
+ ZStandardCompressor_directBufferSize = (*env)->GetFieldID(env, clazz, "directBufferSize", "I");
+ ZStandardCompressor_bytesRead = (*env)->GetFieldID(env, clazz, "bytesRead", "J");
+ ZStandardCompressor_bytesWritten = (*env)->GetFieldID(env, clazz, "bytesWritten", "J");
+}
+
+// Create the compression stream
+JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_create (JNIEnv *env, jobject this) {
+ ZSTD_CStream* const stream = dlsym_ZSTD_createCStream();
+ if (stream == NULL) {
+ THROW(env, "java/lang/InternalError", "Error creating the stream");
+ return (jlong)0;
+ }
+ return (jlong) stream;
+}
+
+// Initialize the compression stream
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_init (JNIEnv *env, jobject this, jint level, jlong stream) {
+ size_t result = dlsym_ZSTD_initCStream((ZSTD_CStream *) stream, level);
+ if (dlsym_ZSTD_isError(result)) {
+ THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(result));
+ return;
+ }
+}
+
+// free the compression stream
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_end (JNIEnv *env, jobject this, jlong stream) {
+ size_t result = dlsym_ZSTD_freeCStream((ZSTD_CStream *) stream);
+ if (dlsym_ZSTD_isError(result)) {
+ THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(result));
+ return;
+ }
+}
+
+JNIEXPORT jint Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_deflateBytesDirect
+(JNIEnv *env, jobject this, jobject uncompressed_direct_buf, jint uncompressed_direct_buf_off, jint uncompressed_direct_buf_len, jobject compressed_direct_buf, jint compressed_direct_buf_len ) {
+ ZSTD_CStream* const stream = (ZSTD_CStream*) (*env)->GetLongField(env, this, ZStandardCompressor_stream);
+ if (!stream) {
+ THROW(env, "java/lang/NullPointerException", NULL);
+ return (jint)0;
+ }
+
+ jlong bytes_read = (*env)->GetLongField(env, this, ZStandardCompressor_bytesRead);
+ jlong bytes_written = (*env)->GetLongField(env, this, ZStandardCompressor_bytesWritten);
+ jboolean finish = (*env)->GetBooleanField(env, this, ZStandardCompressor_finish);
+
+ // Get the input direct buffer
+ void * uncompressed_bytes = (*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+ if (!uncompressed_bytes) {
+ THROW(env, "java/lang/InternalError", "Undefined memory address for uncompressedDirectBuf");
+ return (jint) 0;
+ }
+
+ // Get the output direct buffer
+ void * compressed_bytes = (*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+ if (!compressed_bytes) {
+ THROW(env, "java/lang/InternalError", "Undefined memory address for compressedDirectBuf");
+ return (jint) 0;
+ }
+
+ ZSTD_inBuffer input = { uncompressed_bytes, uncompressed_direct_buf_len, uncompressed_direct_buf_off };
+ ZSTD_outBuffer output = { compressed_bytes, compressed_direct_buf_len, 0 };
+
+ size_t size = dlsym_ZSTD_compressStream(stream, &output, &input);
+ if (dlsym_ZSTD_isError(size)) {
+ THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size));
+ return (jint) 0;
+ }
+ if (finish && input.pos == input.size) {
+ // end the stream, flush and write the frame epilogue
+ size = dlsym_ZSTD_endStream(stream, &output);
+ if (!size) {
+ (*env)->SetBooleanField(env, this, ZStandardCompressor_finished, JNI_TRUE);
+ }
+ } else {
+ // need to flush the output buffer
+ // this also updates the output buffer position.
+ size = dlsym_ZSTD_flushStream(stream, &output);
+ }
+ if (dlsym_ZSTD_isError(size)) {
+ THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size));
+ return (jint) 0;
+ }
+
+ bytes_read += input.pos;
+ bytes_written += output.pos;
+ (*env)->SetLongField(env, this, ZStandardCompressor_bytesRead, bytes_read);
+ (*env)->SetLongField(env, this, ZStandardCompressor_bytesWritten, bytes_written);
+
+ (*env)->SetIntField(env, this, ZStandardCompressor_uncompressedDirectBufOff, input.pos);
+ (*env)->SetIntField(env, this, ZStandardCompressor_uncompressedDirectBufLen, input.size - input.pos);
+ return (jint) output.pos;
+}
+
+JNIEXPORT jstring JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_getLibraryName
+(JNIEnv *env, jclass class) {
+#ifdef UNIX
+ if (dlsym_ZSTD_isError) {
+ Dl_info dl_info;
+ if (dladdr( dlsym_ZSTD_isError, &dl_info)) {
+ return (*env)->NewStringUTF(env, dl_info.dli_fname);
+ }
+ }
+ return (*env)->NewStringUTF(env, HADOOP_ZSTD_LIBRARY);
+#endif
+#ifdef WINDOWS
+ LPWSTR filename = NULL;
+ GetLibraryName(dlsym_ZSTD_isError, &filename);
+ if (filename != NULL) {
+ return (*env)->NewString(env, filename, (jsize) wcslen(filename));
+ } else {
+ return (*env)->NewStringUTF(env, "Unavailable");
+ }
+#endif
+}
+
+// returns the max size of the recommended input and output buffers
+JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_getStreamSize
+(JNIEnv *env, jobject this) {
+ int x = (int) dlsym_ZSTD_CStreamInSize();
+ int y = (int) dlsym_ZSTD_CStreamOutSize();
+ return (x >= y) ? x : y;
+}
+
+#endif //define HADOOP_ZSTD_LIBRARY
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
new file mode 100644
index 0000000..1236756
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "org_apache_hadoop_io_compress_zstd.h"
+
+#if defined HADOOP_ZSTD_LIBRARY
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef UNIX
+#include <dlfcn.h>
+#include "config.h"
+#endif
+
+#include "org_apache_hadoop_io_compress_zstd_ZStandardDecompressor.h"
+
+static jfieldID ZStandardDecompressor_stream;
+static jfieldID ZStandardDecompressor_compressedDirectBufOff;
+static jfieldID ZStandardDecompressor_bytesInCompressedBuffer;
+static jfieldID ZStandardDecompressor_directBufferSize;
+static jfieldID ZStandardDecompressor_finished;
+static jfieldID ZStandardDecompressor_remaining;
+
+#ifdef UNIX
+static size_t (*dlsym_ZSTD_DStreamOutSize)(void);
+static size_t (*dlsym_ZSTD_DStreamInSize)(void);
+static ZSTD_DStream* (*dlsym_ZSTD_createDStream)(void);
+static size_t (*dlsym_ZSTD_initDStream)(ZSTD_DStream*);
+static size_t (*dlsym_ZSTD_freeDStream)(ZSTD_DStream*);
+static size_t (*dlsym_ZSTD_resetDStream)(ZSTD_DStream*);
+static size_t (*dlsym_ZSTD_decompressStream)(ZSTD_DStream*, ZSTD_outBuffer*, ZSTD_inBuffer*);
+static size_t (*dlsym_ZSTD_flushStream)(ZSTD_CStream*, ZSTD_outBuffer*);
+static unsigned (*dlsym_ZSTD_isError)(size_t);
+static const char * (*dlsym_ZSTD_getErrorName)(size_t);
+#endif
+
+#ifdef WINDOWS
+typedef size_t (__cdecl *__dlsym_ZSTD_DStreamOutSize)(void);
+typedef size_t (__cdecl *__dlsym_ZSTD_DStreamInSize)(void);
+typedef ZSTD_DStream* (__cdecl *__dlsym_ZSTD_createDStream)(void);
+typedef size_t (__cdecl *__dlsym_ZSTD_initDStream)(ZSTD_DStream*);
+typedef size_t (__cdecl *__dlsym_ZSTD_freeDStream)(ZSTD_DStream*);
+typedef size_t (__cdecl *__dlsym_ZSTD_resetDStream)(ZSTD_DStream*);
+typedef size_t (__cdecl *__dlsym_ZSTD_decompressStream)(ZSTD_DStream*, ZSTD_outBuffer*, ZSTD_inBuffer*);
+typedef size_t (__cdecl *__dlsym_ZSTD_flushStream)(ZSTD_CStream*, ZSTD_outBuffer*);
+typedef unsigned (__cdecl *__dlsym_ZSTD_isError)(size_t);
+typedef const char * (__cdecl *__dlsym_ZSTD_getErrorName)(size_t);
+
+static __dlsym_ZSTD_DStreamOutSize dlsym_ZSTD_DStreamOutSize;
+static __dlsym_ZSTD_DStreamInSize dlsym_ZSTD_DStreamInSize;
+static __dlsym_ZSTD_createDStream dlsym_ZSTD_createDStream;
+static __dlsym_ZSTD_initDStream dlsym_ZSTD_initDStream;
+static __dlsym_ZSTD_freeDStream dlsym_ZSTD_freeDStream;
+static __dlsym_ZSTD_resetDStream dlsym_ZSTD_resetDStream;
+static __dlsym_ZSTD_decompressStream dlsym_ZSTD_decompressStream;
+static __dlsym_ZSTD_isError dlsym_ZSTD_isError;
+static __dlsym_ZSTD_getErrorName dlsym_ZSTD_getErrorName;
+static __dlsym_ZSTD_flushStream dlsym_ZSTD_flushStream;
+#endif
+
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_initIDs (JNIEnv *env, jclass clazz) {
+ // Load libzstd.so
+#ifdef UNIX
+ void *libzstd = dlopen(HADOOP_ZSTD_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
+ if (!libzstd) {
+ char* msg = (char*)malloc(1000);
+ snprintf(msg, 1000, "%s (%s)!", "Cannot load " HADOOP_ZSTD_LIBRARY, dlerror());
+ THROW(env, "java/lang/UnsatisfiedLinkError", msg);
+ return;
+ }
+#endif
+
+#ifdef WINDOWS
+ HMODULE libzstd = LoadLibrary(HADOOP_ZSTD_LIBRARY);
+ if (!libzstd) {
+ THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load zstd.dll");
+ return;
+ }
+#endif
+
+#ifdef UNIX
+ dlerror();
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_DStreamOutSize, env, libzstd, "ZSTD_DStreamOutSize");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_DStreamInSize, env, libzstd, "ZSTD_DStreamInSize");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_createDStream, env, libzstd, "ZSTD_createDStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_initDStream, env, libzstd, "ZSTD_initDStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_freeDStream, env, libzstd, "ZSTD_freeDStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_resetDStream, env, libzstd, "ZSTD_resetDStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_decompressStream, env, libzstd, "ZSTD_decompressStream");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_isError, env, libzstd, "ZSTD_isError");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_getErrorName, env, libzstd, "ZSTD_getErrorName");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ZSTD_flushStream, env, libzstd, "ZSTD_flushStream");
+#endif
+
+#ifdef WINDOWS
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_DStreamOutSize, dlsym_ZSTD_DStreamOutSize, env, libzstd, "ZSTD_DStreamOutSize");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_DStreamInSize, dlsym_ZSTD_DStreamInSize, env, libzstd, "ZSTD_DStreamInSize");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_createDStream, dlsym_ZSTD_createDStream, env, libzstd, "ZSTD_createDStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_initDStream, dlsym_ZSTD_initDStream, env, libzstd, "ZSTD_initDStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_freeDStream, dlsym_ZSTD_freeDStream, env, libzstd, "ZSTD_freeDStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_resetDStream, dlsym_ZSTD_resetDStream, env, libzstd, "ZSTD_resetDStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_decompressStream, dlsym_ZSTD_decompressStream, env, libzstd, "ZSTD_decompressStream");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_isError, dlsym_ZSTD_isError, env, libzstd, "ZSTD_isError");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_getErrorName, dlsym_ZSTD_getErrorName, env, libzstd, "ZSTD_getErrorName");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ZSTD_flushStream, dlsym_ZSTD_flushStream, env, libzstd, "ZSTD_flushStream");
+#endif
+
+ ZStandardDecompressor_stream = (*env)->GetFieldID(env, clazz, "stream", "J");
+ ZStandardDecompressor_finished = (*env)->GetFieldID(env, clazz, "finished", "Z");
+ ZStandardDecompressor_compressedDirectBufOff = (*env)->GetFieldID(env, clazz, "compressedDirectBufOff", "I");
+ ZStandardDecompressor_bytesInCompressedBuffer = (*env)->GetFieldID(env, clazz, "bytesInCompressedBuffer", "I");
+ ZStandardDecompressor_directBufferSize = (*env)->GetFieldID(env, clazz, "directBufferSize", "I");
+ ZStandardDecompressor_remaining = (*env)->GetFieldID(env, clazz, "remaining", "I");
+}
+
+JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_create(JNIEnv *env, jobject this) {
+ ZSTD_DStream * stream = dlsym_ZSTD_createDStream();
+ if (stream == NULL) {
+ THROW(env, "java/lang/InternalError", "Error creating stream");
+ return (jlong) 0;
+ }
+ return (jlong) stream;
+}
+
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_init(JNIEnv *env, jobject this, jlong stream) {
+ size_t result = dlsym_ZSTD_initDStream((ZSTD_DStream *) stream);
+ if (dlsym_ZSTD_isError(result)) {
+ THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(result));
+ return;
+ }
+ (*env)->SetLongField(env, this, ZStandardDecompressor_remaining, 0);
+}
+
+
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_free(JNIEnv *env, jclass obj, jlong stream) {
+ size_t result = dlsym_ZSTD_freeDStream((ZSTD_DStream *) stream);
+ if (dlsym_ZSTD_isError(result)) {
+ THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(result));
+ return;
+ }
+}
+
+JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_inflateBytesDirect
+(JNIEnv *env, jobject this, jobject compressed_direct_buf, jint compressed_direct_buf_off, jint compressed_direct_buf_len, jobject uncompressed_direct_buf, jint uncompressed_direct_buf_off, jint uncompressed_direct_buf_len) {
+ ZSTD_DStream *stream = (ZSTD_DStream *) (*env)->GetLongField(env, this, ZStandardDecompressor_stream);
+ if (!stream) {
+ THROW(env, "java/lang/NullPointerException", NULL);
+ return (jint)0;
+ }
+
+ // Get the input direct buffer
+ void * compressed_bytes = (*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+ if (!compressed_bytes) {
+ THROW(env, "java/lang/InternalError", "Undefined memory address for compressedDirectBuf");
+ return (jint) 0;
+ }
+
+ // Get the output direct buffer
+ void * uncompressed_bytes = (*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+ if (!uncompressed_bytes) {
+ THROW(env, "java/lang/InternalError", "Undefined memory address for uncompressedDirectBuf");
+ return (jint) 0;
+ }
+ uncompressed_bytes = ((char*) uncompressed_bytes) + uncompressed_direct_buf_off;
+
+ ZSTD_inBuffer input = { compressed_bytes, compressed_direct_buf_len, compressed_direct_buf_off };
+ ZSTD_outBuffer output = { uncompressed_bytes, uncompressed_direct_buf_len, 0 };
+
+ size_t const size = dlsym_ZSTD_decompressStream(stream, &output, &input);
+
+ // check for errors
+ if (dlsym_ZSTD_isError(size)) {
+ THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size));
+ return (jint) 0;
+ }
+ int remaining = input.size - input.pos;
+ (*env)->SetIntField(env, this, ZStandardDecompressor_remaining, remaining);
+
+ // the entire frame has been decoded
+ if (size == 0) {
+ (*env)->SetBooleanField(env, this, ZStandardDecompressor_finished, JNI_TRUE);
+ size_t result = dlsym_ZSTD_initDStream(stream);
+ if (dlsym_ZSTD_isError(result)) {
+ THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(result));
+ return (jint) 0;
+ }
+ }
+ (*env)->SetIntField(env, this, ZStandardDecompressor_compressedDirectBufOff, input.pos);
+ (*env)->SetIntField(env, this, ZStandardDecompressor_bytesInCompressedBuffer, input.size);
+ return (jint) output.pos;
+}
+
+// returns the max size of the recommended input and output buffers
+JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_getStreamSize
+(JNIEnv *env, jclass obj) {
+ int x = (int) dlsym_ZSTD_DStreamInSize();
+ int y = (int) dlsym_ZSTD_DStreamOutSize();
+ return (x >= y) ? x : y;
+}
+
+#endif //define HADOOP_ZSTD_LIBRARY
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/org_apache_hadoop_io_compress_zstd.h
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/org_apache_hadoop_io_compress_zstd.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/org_apache_hadoop_io_compress_zstd.h
new file mode 100644
index 0000000..78fc0a4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/org_apache_hadoop_io_compress_zstd.h
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#ifndef ORG_APACHE_HADOOP_IO_COMPRESS_ZSTD_ZSTD_H
+#define ORG_APACHE_HADOOP_IO_COMPRESS_ZSTD_ZSTD_H
+
+#include "org_apache_hadoop.h"
+
+#ifdef UNIX
+#include <dlfcn.h>
+#endif
+
+#include <jni.h>
+#include <zstd.h>
+#include <stddef.h>
+
+
+#endif //ORG_APACHE_HADOOP_IO_COMPRESS_ZSTD_ZSTD_H
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
index ae8263a..1bd7fa1 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
@@ -39,6 +39,17 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup
#endif
}
+JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsZstd
+ (JNIEnv *env, jclass clazz)
+{
+#ifdef HADOOP_ZSTD_LIBRARY
+ return JNI_TRUE;
+#else
+ return JNI_FALSE;
+#endif
+}
+
+
JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsOpenssl
(JNIEnv *env, jclass clazz)
{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
index df46e32..99b6fb2 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
@@ -17,4 +17,5 @@ org.apache.hadoop.io.compress.DeflateCodec
org.apache.hadoop.io.compress.GzipCodec
org.apache.hadoop.io.compress.Lz4Codec
org.apache.hadoop.io.compress.SnappyCodec
+org.apache.hadoop.io.compress.ZStandardCodec
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm
index 04ff426..e4f720c 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm
@@ -118,6 +118,7 @@ NativeLibraryChecker is a tool to check whether native libraries are loaded corr
hadoop: true /home/ozawa/hadoop/lib/native/libhadoop.so.1.0.0
zlib: true /lib/x86_64-linux-gnu/libz.so.1
snappy: true /usr/lib/libsnappy.so.1
+ zstd: true /usr/lib/libzstd.so.1
lz4: true revision:99
bzip2: false
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
index 5443ca7..3955aa2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
@@ -505,6 +505,18 @@ public class TestCodec {
}
@Test(timeout=20000)
+ public void testSequenceFileZStandardCodec() throws Exception {
+ assumeTrue(ZStandardCodec.isNativeCodeLoaded());
+ Configuration conf = new Configuration();
+ sequenceFileCodecTest(conf, 0,
+ "org.apache.hadoop.io.compress.ZStandardCodec", 100);
+ sequenceFileCodecTest(conf, 100,
+ "org.apache.hadoop.io.compress.ZStandardCodec", 100);
+ sequenceFileCodecTest(conf, 200000,
+ "org.apache.hadoop.io.compress.ZStandardCodec", 1000000);
+ }
+
+ @Test(timeout=20000)
public void testSequenceFileBZip2NativeCodec() throws IOException,
ClassNotFoundException, InstantiationException,
IllegalAccessException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0a27616/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java
index 2d75a2d..dd7bdd2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
public class TestCompressionStreamReuse {
private static final Log LOG = LogFactory
@@ -59,6 +60,13 @@ public class TestCompressionStreamReuse {
}
@Test
+ public void testZStandardCompressStreamReuse() throws IOException {
+ assumeTrue(ZStandardCodec.isNativeCodeLoaded());
+ resetStateTest(conf, seed, count,
+ "org.apache.hadoop.io.compress.ZStandardCodec");
+ }
+
+ @Test
public void testGzipCompressStreamReuseWithParam() throws IOException {
Configuration conf = new Configuration(this.conf);
ZlibFactory
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-6040. Introduce api
independent PendingAsk to replace usage of ResourceRequest within Scheduler
classes. (Wangda Tan via asuresh)
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
index 3cf5fa2..3e0620e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import java.util.Collection;
@@ -70,22 +72,38 @@ public interface SchedulingPlacementSet<N extends SchedulerNode> {
Map<String, ResourceRequest> getResourceRequests();
/**
- * Get ResourceRequest by given schedulerKey and resourceName
+ * Get pending ask for given resourceName. If there's no such pendingAsk,
+ * returns {@link PendingAsk#ZERO}
+ *
+ * @param resourceName resourceName
+ * @return PendingAsk
+ */
+ PendingAsk getPendingAsk(String resourceName);
+
+ /**
+ * Get #pending-allocations for given resourceName. If there's no such
+ * pendingAsk, returns 0
+ *
* @param resourceName resourceName
- * @return ResourceRequest
+ * @return #pending-allocations
*/
- ResourceRequest getResourceRequest(String resourceName);
+ int getOutstandingAsksCount(String resourceName);
/**
* Notify container allocated.
* @param schedulerKey SchedulerRequestKey for this ResourceRequest
* @param type Type of the allocation
* @param node Which node this container allocated on
- * @param request Which resource request to allocate
* @return list of ResourceRequests deducted
*/
List<ResourceRequest> allocate(SchedulerRequestKey schedulerKey,
- NodeType type, SchedulerNode node, ResourceRequest request);
+ NodeType type, SchedulerNode node);
+
+ /**
+ * Returns list of accepted resourceNames.
+ * @return Iterator of accepted resourceNames
+ */
+ Iterator<String> getAcceptedResouceNames();
/**
* We can still have pending requirement for a given NodeType and node
@@ -94,4 +112,47 @@ public interface SchedulingPlacementSet<N extends SchedulerNode> {
* @return true if we has pending requirement
*/
boolean canAllocate(NodeType type, SchedulerNode node);
+
+ /**
+ * Can delay to give locality?
+ * TODO (wangda): This should be moved out of SchedulingPlacementSet
+ * and should belong to specific delay scheduling policy impl.
+ *
+ * @param resourceName resourceName
+ * @return can/cannot
+ */
+ boolean canDelayTo(String resourceName);
+
+ /**
+ * Does this {@link SchedulingPlacementSet} accept resources on nodePartition?
+ *
+ * @param nodePartition nodePartition
+ * @param schedulingMode schedulingMode
+ * @return accepted/not
+ */
+ boolean acceptNodePartition(String nodePartition,
+ SchedulingMode schedulingMode);
+
+ /**
+ * It is possible that one request can accept multiple node partition,
+ * So this method returns primary node partition for pending resource /
+ * headroom calculation.
+ *
+ * @return primary requested node partition
+ */
+ String getPrimaryRequestedNodePartition();
+
+ /**
+ * @return number of unique location asks with #pending greater than 0,
+ * (like /rack1, host1, etc.).
+ *
+ * TODO (wangda): This should be moved out of SchedulingPlacementSet
+ * and should belong to specific delay scheduling policy impl.
+ */
+ int getUniqueLocationAsks();
+
+ /**
+ * Print human-readable requests to LOG debug.
+ */
+ void showRequests();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index 6395339..b32f138 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
@@ -588,12 +589,14 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
// The core part of this test
// The killed containers' ResourceRequests are recovered back to the
// original app-attempt, not the new one
- for (ResourceRequest request : firstSchedulerAppAttempt
- .getAppSchedulingInfo().getAllResourceRequests()) {
- if (request.getPriority().getPriority() == 0) {
- Assert.assertEquals(0, request.getNumContainers());
- } else if (request.getPriority().getPriority() == ALLOCATED_CONTAINER_PRIORITY) {
- Assert.assertEquals(1, request.getNumContainers());
+ for (SchedulerRequestKey key : firstSchedulerAppAttempt.getSchedulerKeys()) {
+ if (key.getPriority().getPriority() == 0) {
+ Assert.assertEquals(0,
+ firstSchedulerAppAttempt.getOutstandingAsksCount(key));
+ } else if (key.getPriority().getPriority() ==
+ ALLOCATED_CONTAINER_PRIORITY) {
+ Assert.assertEquals(1,
+ firstSchedulerAppAttempt.getOutstandingAsksCount(key));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
index 468e760..bb29889 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
@@ -141,7 +141,7 @@ public class TestAppSchedulingInfo {
// iterate to verify no ConcurrentModificationException
for (SchedulerRequestKey schedulerKey : info.getSchedulerKeys()) {
- info.allocate(NodeType.OFF_SWITCH, null, schedulerKey, req1, null);
+ info.allocate(NodeType.OFF_SWITCH, null, schedulerKey, null);
}
Assert.assertEquals(1, info.getSchedulerKeys().size());
Assert.assertEquals(SchedulerRequestKey.create(req2),
@@ -153,7 +153,7 @@ public class TestAppSchedulingInfo {
reqs.add(req2);
info.updateResourceRequests(reqs, false);
info.allocate(NodeType.OFF_SWITCH, null, SchedulerRequestKey.create(req2),
- req2, null);
+ null);
Assert.assertEquals(0, info.getSchedulerKeys().size());
req1 = ResourceRequest.newInstance(pri1,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
index ff5dc02..9a6c8d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
@@ -93,8 +93,7 @@ public class TestSchedulerApplicationAttempt {
app.liveContainers.put(container1.getContainerId(), container1);
SchedulerNode node = createNode();
app.appSchedulingInfo.allocate(NodeType.OFF_SWITCH, node,
- toSchedulerKey(requestedPriority),
- request, container1.getContainer());
+ toSchedulerKey(requestedPriority), container1.getContainer());
// Reserved container
Priority prio1 = Priority.newInstance(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 43bdc8e..6230098 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -187,7 +187,7 @@ public class TestCapacityScheduler {
private ResourceManager resourceManager = null;
private RMContext mockContext;
-
+
@Before
public void setUp() throws Exception {
resourceManager = new ResourceManager() {
@@ -198,11 +198,11 @@ public class TestCapacityScheduler {
return mgr;
}
};
- CapacitySchedulerConfiguration csConf
+ CapacitySchedulerConfiguration csConf
= new CapacitySchedulerConfiguration();
setupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
- conf.setClass(YarnConfiguration.RM_SCHEDULER,
+ conf.setClass(YarnConfiguration.RM_SCHEDULER,
CapacityScheduler.class, ResourceScheduler.class);
resourceManager.init(conf);
resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
@@ -262,7 +262,7 @@ public class TestCapacityScheduler {
new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
hostName, containerManagerPort, httpPort, rackName, capability,
resourceManager);
- NodeAddedSchedulerEvent nodeAddEvent1 =
+ NodeAddedSchedulerEvent nodeAddEvent1 =
new NodeAddedSchedulerEvent(resourceManager.getRMContext()
.getRMNodes().get(nm.getNodeId()));
resourceManager.getResourceScheduler().handle(nodeAddEvent1);
@@ -273,89 +273,89 @@ public class TestCapacityScheduler {
public void testCapacityScheduler() throws Exception {
LOG.info("--- START: testCapacityScheduler ---");
-
+
// Register node1
String host_0 = "host_0";
- org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 =
- registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
- Resources.createResource(4 * GB, 1));
-
+ org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 =
+ registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+ Resources.createResource(4 * GB, 1));
+
// Register node2
String host_1 = "host_1";
- org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_1 =
- registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
- Resources.createResource(2 * GB, 1));
+ org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_1 =
+ registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+ Resources.createResource(2 * GB, 1));
// ResourceRequest priorities
Priority priority_0 = Priority.newInstance(0);
Priority priority_1 = Priority.newInstance(1);
-
+
// Submit an application
Application application_0 = new Application("user_0", "a1", resourceManager);
application_0.submit();
-
+
application_0.addNodeManager(host_0, 1234, nm_0);
application_0.addNodeManager(host_1, 1234, nm_1);
Resource capability_0_0 = Resources.createResource(1 * GB, 1);
application_0.addResourceRequestSpec(priority_1, capability_0_0);
-
+
Resource capability_0_1 = Resources.createResource(2 * GB, 1);
application_0.addResourceRequestSpec(priority_0, capability_0_1);
- Task task_0_0 = new Task(application_0, priority_1,
+ Task task_0_0 = new Task(application_0, priority_1,
new String[] {host_0, host_1});
application_0.addTask(task_0_0);
-
+
// Submit another application
Application application_1 = new Application("user_1", "b2", resourceManager);
application_1.submit();
-
+
application_1.addNodeManager(host_0, 1234, nm_0);
application_1.addNodeManager(host_1, 1234, nm_1);
-
+
Resource capability_1_0 = Resources.createResource(3 * GB, 1);
application_1.addResourceRequestSpec(priority_1, capability_1_0);
-
+
Resource capability_1_1 = Resources.createResource(2 * GB, 1);
application_1.addResourceRequestSpec(priority_0, capability_1_1);
- Task task_1_0 = new Task(application_1, priority_1,
+ Task task_1_0 = new Task(application_1, priority_1,
new String[] {host_0, host_1});
application_1.addTask(task_1_0);
-
+
// Send resource requests to the scheduler
application_0.schedule();
application_1.schedule();
// Send a heartbeat to kick the tires on the Scheduler
LOG.info("Kick!");
-
+
// task_0_0 and task_1_0 allocated, used=4G
nodeUpdate(nm_0);
-
+
// nothing allocated
nodeUpdate(nm_1);
-
+
// Get allocations from the scheduler
application_0.schedule(); // task_0_0
checkApplicationResourceUsage(1 * GB, application_0);
application_1.schedule(); // task_1_0
checkApplicationResourceUsage(3 * GB, application_1);
-
+
checkNodeResourceUsage(4*GB, nm_0); // task_0_0 (1G) and task_1_0 (3G)
checkNodeResourceUsage(0*GB, nm_1); // no tasks, 2G available
LOG.info("Adding new tasks...");
-
- Task task_1_1 = new Task(application_1, priority_0,
+
+ Task task_1_1 = new Task(application_1, priority_0,
new String[] {ResourceRequest.ANY});
application_1.addTask(task_1_1);
application_1.schedule();
- Task task_0_1 = new Task(application_0, priority_0,
+ Task task_0_1 = new Task(application_0, priority_0,
new String[] {host_0, host_1});
application_0.addTask(task_0_1);
@@ -365,11 +365,11 @@ public class TestCapacityScheduler {
LOG.info("Sending hb from " + nm_0.getHostName());
// nothing new, used=4G
nodeUpdate(nm_0);
-
+
LOG.info("Sending hb from " + nm_1.getHostName());
// task_0_1 is prefer as locality, used=2G
nodeUpdate(nm_1);
-
+
// Get allocations from the scheduler
LOG.info("Trying to allocate...");
application_0.schedule();
@@ -377,10 +377,10 @@ public class TestCapacityScheduler {
application_1.schedule();
checkApplicationResourceUsage(5 * GB, application_1);
-
+
nodeUpdate(nm_0);
nodeUpdate(nm_1);
-
+
checkNodeResourceUsage(4*GB, nm_0);
checkNodeResourceUsage(2*GB, nm_1);
@@ -394,23 +394,23 @@ public class TestCapacityScheduler {
NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
resourceManager.getResourceScheduler().handle(nodeUpdate);
}
-
+
private CapacitySchedulerConfiguration setupQueueConfiguration(
CapacitySchedulerConfiguration conf) {
-
+
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
conf.setCapacity(A, A_CAPACITY);
conf.setCapacity(B, B_CAPACITY);
-
+
// Define 2nd-level queues
conf.setQueues(A, new String[] {"a1", "a2"});
conf.setCapacity(A1, A1_CAPACITY);
conf.setUserLimitFactor(A1, 100.0f);
conf.setCapacity(A2, A2_CAPACITY);
conf.setUserLimitFactor(A2, 100.0f);
-
+
conf.setQueues(B, new String[] {"b1", "b2", "b3"});
conf.setCapacity(B1, B1_CAPACITY);
conf.setUserLimitFactor(B1, 100.0f);
@@ -478,8 +478,8 @@ public class TestCapacityScheduler {
conf.setMaximumCapacity(A, -1);
assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta);
}
-
-
+
+
@Test
public void testRefreshQueues() throws Exception {
CapacityScheduler cs = new CapacityScheduler();
@@ -564,11 +564,11 @@ public class TestCapacityScheduler {
return null;
}
- private void checkApplicationResourceUsage(int expected,
+ private void checkApplicationResourceUsage(int expected,
Application application) {
Assert.assertEquals(expected, application.getUsedResources().getMemorySize());
}
-
+
private void checkNodeResourceUsage(int expected,
org.apache.hadoop.yarn.server.resourcemanager.NodeManager node) {
Assert.assertEquals(expected, node.getUsed().getMemorySize());
@@ -649,7 +649,7 @@ public class TestCapacityScheduler {
// Add a new queue b4
String B4 = B + ".b4";
float B4_CAPACITY = 10;
-
+
B3_CAPACITY -= B4_CAPACITY;
try {
conf.setCapacity(A, 80f);
@@ -661,7 +661,7 @@ public class TestCapacityScheduler {
conf.setCapacity(B4, B4_CAPACITY);
cs.reinitialize(conf,mockContext);
checkQueueCapacities(cs, 80f, 20f);
-
+
// Verify parent for B4
CSQueue rootQueue = cs.getRootQueue();
CSQueue queueB = findQueue(rootQueue, B);
@@ -879,7 +879,7 @@ public class TestCapacityScheduler {
ResourceScheduler.class);
MockRM rm = new MockRM(conf);
rm.start();
-
+
MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB);
RMApp app1 = rm.submitApp(2048);
// kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1
@@ -909,7 +909,7 @@ public class TestCapacityScheduler {
Assert.assertEquals(1, allocated1.size());
Assert.assertEquals(2 * GB, allocated1.get(0).getResource().getMemorySize());
Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId());
-
+
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
@@ -918,13 +918,13 @@ public class TestCapacityScheduler {
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
Assert.assertEquals(2 * GB, c1.getResource().getMemorySize());
-
+
// update node resource to 2 GB, so resource is over-consumed.
- Map<NodeId, ResourceOption> nodeResourceMap =
+ Map<NodeId, ResourceOption> nodeResourceMap =
new HashMap<NodeId, ResourceOption>();
- nodeResourceMap.put(nm1.getNodeId(),
+ nodeResourceMap.put(nm1.getNodeId(),
ResourceOption.newInstance(Resource.newInstance(2 * GB, 1), -1));
- UpdateNodeResourceRequest request =
+ UpdateNodeResourceRequest request =
UpdateNodeResourceRequest.newInstance(nodeResourceMap);
AdminService as = ((MockRM)rm).getAdminService();
as.updateNodeResource(request);
@@ -943,7 +943,7 @@ public class TestCapacityScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
-
+
// Check container can complete successfully in case of resource over-commitment.
ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
c1.getId(), ContainerState.COMPLETE, "", 0, c1.getResource());
@@ -961,7 +961,7 @@ public class TestCapacityScheduler {
Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
-
+
// Verify no NPE is trigger in schedule after resource is updated.
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1);
alloc1Response = am1.schedule();
@@ -979,7 +979,7 @@ public class TestCapacityScheduler {
0, alloc1Response.getAllocatedContainers().size());
rm.stop();
}
-
+
@Test
public void testGetAppsInQueue() throws Exception {
Application application_0 = new Application("user_0", "a1", resourceManager);
@@ -1027,7 +1027,7 @@ public class TestCapacityScheduler {
cs.getSchedulerApplications(), cs, "a1");
Assert.assertEquals("a1", app.getQueue().getQueueName());
}
-
+
@Test
public void testAsyncScheduling() throws Exception {
Configuration conf = new Configuration();
@@ -1038,7 +1038,7 @@ public class TestCapacityScheduler {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
final int NODES = 100;
-
+
// Register nodes
for (int i=0; i < NODES; ++i) {
String host = "192.168.1." + i;
@@ -1046,7 +1046,7 @@ public class TestCapacityScheduler {
MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
cs.handle(new NodeAddedSchedulerEvent(node));
}
-
+
// Now directly exercise the scheduling loop
for (int i=0; i < NODES; ++i) {
CapacityScheduler.schedule(cs);
@@ -1068,7 +1068,7 @@ public class TestCapacityScheduler {
&& attemptPM.getResourcePreempted().equals(currentAttemptPreempted)
&& app.getCurrentAppAttempt().getRMAppAttemptMetrics()
.getIsPreempted() == currentAttemptAMPreempted
- && attemptPM.getNumNonAMContainersPreempted() ==
+ && attemptPM.getNumNonAMContainersPreempted() ==
numLatestAttemptTaskPreempted) {
return;
}
@@ -1082,7 +1082,7 @@ public class TestCapacityScheduler {
Thread.sleep(500);
}
}
-
+
@Test(timeout = 30000)
public void testAllocateDoesNotBlockOnSchedulerLock() throws Exception {
final YarnConfiguration conf = new YarnConfiguration();
@@ -1301,7 +1301,7 @@ public class TestCapacityScheduler {
rm1.stop();
}
-
+
@Test(timeout = 300000)
public void testRecoverRequestAfterPreemption() throws Exception {
Configuration conf = new Configuration();
@@ -1335,8 +1335,9 @@ public class TestCapacityScheduler {
// Already the node local resource request is cleared from RM after
// allocation.
- Assert.assertNull(app.getResourceRequest(
- SchedulerRequestKey.create(request), request.getResourceName()));
+ Assert.assertEquals(0,
+ app.getOutstandingAsksCount(SchedulerRequestKey.create(request),
+ request.getResourceName()));
}
// Call killContainer to preempt the container
@@ -1346,10 +1347,9 @@ public class TestCapacityScheduler {
for (ResourceRequest request : requests) {
// Resource request must have added back in RM after preempt event
// handling.
- Assert.assertEquals(
- 1,
- app.getResourceRequest(SchedulerRequestKey.create(request),
- request.getResourceName()).getNumContainers());
+ Assert.assertEquals(1,
+ app.getOutstandingAsksCount(SchedulerRequestKey.create(request),
+ request.getResourceName()));
}
// New container will be allocated and will move to ALLOCATED state
@@ -2617,7 +2617,7 @@ public class TestCapacityScheduler {
assertEquals("queue B2 max vcores allocation", 12,
((LeafQueue) queueB2).getMaximumAllocation().getVirtualCores());
}
-
+
private void waitContainerAllocated(MockAM am, int mem, int nContainer,
int startContainerId, MockRM rm, MockNM nm) throws Exception {
for (int cId = startContainerId; cId < startContainerId + nContainer; cId++) {
@@ -2651,44 +2651,44 @@ public class TestCapacityScheduler {
MockNM nm1 =
new MockNM("127.0.0.1:1234", 100 * GB, rm1.getResourceTrackerService());
nm1.registerNode();
-
+
RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
-
+
waitContainerAllocated(am1, 1 * GB, 1, 2, rm1, nm1);
// Maximum resoure of b1 is 100 * 0.895 * 0.792 = 71 GB
// 2 GBs used by am, so it's 71 - 2 = 69G.
Assert.assertEquals(69 * GB,
am1.doHeartbeat().getAvailableResources().getMemorySize());
-
+
RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b2");
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
-
+
// Allocate 5 containers, each one is 8 GB in am2 (40 GB in total)
waitContainerAllocated(am2, 8 * GB, 5, 2, rm1, nm1);
-
+
// Allocated one more container with 1 GB resource in b1
waitContainerAllocated(am1, 1 * GB, 1, 3, rm1, nm1);
-
+
// Total is 100 GB,
// B2 uses 41 GB (5 * 8GB containers and 1 AM container)
// B1 uses 3 GB (2 * 1GB containers and 1 AM container)
// Available is 100 - 41 - 3 = 56 GB
Assert.assertEquals(56 * GB,
am1.doHeartbeat().getAvailableResources().getMemorySize());
-
+
// Now we submit app3 to a1 (in higher level hierarchy), to see if headroom
// of app1 (in queue b1) updated correctly
RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "a1");
MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1);
-
+
// Allocate 3 containers, each one is 8 GB in am3 (24 GB in total)
waitContainerAllocated(am3, 8 * GB, 3, 2, rm1, nm1);
-
+
// Allocated one more container with 4 GB resource in b1
waitContainerAllocated(am1, 1 * GB, 1, 4, rm1, nm1);
-
+
// Total is 100 GB,
// B2 uses 41 GB (5 * 8GB containers and 1 AM container)
// B1 uses 4 GB (3 * 1GB containers and 1 AM container)
@@ -2697,7 +2697,7 @@ public class TestCapacityScheduler {
Assert.assertEquals(30 * GB,
am1.doHeartbeat().getAvailableResources().getMemorySize());
}
-
+
@Test
public void testParentQueueMaxCapsAreRespected() throws Exception {
/*
@@ -2713,7 +2713,7 @@ public class TestCapacityScheduler {
csConf.setCapacity(A, 50);
csConf.setMaximumCapacity(A, 50);
csConf.setCapacity(B, 50);
-
+
// Define 2nd-level queues
csConf.setQueues(A, new String[] {"a1", "a2"});
csConf.setCapacity(A1, 50);
@@ -2722,7 +2722,7 @@ public class TestCapacityScheduler {
csConf.setUserLimitFactor(A2, 100.0f);
csConf.setCapacity(B1, B1_CAPACITY);
csConf.setUserLimitFactor(B1, 100.0f);
-
+
YarnConfiguration conf = new YarnConfiguration(csConf);
conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
@@ -2733,12 +2733,12 @@ public class TestCapacityScheduler {
MockNM nm1 =
new MockNM("127.0.0.1:1234", 24 * GB, rm1.getResourceTrackerService());
nm1.registerNode();
-
+
// Launch app1 in a1, resource usage is 1GB (am) + 4GB * 2 = 9GB
RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
waitContainerAllocated(am1, 4 * GB, 2, 2, rm1, nm1);
-
+
// Try to launch app2 in a2, asked 2GB, should success
RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "a2");
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
@@ -2755,24 +2755,24 @@ public class TestCapacityScheduler {
Assert.fail("Shouldn't successfully allocate containers for am2, "
+ "queue-a's max capacity will be violated if container allocated");
}
-
+
@SuppressWarnings("unchecked")
private <E> Set<E> toSet(E... elements) {
Set<E> set = Sets.newHashSet(elements);
return set;
}
-
+
@Test
public void testQueueHierarchyPendingResourceUpdate() throws Exception {
Configuration conf =
TestUtils.getConfigurationWithQueueLabels(new Configuration(false));
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
-
+
final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
mgr.init(conf);
mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y"));
mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
-
+
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
MockRM rm = new MockRM(conf, memStore) {
@@ -2780,74 +2780,74 @@ public class TestCapacityScheduler {
return mgr;
}
};
-
+
rm.start();
MockNM nm1 = // label = x
new MockNM("h1:1234", 200 * GB, rm.getResourceTrackerService());
nm1.registerNode();
-
+
MockNM nm2 = // label = ""
new MockNM("h2:1234", 200 * GB, rm.getResourceTrackerService());
nm2.registerNode();
-
+
// Launch app1 in queue=a1
RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2);
-
+
// Launch app2 in queue=b1
RMApp app2 = rm.submitApp(8 * GB, "app", "user", null, "b1");
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2);
-
+
// am1 asks for 8 * 1GB container for no label
am1.allocate(Arrays.asList(ResourceRequest.newInstance(
Priority.newInstance(1), "*", Resources.createResource(1 * GB), 8)),
null);
-
+
checkPendingResource(rm, "a1", 8 * GB, null);
checkPendingResource(rm, "a", 8 * GB, null);
checkPendingResource(rm, "root", 8 * GB, null);
-
+
// am2 asks for 8 * 1GB container for no label
am2.allocate(Arrays.asList(ResourceRequest.newInstance(
Priority.newInstance(1), "*", Resources.createResource(1 * GB), 8)),
null);
-
+
checkPendingResource(rm, "a1", 8 * GB, null);
checkPendingResource(rm, "a", 8 * GB, null);
checkPendingResource(rm, "b1", 8 * GB, null);
checkPendingResource(rm, "b", 8 * GB, null);
// root = a + b
checkPendingResource(rm, "root", 16 * GB, null);
-
+
// am2 asks for 8 * 1GB container in another priority for no label
am2.allocate(Arrays.asList(ResourceRequest.newInstance(
Priority.newInstance(2), "*", Resources.createResource(1 * GB), 8)),
null);
-
+
checkPendingResource(rm, "a1", 8 * GB, null);
checkPendingResource(rm, "a", 8 * GB, null);
checkPendingResource(rm, "b1", 16 * GB, null);
checkPendingResource(rm, "b", 16 * GB, null);
// root = a + b
checkPendingResource(rm, "root", 24 * GB, null);
-
+
// am1 asks 4 GB resource instead of 8 * GB for priority=1
am1.allocate(Arrays.asList(ResourceRequest.newInstance(
Priority.newInstance(1), "*", Resources.createResource(4 * GB), 1)),
null);
-
+
checkPendingResource(rm, "a1", 4 * GB, null);
checkPendingResource(rm, "a", 4 * GB, null);
checkPendingResource(rm, "b1", 16 * GB, null);
checkPendingResource(rm, "b", 16 * GB, null);
// root = a + b
checkPendingResource(rm, "root", 20 * GB, null);
-
+
// am1 asks 8 * GB resource which label=x
am1.allocate(Arrays.asList(ResourceRequest.newInstance(
Priority.newInstance(2), "*", Resources.createResource(8 * GB), 1,
true, "x")), null);
-
+
checkPendingResource(rm, "a1", 4 * GB, null);
checkPendingResource(rm, "a", 4 * GB, null);
checkPendingResource(rm, "a1", 8 * GB, "x");
@@ -2857,7 +2857,7 @@ public class TestCapacityScheduler {
// root = a + b
checkPendingResource(rm, "root", 20 * GB, null);
checkPendingResource(rm, "root", 8 * GB, "x");
-
+
// some containers allocated for am1, pending resource should decrease
ContainerId containerId =
ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
@@ -2866,7 +2866,7 @@ public class TestCapacityScheduler {
containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
Assert.assertTrue(rm.waitForState(nm2, containerId,
RMContainerState.ALLOCATED));
-
+
checkPendingResource(rm, "a1", 0 * GB, null);
checkPendingResource(rm, "a", 0 * GB, null);
checkPendingResource(rm, "a1", 0 * GB, "x");
@@ -2878,23 +2878,23 @@ public class TestCapacityScheduler {
// root = a + b
checkPendingResourceGreaterThanZero(rm, "root", null);
checkPendingResource(rm, "root", 0 * GB, "x");
-
+
// complete am2, pending resource should be 0 now
AppAttemptRemovedSchedulerEvent appRemovedEvent =
new AppAttemptRemovedSchedulerEvent(
am2.getApplicationAttemptId(), RMAppAttemptState.FINISHED, false);
rm.getResourceScheduler().handle(appRemovedEvent);
-
+
checkPendingResource(rm, "a1", 0 * GB, null);
checkPendingResource(rm, "a", 0 * GB, null);
checkPendingResource(rm, "a1", 0 * GB, "x");
- checkPendingResource(rm, "a", 0 * GB, "x");
+ checkPendingResource(rm, "a", 0 * GB, "x");
checkPendingResource(rm, "b1", 0 * GB, null);
checkPendingResource(rm, "b", 0 * GB, null);
checkPendingResource(rm, "root", 0 * GB, null);
checkPendingResource(rm, "root", 0 * GB, "x");
}
-
+
private void checkPendingResource(MockRM rm, String queueName, int memory,
String label) {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -2932,10 +2932,10 @@ public class TestCapacityScheduler {
Resource minAllocResource = Resource.newInstance(minAllocMb, 1);
String queueName = "a1";
RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName);
-
+
assertEquals("RMApp does not containes minimum allocation",
minAllocResource, rmApp.getAMResourceRequest().getCapability());
-
+
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
LeafQueue queueA =
(LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName);
@@ -3164,7 +3164,7 @@ public class TestCapacityScheduler {
DominantResourceCalculator.class.getName());
verifyAMLimitForLeafQueue(config);
}
-
+
private FiCaSchedulerApp getFiCaSchedulerApp(MockRM rm,
ApplicationId appId) {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -3177,10 +3177,10 @@ public class TestCapacityScheduler {
Configuration conf =
TestUtils.getConfigurationWithQueueLabels(new Configuration(false));
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
-
+
final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
mgr.init(conf);
-
+
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
MockRM rm = new MockRM(conf, memStore) {
@@ -3188,17 +3188,17 @@ public class TestCapacityScheduler {
return mgr;
}
};
-
+
rm.start();
-
+
MockNM nm1 = // label = ""
new MockNM("h1:1234", 200 * GB, rm.getResourceTrackerService());
nm1.registerNode();
-
+
// Launch app1 in queue=a1
RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
-
+
// Allocate two more containers
am1.allocate(
Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1),
@@ -3227,15 +3227,15 @@ public class TestCapacityScheduler {
.newInstance(0, containerId1,
ContainerUpdateType.INCREASE_RESOURCE,
Resources.createResource(3 * GB), null)));
-
+
FiCaSchedulerApp app = getFiCaSchedulerApp(rm, app1.getApplicationId());
-
+
Assert.assertEquals(2 * GB,
app.getAppAttemptResourceUsage().getPending().getMemorySize());
checkPendingResource(rm, "a1", 2 * GB, null);
checkPendingResource(rm, "a", 2 * GB, null);
checkPendingResource(rm, "root", 2 * GB, null);
-
+
// am1 asks to change containerId2 (2G -> 3G) and containerId3 (2G -> 5G)
am1.sendContainerResizingRequest(Arrays.asList(
UpdateContainerRequest
@@ -3246,13 +3246,13 @@ public class TestCapacityScheduler {
.newInstance(0, containerId3,
ContainerUpdateType.INCREASE_RESOURCE,
Resources.createResource(5 * GB), null)));
-
+
Assert.assertEquals(6 * GB,
app.getAppAttemptResourceUsage().getPending().getMemorySize());
checkPendingResource(rm, "a1", 6 * GB, null);
checkPendingResource(rm, "a", 6 * GB, null);
checkPendingResource(rm, "root", 6 * GB, null);
-
+
// am1 asks to change containerId1 (1G->3G), containerId2 (2G -> 4G) and
// containerId3 (2G -> 2G)
am1.sendContainerResizingRequest(Arrays.asList(
@@ -3335,7 +3335,7 @@ public class TestCapacityScheduler {
+ CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES;
conf.setInt(propName, maxAllocVcores);
}
-
+
private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
RMContainer rmContainer = cs.getRMContainer(containerId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index e495328..b2695bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
@@ -1056,9 +1055,13 @@ public class TestLeafQueue {
//test case 3
qb.finishApplication(app_0.getApplicationId(), user_0);
qb.finishApplication(app_2.getApplicationId(), user_1);
- qb.releaseResource(clusterResource, app_0, app_0.getResource(u0SchedKey),
+ qb.releaseResource(clusterResource, app_0,
+ app_0.getAppSchedulingInfo().getPendingAsk(u0SchedKey)
+ .getPerAllocationResource(),
null, null, false);
- qb.releaseResource(clusterResource, app_2, app_2.getResource(u1SchedKey),
+ qb.releaseResource(clusterResource, app_2,
+ app_2.getAppSchedulingInfo().getPendingAsk(u1SchedKey)
+ .getPerAllocationResource(),
null, null, false);
qb.setUserLimit(50);
@@ -1956,7 +1959,7 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyNoContainerAllocated(assignment);
assertEquals(1, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(3, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(3, app_0.getOutstandingAsksCount(schedulerKey));
assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL
// Another off switch, shouldn't allocate due to delay scheduling
@@ -1965,7 +1968,7 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyNoContainerAllocated(assignment);
assertEquals(2, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(3, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(3, app_0.getOutstandingAsksCount(schedulerKey));
assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL
// Another off switch, shouldn't allocate due to delay scheduling
@@ -1974,7 +1977,7 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyNoContainerAllocated(assignment);
assertEquals(3, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(3, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(3, app_0.getOutstandingAsksCount(schedulerKey));
assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL
// Another off switch, now we should allocate
@@ -1985,7 +1988,7 @@ public class TestLeafQueue {
verifyContainerAllocated(assignment, NodeType.OFF_SWITCH);
// should NOT reset
assertEquals(4, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(2, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(2, app_0.getOutstandingAsksCount(schedulerKey));
// NODE_LOCAL - node_0
assignment = a.assignContainers(clusterResource, node_0,
@@ -1994,7 +1997,7 @@ public class TestLeafQueue {
verifyContainerAllocated(assignment, NodeType.NODE_LOCAL);
// should reset
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(1, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(1, app_0.getOutstandingAsksCount(schedulerKey));
// NODE_LOCAL - node_1
assignment = a.assignContainers(clusterResource, node_1,
@@ -2003,7 +2006,7 @@ public class TestLeafQueue {
verifyContainerAllocated(assignment, NodeType.NODE_LOCAL);
// should reset
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(0, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(0, app_0.getOutstandingAsksCount(schedulerKey));
assertEquals(NodeType.NODE_LOCAL, assignment.getType());
// Add 1 more request to check for RACK_LOCAL
@@ -2018,7 +2021,7 @@ public class TestLeafQueue {
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 4, // one extra
true, priority, recordFactory));
app_0.updateResourceRequests(app_0_requests_0);
- assertEquals(4, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(4, app_0.getOutstandingAsksCount(schedulerKey));
// Rack-delay
doReturn(true).when(a).getRackLocalityFullReset();
@@ -2029,7 +2032,7 @@ public class TestLeafQueue {
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
assertEquals(1, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(4, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(4, app_0.getOutstandingAsksCount(schedulerKey));
// Should assign RACK_LOCAL now
assignment = a.assignContainers(clusterResource, node_3,
@@ -2038,14 +2041,14 @@ public class TestLeafQueue {
verifyContainerAllocated(assignment, NodeType.RACK_LOCAL);
// should reset
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(3, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(3, app_0.getOutstandingAsksCount(schedulerKey));
// Shouldn't assign RACK_LOCAL because schedulingOpportunities should have gotten reset.
assignment = a.assignContainers(clusterResource, node_3,
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
assertEquals(1, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(3, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(3, app_0.getOutstandingAsksCount(schedulerKey));
// Next time we schedule RACK_LOCAL, don't reset
doReturn(false).when(a).getRackLocalityFullReset();
@@ -2057,7 +2060,7 @@ public class TestLeafQueue {
verifyContainerAllocated(assignment, NodeType.RACK_LOCAL);
// should NOT reset
assertEquals(2, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(2, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(2, app_0.getOutstandingAsksCount(schedulerKey));
// Another RACK_LOCAL since schedulingOpportunities not reset
assignment = a.assignContainers(clusterResource, node_3,
@@ -2066,7 +2069,7 @@ public class TestLeafQueue {
verifyContainerAllocated(assignment, NodeType.RACK_LOCAL);
// should NOT reset
assertEquals(3, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(1, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(1, app_0.getOutstandingAsksCount(schedulerKey));
// Add a request larger than cluster size to verify
// OFF_SWITCH delay is capped by cluster size
@@ -2185,9 +2188,9 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyNoContainerAllocated(assignment);
assertEquals(1, app_0.getSchedulingOpportunities(schedulerKey1));
- assertEquals(2, app_0.getTotalRequiredResources(schedulerKey1));
+ assertEquals(2, app_0.getOutstandingAsksCount(schedulerKey1));
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey2));
- assertEquals(1, app_0.getTotalRequiredResources(schedulerKey2));
+ assertEquals(1, app_0.getOutstandingAsksCount(schedulerKey2));
// Another off-switch, shouldn't allocate P1 due to delay scheduling
// thus, no P2 either!
@@ -2196,9 +2199,9 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyNoContainerAllocated(assignment);
assertEquals(2, app_0.getSchedulingOpportunities(schedulerKey1));
- assertEquals(2, app_0.getTotalRequiredResources(schedulerKey1));
+ assertEquals(2, app_0.getOutstandingAsksCount(schedulerKey1));
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey2));
- assertEquals(1, app_0.getTotalRequiredResources(schedulerKey2));
+ assertEquals(1, app_0.getOutstandingAsksCount(schedulerKey2));
// Another off-switch, shouldn't allocate OFF_SWITCH P1
assignment = a.assignContainers(clusterResource, node_2,
@@ -2206,9 +2209,9 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyContainerAllocated(assignment, NodeType.OFF_SWITCH);
assertEquals(3, app_0.getSchedulingOpportunities(schedulerKey1));
- assertEquals(1, app_0.getTotalRequiredResources(schedulerKey1));
+ assertEquals(1, app_0.getOutstandingAsksCount(schedulerKey1));
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey2));
- assertEquals(1, app_0.getTotalRequiredResources(schedulerKey2));
+ assertEquals(1, app_0.getOutstandingAsksCount(schedulerKey2));
// Now, DATA_LOCAL for P1
assignment = a.assignContainers(clusterResource, node_0,
@@ -2216,9 +2219,9 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyContainerAllocated(assignment, NodeType.NODE_LOCAL);
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey1));
- assertEquals(0, app_0.getTotalRequiredResources(schedulerKey1));
+ assertEquals(0, app_0.getOutstandingAsksCount(schedulerKey1));
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey2));
- assertEquals(1, app_0.getTotalRequiredResources(schedulerKey2));
+ assertEquals(1, app_0.getOutstandingAsksCount(schedulerKey2));
// Now, OFF_SWITCH for P2
assignment = a.assignContainers(clusterResource, node_1,
@@ -2226,9 +2229,9 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyContainerAllocated(assignment, NodeType.OFF_SWITCH);
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey1));
- assertEquals(0, app_0.getTotalRequiredResources(schedulerKey1));
+ assertEquals(0, app_0.getOutstandingAsksCount(schedulerKey1));
assertEquals(1, app_0.getSchedulingOpportunities(schedulerKey2));
- assertEquals(0, app_0.getTotalRequiredResources(schedulerKey2));
+ assertEquals(0, app_0.getOutstandingAsksCount(schedulerKey2));
}
@@ -2309,7 +2312,7 @@ public class TestLeafQueue {
verifyContainerAllocated(assignment, NodeType.NODE_LOCAL);
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey));
// should reset
- assertEquals(0, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(0, app_0.getOutstandingAsksCount(schedulerKey));
// No allocation on node_1_0 even though it's node/rack local since
// required(ANY) == 0
@@ -2320,7 +2323,7 @@ public class TestLeafQueue {
// Still zero
// since #req=0
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(0, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(0, app_0.getOutstandingAsksCount(schedulerKey));
// Add one request
app_0_requests_0.clear();
@@ -2336,7 +2339,7 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyNoContainerAllocated(assignment);
assertEquals(1, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(1, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(1, app_0.getOutstandingAsksCount(schedulerKey));
// NODE_LOCAL - node_1
assignment = a.assignContainers(clusterResource, node_1_0,
@@ -2345,7 +2348,7 @@ public class TestLeafQueue {
verifyContainerAllocated(assignment, NodeType.NODE_LOCAL);
// should reset
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(0, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(0, app_0.getOutstandingAsksCount(schedulerKey));
}
@Test (timeout = 30000)
@@ -2721,7 +2724,7 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyNoContainerAllocated(assignment);
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(1, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(1, app_0.getOutstandingAsksCount(schedulerKey));
// Now sanity-check node_local
app_0_requests_0.add(
@@ -2752,7 +2755,7 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyContainerAllocated(assignment, NodeType.NODE_LOCAL);
assertEquals(0, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(0, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(0, app_0.getOutstandingAsksCount(schedulerKey));
}
@@ -3205,7 +3208,7 @@ public class TestLeafQueue {
applyCSAssignment(clusterResource, assignment, a, nodes, apps);
verifyContainerAllocated(assignment, NodeType.NODE_LOCAL);
assertEquals(1, app_0.getSchedulingOpportunities(schedulerKey));
- assertEquals(3, app_0.getTotalRequiredResources(schedulerKey));
+ assertEquals(3, app_0.getOutstandingAsksCount(schedulerKey));
assertEquals(0, app_0.getLiveContainers().size());
assertEquals(1, app_1.getLiveContainers().size());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index d2b5ae1..c5b3f00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
@@ -56,6 +57,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -548,11 +551,12 @@ public class TestNodeLabelContainerAllocation {
ApplicationAttemptId attemptId, int memory) {
CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
FiCaSchedulerApp app = cs.getApplicationAttempt(attemptId);
- ResourceRequest rr =
- app.getAppSchedulingInfo().getResourceRequest(
+ PendingAsk ask =
+ app.getAppSchedulingInfo().getPendingAsk(
TestUtils.toSchedulerKey(priority), "*");
Assert.assertEquals(memory,
- rr.getCapability().getMemorySize() * rr.getNumContainers());
+ ask.getPerAllocationResource().getMemorySize() * ask
+ .getCount());
}
private void checkLaunchedContainerNumOnNode(MockRM rm, NodeId nodeId,
@@ -607,18 +611,10 @@ public class TestNodeLabelContainerAllocation {
(CapacityScheduler) rm1.getRMContext().getScheduler();
FiCaSchedulerApp app =
cs.getApplicationAttempt(am1.getApplicationAttemptId());
- List<ResourceRequest> allResourceRequests =
- app.getAppSchedulingInfo().getAllResourceRequests();
- for (ResourceRequest changeReq : allResourceRequests) {
- if (changeReq.getPriority().getPriority() == 2
- || changeReq.getPriority().getPriority() == 3) {
- Assert.assertEquals("Expected label y", "y",
- changeReq.getNodeLabelExpression());
- } else if (changeReq.getPriority().getPriority() == 4) {
- Assert.assertEquals("Expected label EMPTY",
- RMNodeLabelsManager.NO_LABEL, changeReq.getNodeLabelExpression());
- }
- }
+ checkNodePartitionOfRequestedPriority(app.getAppSchedulingInfo(), 2, "y");
+ checkNodePartitionOfRequestedPriority(app.getAppSchedulingInfo(), 3, "y");
+ checkNodePartitionOfRequestedPriority(app.getAppSchedulingInfo(), 4,
+ RMNodeLabelsManager.NO_LABEL);
// Previous any request was Y trying to update with z and the
// request before ANY label is null
@@ -628,17 +624,11 @@ public class TestNodeLabelContainerAllocation {
newReq.add(am1.createResourceReq("h1:1234", 1024, 3, 4, null));
newReq.add(am1.createResourceReq("*", 1024, 4, 5, "z"));
am1.allocate(newReq, new ArrayList<ContainerId>());
- allResourceRequests = app.getAppSchedulingInfo().getAllResourceRequests();
- for (ResourceRequest changeReq : allResourceRequests) {
- if (changeReq.getPriority().getPriority() == 3
- || changeReq.getPriority().getPriority() == 4) {
- Assert.assertEquals("Expected label z", "z",
- changeReq.getNodeLabelExpression());
- } else if (changeReq.getPriority().getPriority() == 2) {
- Assert.assertEquals("Expected label y", "y",
- changeReq.getNodeLabelExpression());
- }
- }
+
+ checkNodePartitionOfRequestedPriority(app.getAppSchedulingInfo(), 3, "z");
+ checkNodePartitionOfRequestedPriority(app.getAppSchedulingInfo(), 4, "z");
+ checkNodePartitionOfRequestedPriority(app.getAppSchedulingInfo(), 2, "y");
+
// Request before ANY and ANY request is set as NULL. Request should be set
// with Empty Label
List<ResourceRequest> resourceRequest1 = new ArrayList<ResourceRequest>();
@@ -653,14 +643,21 @@ public class TestNodeLabelContainerAllocation {
RMNodeLabelsManager.NO_LABEL));
resourceRequest1.add(am1.createResourceReq("h2:1234", 1024, 2, 4, null));
am1.allocate(resourceRequest1, new ArrayList<ContainerId>());
- allResourceRequests = app.getAppSchedulingInfo().getAllResourceRequests();
- for (ResourceRequest changeReq : allResourceRequests) {
- if (changeReq.getPriority().getPriority() == 3) {
- Assert.assertEquals("Expected label Empty",
- RMNodeLabelsManager.NO_LABEL, changeReq.getNodeLabelExpression());
- } else if (changeReq.getPriority().getPriority() == 2) {
- Assert.assertEquals("Expected label y", RMNodeLabelsManager.NO_LABEL,
- changeReq.getNodeLabelExpression());
+
+ checkNodePartitionOfRequestedPriority(app.getAppSchedulingInfo(), 3,
+ RMNodeLabelsManager.NO_LABEL);
+ checkNodePartitionOfRequestedPriority(app.getAppSchedulingInfo(), 2,
+ RMNodeLabelsManager.NO_LABEL);
+ }
+
+ private void checkNodePartitionOfRequestedPriority(AppSchedulingInfo info,
+ int priority, String expectedPartition) {
+ for (SchedulerRequestKey key : info.getSchedulerKeys()) {
+ if (key.getPriority().getPriority() == priority) {
+ Assert.assertEquals("Expected partition is " + expectedPartition,
+ expectedPartition,
+ info.getSchedulingPlacementSet(key)
+ .getPrimaryRequestedNodePartition());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index f9bf89d..3a154b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -329,7 +329,7 @@ public class TestReservations {
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemorySize());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemorySize());
- assertEquals(2, app_0.getTotalRequiredResources(
+ assertEquals(2, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
// try to assign reducer (5G on node 0 and should reserve)
@@ -348,7 +348,7 @@ public class TestReservations {
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemorySize());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemorySize());
- assertEquals(2, app_0.getTotalRequiredResources(
+ assertEquals(2, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
// assign reducer to node 2
@@ -367,7 +367,7 @@ public class TestReservations {
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemorySize());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemorySize());
- assertEquals(1, app_0.getTotalRequiredResources(
+ assertEquals(1, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
// node_1 heartbeat and unreserves from node_0 in order to allocate
@@ -386,7 +386,7 @@ public class TestReservations {
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(8 * GB, node_1.getAllocatedResource().getMemorySize());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemorySize());
- assertEquals(0, app_0.getTotalRequiredResources(
+ assertEquals(0, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
}
@@ -662,7 +662,7 @@ public class TestReservations {
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemorySize());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemorySize());
- assertEquals(2, app_0.getTotalRequiredResources(
+ assertEquals(2, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
// try to assign reducer (5G on node 0 and should reserve)
@@ -681,7 +681,7 @@ public class TestReservations {
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemorySize());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemorySize());
- assertEquals(2, app_0.getTotalRequiredResources(
+ assertEquals(2, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
// assign reducer to node 2
@@ -700,7 +700,7 @@ public class TestReservations {
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemorySize());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemorySize());
- assertEquals(1, app_0.getTotalRequiredResources(
+ assertEquals(1, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
// node_1 heartbeat and won't unreserve from node_0, potentially stuck
@@ -720,7 +720,7 @@ public class TestReservations {
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemorySize());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemorySize());
- assertEquals(1, app_0.getTotalRequiredResources(
+ assertEquals(1, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
}
@@ -841,7 +841,7 @@ public class TestReservations {
assertEquals(null, node_0.getReservedContainer());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemorySize());
- assertEquals(2, app_0.getTotalRequiredResources(
+ assertEquals(2, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
// try to assign reducer (5G on node 0 and should reserve)
@@ -859,7 +859,7 @@ public class TestReservations {
.getMemorySize());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemorySize());
- assertEquals(2, app_0.getTotalRequiredResources(
+ assertEquals(2, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
// could allocate but told need to unreserve first
@@ -876,7 +876,7 @@ public class TestReservations {
assertEquals(null, node_0.getReservedContainer());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemorySize());
assertEquals(8 * GB, node_1.getAllocatedResource().getMemorySize());
- assertEquals(1, app_0.getTotalRequiredResources(
+ assertEquals(1, app_0.getOutstandingAsksCount(
toSchedulerKey(priorityReduce)));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-5959. RM changes to support
change of container ExecutionType. (Arun Suresh via wangda)
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
index 73d9e5c..641ef64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
@@ -34,11 +34,15 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocolPB;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -64,8 +68,11 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterDistrib
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -75,13 +82,17 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateS
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
/**
@@ -91,8 +102,10 @@ public class TestOpportunisticContainerAllocatorAMService {
private static final int GB = 1024;
- @Test(timeout = 60000)
- public void testNodeRemovalDuringAllocate() throws Exception {
+ private MockRM rm;
+
+ @Before
+ public void createAndStartRM() {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
YarnConfiguration conf = new YarnConfiguration(csConf);
@@ -102,8 +115,445 @@ public class TestOpportunisticContainerAllocatorAMService {
YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
conf.setInt(
YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS, 100);
- MockRM rm = new MockRM(conf);
+ rm = new MockRM(conf);
rm.start();
+ }
+
+ @After
+ public void stopRM() {
+ if (rm != null) {
+ rm.stop();
+ }
+ }
+
+ @Test(timeout = 600000)
+ public void testContainerPromoteAndDemoteBeforeContainerStart() throws Exception {
+ HashMap<NodeId, MockNM> nodes = new HashMap<>();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm1.getNodeId(), nm1);
+ MockNM nm2 = new MockNM("h1:4321", 4096, rm.getResourceTrackerService());
+ nodes.put(nm2.getNodeId(), nm2);
+ MockNM nm3 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm3.getNodeId(), nm3);
+ MockNM nm4 = new MockNM("h2:4321", 4096, rm.getResourceTrackerService());
+ nodes.put(nm4.getNodeId(), nm4);
+ nm1.registerNode();
+ nm2.registerNode();
+ nm3.registerNode();
+ nm4.registerNode();
+
+ OpportunisticContainerAllocatorAMService amservice =
+ (OpportunisticContainerAllocatorAMService) rm
+ .getApplicationMasterService();
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ ApplicationAttemptId attemptId =
+ app1.getCurrentAppAttempt().getAppAttemptId();
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2);
+ ResourceScheduler scheduler = rm.getResourceScheduler();
+ RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+ RMNode rmNode2 = rm.getRMContext().getRMNodes().get(nm2.getNodeId());
+ RMNode rmNode3 = rm.getRMContext().getRMNodes().get(nm3.getNodeId());
+ RMNode rmNode4 = rm.getRMContext().getRMNodes().get(nm4.getNodeId());
+
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ nm4.nodeHeartbeat(true);
+
+ ((RMNodeImpl) rmNode1)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+ ((RMNodeImpl) rmNode2)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+ ((RMNodeImpl) rmNode3)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+ ((RMNodeImpl) rmNode4)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+
+ OpportunisticContainerContext ctxt = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(attemptId).getOpportunisticContainerContext();
+ // Send add and update node events to AM Service.
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode1));
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode2));
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode3));
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode4));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode2));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode3));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode4));
+ // All nodes 1 - 4 will be applicable for scheduling.
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ nm4.nodeHeartbeat(true);
+
+ Thread.sleep(1000);
+
+ QueueMetrics metrics = ((CapacityScheduler) scheduler).getRootQueue()
+ .getMetrics();
+
+ // Verify Metrics
+ verifyMetrics(metrics, 15360, 15, 1024, 1, 1);
+
+ AllocateResponse allocateResponse = am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1),
+ "*", Resources.createResource(1 * GB), 2, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true))),
+ null);
+ List<Container> allocatedContainers = allocateResponse
+ .getAllocatedContainers();
+ Assert.assertEquals(2, allocatedContainers.size());
+ Container container = allocatedContainers.get(0);
+ MockNM allocNode = nodes.get(container.getNodeId());
+ MockNM sameHostDiffNode = null;
+ for (NodeId n : nodes.keySet()) {
+ if (n.getHost().equals(allocNode.getNodeId().getHost()) &&
+ n.getPort() != allocNode.getNodeId().getPort()) {
+ sameHostDiffNode = nodes.get(n);
+ }
+ }
+
+ // Verify Metrics After OPP allocation (Nothing should change)
+ verifyMetrics(metrics, 15360, 15, 1024, 1, 1);
+
+ am1.sendContainerUpdateRequest(
+ Arrays.asList(UpdateContainerRequest.newInstance(0,
+ container.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ null, ExecutionType.GUARANTEED)));
+ // Node on same host should not result in allocation
+ sameHostDiffNode.nodeHeartbeat(true);
+ Thread.sleep(200);
+ allocateResponse = am1.allocate(new ArrayList<>(), new ArrayList<>());
+ Assert.assertEquals(0, allocateResponse.getUpdatedContainers().size());
+
+ // Verify Metrics After OPP allocation (Nothing should change again)
+ verifyMetrics(metrics, 15360, 15, 1024, 1, 1);
+
+ // Send Promotion req again... this should result in update error
+ allocateResponse = am1.sendContainerUpdateRequest(
+ Arrays.asList(UpdateContainerRequest.newInstance(0,
+ container.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ null, ExecutionType.GUARANTEED)));
+ Assert.assertEquals(0, allocateResponse.getUpdatedContainers().size());
+ Assert.assertEquals(1, allocateResponse.getUpdateErrors().size());
+ Assert.assertEquals("UPDATE_OUTSTANDING_ERROR",
+ allocateResponse.getUpdateErrors().get(0).getReason());
+ Assert.assertEquals(container.getId(),
+ allocateResponse.getUpdateErrors().get(0)
+ .getUpdateContainerRequest().getContainerId());
+
+ // Send Promotion req again with incorrect version...
+ // this should also result in update error
+ allocateResponse = am1.sendContainerUpdateRequest(
+ Arrays.asList(UpdateContainerRequest.newInstance(1,
+ container.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ null, ExecutionType.GUARANTEED)));
+
+ Assert.assertEquals(0, allocateResponse.getUpdatedContainers().size());
+ Assert.assertEquals(1, allocateResponse.getUpdateErrors().size());
+ Assert.assertEquals("INCORRECT_CONTAINER_VERSION_ERROR|1|0",
+ allocateResponse.getUpdateErrors().get(0).getReason());
+ Assert.assertEquals(container.getId(),
+ allocateResponse.getUpdateErrors().get(0)
+ .getUpdateContainerRequest().getContainerId());
+
+ // Ensure after correct node heartbeats, we should get the allocation
+ allocNode.nodeHeartbeat(true);
+ Thread.sleep(200);
+ allocateResponse = am1.allocate(new ArrayList<>(), new ArrayList<>());
+ Assert.assertEquals(1, allocateResponse.getUpdatedContainers().size());
+ Container uc =
+ allocateResponse.getUpdatedContainers().get(0).getContainer();
+ Assert.assertEquals(ExecutionType.GUARANTEED, uc.getExecutionType());
+ Assert.assertEquals(uc.getId(), container.getId());
+ Assert.assertEquals(uc.getVersion(), container.getVersion() + 1);
+
+ // Verify Metrics After OPP allocation :
+ // Allocated cores+mem should have increased, available should decrease
+ verifyMetrics(metrics, 14336, 14, 2048, 2, 2);
+
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ nm4.nodeHeartbeat(true);
+ Thread.sleep(200);
+
+ // Verify that the container is still in ACQUIRED state wrt the RM.
+ RMContainer rmContainer = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(
+ uc.getId().getApplicationAttemptId()).getRMContainer(uc.getId());
+ Assert.assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());
+
+ // Now demote the container back..
+ allocateResponse = am1.sendContainerUpdateRequest(
+ Arrays.asList(UpdateContainerRequest.newInstance(uc.getVersion(),
+ uc.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE,
+ null, ExecutionType.OPPORTUNISTIC)));
+ // This should happen in the same heartbeat..
+ Assert.assertEquals(1, allocateResponse.getUpdatedContainers().size());
+ uc = allocateResponse.getUpdatedContainers().get(0).getContainer();
+ Assert.assertEquals(ExecutionType.OPPORTUNISTIC, uc.getExecutionType());
+ Assert.assertEquals(uc.getId(), container.getId());
+ Assert.assertEquals(uc.getVersion(), container.getVersion() + 2);
+
+ // Verify Metrics After OPP allocation :
+ // Everything should have reverted to what it was
+ verifyMetrics(metrics, 15360, 15, 1024, 1, 1);
+ }
+
+ @Test(timeout = 60000)
+ public void testContainerPromoteAfterContainerStart() throws Exception {
+ HashMap<NodeId, MockNM> nodes = new HashMap<>();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm1.getNodeId(), nm1);
+ MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm2.getNodeId(), nm2);
+ nm1.registerNode();
+ nm2.registerNode();
+
+ OpportunisticContainerAllocatorAMService amservice =
+ (OpportunisticContainerAllocatorAMService) rm
+ .getApplicationMasterService();
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ ApplicationAttemptId attemptId =
+ app1.getCurrentAppAttempt().getAppAttemptId();
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2);
+ ResourceScheduler scheduler = rm.getResourceScheduler();
+ RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+ RMNode rmNode2 = rm.getRMContext().getRMNodes().get(nm2.getNodeId());
+
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+
+ ((RMNodeImpl) rmNode1)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+ ((RMNodeImpl) rmNode2)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+
+ OpportunisticContainerContext ctxt = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(attemptId).getOpportunisticContainerContext();
+ // Send add and update node events to AM Service.
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode1));
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode2));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode2));
+
+ // All nodes 1 to 2 will be applicable for scheduling.
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+
+ Thread.sleep(1000);
+
+ QueueMetrics metrics = ((CapacityScheduler) scheduler).getRootQueue()
+ .getMetrics();
+
+ // Verify Metrics
+ verifyMetrics(metrics, 7168, 7, 1024, 1, 1);
+
+ AllocateResponse allocateResponse = am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1),
+ "*", Resources.createResource(1 * GB), 2, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true))),
+ null);
+ List<Container> allocatedContainers = allocateResponse
+ .getAllocatedContainers();
+ Assert.assertEquals(2, allocatedContainers.size());
+ Container container = allocatedContainers.get(0);
+ MockNM allocNode = nodes.get(container.getNodeId());
+
+ // Start Container in NM
+ allocNode.nodeHeartbeat(Arrays.asList(
+ ContainerStatus.newInstance(container.getId(),
+ ExecutionType.OPPORTUNISTIC, ContainerState.RUNNING, "", 0)),
+ true);
+ Thread.sleep(200);
+
+ // Verify that container is actually running wrt the RM..
+ RMContainer rmContainer = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(
+ container.getId().getApplicationAttemptId()).getRMContainer(
+ container.getId());
+ Assert.assertEquals(RMContainerState.RUNNING, rmContainer.getState());
+
+ // Verify Metrics After OPP allocation (Nothing should change)
+ verifyMetrics(metrics, 7168, 7, 1024, 1, 1);
+
+ am1.sendContainerUpdateRequest(
+ Arrays.asList(UpdateContainerRequest.newInstance(0,
+ container.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ null, ExecutionType.GUARANTEED)));
+
+ // Verify Metrics After OPP allocation (Nothing should change again)
+ verifyMetrics(metrics, 7168, 7, 1024, 1, 1);
+
+ // Send Promotion req again... this should result in update error
+ allocateResponse = am1.sendContainerUpdateRequest(
+ Arrays.asList(UpdateContainerRequest.newInstance(0,
+ container.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ null, ExecutionType.GUARANTEED)));
+ Assert.assertEquals(0, allocateResponse.getUpdatedContainers().size());
+ Assert.assertEquals(1, allocateResponse.getUpdateErrors().size());
+ Assert.assertEquals("UPDATE_OUTSTANDING_ERROR",
+ allocateResponse.getUpdateErrors().get(0).getReason());
+ Assert.assertEquals(container.getId(),
+ allocateResponse.getUpdateErrors().get(0)
+ .getUpdateContainerRequest().getContainerId());
+
+ // Start Container in NM
+ allocNode.nodeHeartbeat(Arrays.asList(
+ ContainerStatus.newInstance(container.getId(),
+ ExecutionType.OPPORTUNISTIC, ContainerState.RUNNING, "", 0)),
+ true);
+ Thread.sleep(200);
+
+ allocateResponse = am1.allocate(new ArrayList<>(), new ArrayList<>());
+ Assert.assertEquals(1, allocateResponse.getUpdatedContainers().size());
+ Container uc =
+ allocateResponse.getUpdatedContainers().get(0).getContainer();
+ Assert.assertEquals(ExecutionType.GUARANTEED, uc.getExecutionType());
+ Assert.assertEquals(uc.getId(), container.getId());
+ Assert.assertEquals(uc.getVersion(), container.getVersion() + 1);
+
+ // Verify that the Container is still in RUNNING state wrt RM..
+ rmContainer = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(
+ uc.getId().getApplicationAttemptId()).getRMContainer(uc.getId());
+ Assert.assertEquals(RMContainerState.RUNNING, rmContainer.getState());
+
+ // Verify Metrics After OPP allocation :
+ // Allocated cores+mem should have increased, available should decrease
+ verifyMetrics(metrics, 6144, 6, 2048, 2, 2);
+ }
+
+ @Test(timeout = 600000)
+ public void testContainerPromoteAfterContainerComplete() throws Exception {
+ HashMap<NodeId, MockNM> nodes = new HashMap<>();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm1.getNodeId(), nm1);
+ MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm2.getNodeId(), nm2);
+ nm1.registerNode();
+ nm2.registerNode();
+
+ OpportunisticContainerAllocatorAMService amservice =
+ (OpportunisticContainerAllocatorAMService) rm
+ .getApplicationMasterService();
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ ApplicationAttemptId attemptId =
+ app1.getCurrentAppAttempt().getAppAttemptId();
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2);
+ ResourceScheduler scheduler = rm.getResourceScheduler();
+ RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+ RMNode rmNode2 = rm.getRMContext().getRMNodes().get(nm2.getNodeId());
+
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+
+ ((RMNodeImpl) rmNode1)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+ ((RMNodeImpl) rmNode2)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+
+ OpportunisticContainerContext ctxt = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(attemptId).getOpportunisticContainerContext();
+ // Send add and update node events to AM Service.
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode1));
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode2));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode2));
+
+ // All nodes 1 to 2 will be applicable for scheduling.
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+
+ Thread.sleep(1000);
+
+ QueueMetrics metrics = ((CapacityScheduler) scheduler).getRootQueue()
+ .getMetrics();
+
+ // Verify Metrics
+ verifyMetrics(metrics, 7168, 7, 1024, 1, 1);
+
+ AllocateResponse allocateResponse = am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1),
+ "*", Resources.createResource(1 * GB), 2, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true))),
+ null);
+ List<Container> allocatedContainers = allocateResponse
+ .getAllocatedContainers();
+ Assert.assertEquals(2, allocatedContainers.size());
+ Container container = allocatedContainers.get(0);
+ MockNM allocNode = nodes.get(container.getNodeId());
+
+ // Start Container in NM
+ allocNode.nodeHeartbeat(Arrays.asList(
+ ContainerStatus.newInstance(container.getId(),
+ ExecutionType.OPPORTUNISTIC, ContainerState.RUNNING, "", 0)),
+ true);
+ Thread.sleep(200);
+
+ // Verify that container is actually running wrt the RM..
+ RMContainer rmContainer = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(
+ container.getId().getApplicationAttemptId()).getRMContainer(
+ container.getId());
+ Assert.assertEquals(RMContainerState.RUNNING, rmContainer.getState());
+
+ // Container Completed in the NM
+ allocNode.nodeHeartbeat(Arrays.asList(
+ ContainerStatus.newInstance(container.getId(),
+ ExecutionType.OPPORTUNISTIC, ContainerState.COMPLETE, "", 0)),
+ true);
+ Thread.sleep(200);
+
+ // Verify that container has been removed..
+ rmContainer = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(
+ container.getId().getApplicationAttemptId()).getRMContainer(
+ container.getId());
+ Assert.assertNull(rmContainer);
+
+ // Verify Metrics After OPP allocation (Nothing should change)
+ verifyMetrics(metrics, 7168, 7, 1024, 1, 1);
+
+ // Send Promotion req... this should result in update error
+ // Since the container doesn't exist anymore..
+ allocateResponse = am1.sendContainerUpdateRequest(
+ Arrays.asList(UpdateContainerRequest.newInstance(0,
+ container.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ null, ExecutionType.GUARANTEED)));
+
+ Assert.assertEquals(1,
+ allocateResponse.getCompletedContainersStatuses().size());
+ Assert.assertEquals(container.getId(),
+ allocateResponse.getCompletedContainersStatuses().get(0)
+ .getContainerId());
+ Assert.assertEquals(0, allocateResponse.getUpdatedContainers().size());
+ Assert.assertEquals(1, allocateResponse.getUpdateErrors().size());
+ Assert.assertEquals("INVALID_CONTAINER_ID",
+ allocateResponse.getUpdateErrors().get(0).getReason());
+ Assert.assertEquals(container.getId(),
+ allocateResponse.getUpdateErrors().get(0)
+ .getUpdateContainerRequest().getContainerId());
+
+ // Verify Metrics After OPP allocation (Nothing should change again)
+ verifyMetrics(metrics, 7168, 7, 1024, 1, 1);
+ }
+
+ private void verifyMetrics(QueueMetrics metrics, long availableMB,
+ int availableVirtualCores, long allocatedMB,
+ int allocatedVirtualCores, int allocatedContainers) {
+ Assert.assertEquals(availableMB, metrics.getAvailableMB());
+ Assert.assertEquals(availableVirtualCores, metrics.getAvailableVirtualCores());
+ Assert.assertEquals(allocatedMB, metrics.getAllocatedMB());
+ Assert.assertEquals(allocatedVirtualCores, metrics.getAllocatedVirtualCores());
+ Assert.assertEquals(allocatedContainers, metrics.getAllocatedContainers());
+ }
+
+ @Test(timeout = 60000)
+ public void testNodeRemovalDuringAllocate() throws Exception {
MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
nm1.registerNode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 497c6d0..786cc50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -100,6 +100,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@@ -478,7 +479,7 @@ public class TestRMAppAttemptTransitions {
assertEquals(expectedState, applicationAttempt.getAppAttemptState());
verify(scheduler, times(expectedAllocateCount)).allocate(
any(ApplicationAttemptId.class), any(List.class), any(List.class),
- any(List.class), any(List.class), any(List.class), any(List.class));
+ any(List.class), any(List.class), any(ContainerUpdates.class));
assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
assertNull(applicationAttempt.getMasterContainer());
@@ -499,7 +500,7 @@ public class TestRMAppAttemptTransitions {
verify(applicationMasterLauncher).handle(any(AMLauncherEvent.class));
verify(scheduler, times(2)).allocate(any(ApplicationAttemptId.class),
any(List.class), any(List.class), any(List.class), any(List.class),
- any(List.class), any(List.class));
+ any(ContainerUpdates.class));
verify(nmTokenManager).clearNodeSetForAttempt(
applicationAttempt.getAppAttemptId());
}
@@ -526,7 +527,7 @@ public class TestRMAppAttemptTransitions {
}
/**
- * {@link RMAppAttemptState#LAUNCH}
+ * {@link RMAppAttemptState#LAUNCHED}
*/
private void testAppAttemptLaunchedState(Container container) {
assertEquals(RMAppAttemptState.LAUNCHED,
@@ -649,8 +650,8 @@ public class TestRMAppAttemptTransitions {
when(allocation.getContainers()).
thenReturn(Collections.singletonList(container));
when(scheduler.allocate(any(ApplicationAttemptId.class), any(List.class),
- any(List.class), any(List.class), any(List.class), any(List.class),
- any(List.class))).
+ any(List.class), any(List.class), any(List.class),
+ any(ContainerUpdates.class))).
thenReturn(allocation);
RMContainer rmContainer = mock(RMContainerImpl.class);
when(scheduler.getRMContainer(container.getId())).
@@ -1129,8 +1130,9 @@ public class TestRMAppAttemptTransitions {
when(allocation.getContainers()).
thenReturn(Collections.singletonList(amContainer));
when(scheduler.allocate(any(ApplicationAttemptId.class), any(List.class),
- any(List.class), any(List.class), any(List.class), any(List.class),
- any(List.class))).thenReturn(allocation);
+ any(List.class), any(List.class), any(List.class),
+ any(ContainerUpdates.class)))
+ .thenReturn(allocation);
RMContainer rmContainer = mock(RMContainerImpl.class);
when(scheduler.getRMContainer(amContainer.getId())).thenReturn(rmContainer);
@@ -1610,7 +1612,8 @@ public class TestRMAppAttemptTransitions {
YarnScheduler mockScheduler = mock(YarnScheduler.class);
when(mockScheduler.allocate(any(ApplicationAttemptId.class),
any(List.class), any(List.class), any(List.class), any(List.class),
- any(List.class), any(List.class))).thenAnswer(new Answer<Allocation>() {
+ any(ContainerUpdates.class)))
+ .thenAnswer(new Answer<Allocation>() {
@SuppressWarnings("rawtypes")
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index e737a84..893f802 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -114,7 +115,8 @@ public class TestRMContainerImpl {
YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
true);
when(rmContext.getYarnConfiguration()).thenReturn(conf);
- RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
+ RMContainer rmContainer = new RMContainerImpl(container,
+ SchedulerRequestKey.extractFrom(container), appAttemptId,
nodeId, "user", rmContext);
assertEquals(RMContainerState.NEW, rmContainer.getState());
@@ -216,7 +218,8 @@ public class TestRMContainerImpl {
when(rmContext.getYarnConfiguration()).thenReturn(conf);
when(rmContext.getRMApps()).thenReturn(appMap);
- RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
+ RMContainer rmContainer = new RMContainerImpl(container,
+ SchedulerRequestKey.extractFrom(container), appAttemptId,
nodeId, "user", rmContext);
assertEquals(RMContainerState.NEW, rmContainer.getState());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 27860a6..43bdc8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -114,6 +114,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.
ContainerExpiredSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -160,7 +161,9 @@ import com.google.common.collect.Sets;
public class TestCapacityScheduler {
private static final Log LOG = LogFactory.getLog(TestCapacityScheduler.class);
private final int GB = 1024;
-
+ private final static ContainerUpdates NULL_UPDATE_REQUESTS =
+ new ContainerUpdates();
+
private static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
private static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
private static final String A1 = A + ".a1";
@@ -738,12 +741,12 @@ public class TestCapacityScheduler {
// Verify the blacklist can be updated independent of requesting containers
cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(),
- Collections.singletonList(host), null, null, null);
+ Collections.singletonList(host), null, NULL_UPDATE_REQUESTS);
Assert.assertTrue(cs.getApplicationAttempt(appAttemptId)
.isPlaceBlacklisted(host));
cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(), null,
- Collections.singletonList(host), null, null);
+ Collections.singletonList(host), NULL_UPDATE_REQUESTS);
Assert.assertFalse(cs.getApplicationAttempt(appAttemptId)
.isPlaceBlacklisted(host));
rm.stop();
@@ -839,7 +842,7 @@ public class TestCapacityScheduler {
cs.allocate(appAttemptId1,
Collections.<ResourceRequest>singletonList(r1),
Collections.<ContainerId>emptyList(),
- null, null, null, null);
+ null, null, NULL_UPDATE_REQUESTS);
//And this will result in container assignment for app1
CapacityScheduler.schedule(cs);
@@ -856,7 +859,7 @@ public class TestCapacityScheduler {
cs.allocate(appAttemptId2,
Collections.<ResourceRequest>singletonList(r2),
Collections.<ContainerId>emptyList(),
- null, null, null, null);
+ null, null, NULL_UPDATE_REQUESTS);
//In this case we do not perform container assignment because we want to
//verify re-ordering based on the allocation alone
@@ -2981,7 +2984,8 @@ public class TestCapacityScheduler {
Allocation allocate =
cs.allocate(appAttemptId, Collections.<ResourceRequest> emptyList(),
- Collections.<ContainerId> emptyList(), null, null, null, null);
+ Collections.<ContainerId> emptyList(), null, null,
+ NULL_UPDATE_REQUESTS);
Assert.assertNotNull(attempt);
@@ -2997,7 +3001,8 @@ public class TestCapacityScheduler {
allocate =
cs.allocate(appAttemptId, Collections.<ResourceRequest> emptyList(),
- Collections.<ContainerId> emptyList(), null, null, null, null);
+ Collections.<ContainerId> emptyList(), null, null,
+ NULL_UPDATE_REQUESTS);
// All resources should be sent as headroom
Assert.assertEquals(newResource, allocate.getResourceLimit());
@@ -3504,7 +3509,7 @@ public class TestCapacityScheduler {
cs.allocate(appAttemptId3,
Collections.<ResourceRequest>singletonList(y1Req),
Collections.<ContainerId>emptyList(),
- null, null, null, null);
+ null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
}
assertEquals("Y1 Used Resource should be 4 GB", 4 * GB,
@@ -3518,7 +3523,7 @@ public class TestCapacityScheduler {
cs.allocate(appAttemptId1,
Collections.<ResourceRequest>singletonList(x1Req),
Collections.<ContainerId>emptyList(),
- null, null, null, null);
+ null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
}
assertEquals("X1 Used Resource should be 7 GB", 7 * GB,
@@ -3531,7 +3536,7 @@ public class TestCapacityScheduler {
cs.allocate(appAttemptId2,
Collections.<ResourceRequest>singletonList(x2Req),
Collections.<ContainerId>emptyList(),
- null, null, null, null);
+ null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
assertEquals("X2 Used Resource should be 0", 0,
cs.getQueue("x2").getUsedResources().getMemorySize());
@@ -3543,7 +3548,7 @@ public class TestCapacityScheduler {
cs.allocate(appAttemptId1,
Collections.<ResourceRequest>singletonList(x1Req),
Collections.<ContainerId>emptyList(),
- null, null, null, null);
+ null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
assertEquals("X1 Used Resource should be 7 GB", 7 * GB,
cs.getQueue("x1").getUsedResources().getMemorySize());
@@ -3557,7 +3562,7 @@ public class TestCapacityScheduler {
cs.allocate(appAttemptId3,
Collections.<ResourceRequest>singletonList(y1Req),
Collections.<ContainerId>emptyList(),
- null, null, null, null);
+ null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
}
assertEquals("P2 Used Resource should be 8 GB", 8 * GB,
@@ -3616,7 +3621,7 @@ public class TestCapacityScheduler {
//This will allocate for app1
cs.allocate(appAttemptId1, Collections.<ResourceRequest>singletonList(r1),
Collections.<ContainerId>emptyList(),
- null, null, null, null).getContainers().size();
+ null, null, NULL_UPDATE_REQUESTS).getContainers().size();
CapacityScheduler.schedule(cs);
ResourceRequest r2 = null;
for (int i =0; i < 13; i++) {
@@ -3625,7 +3630,7 @@ public class TestCapacityScheduler {
cs.allocate(appAttemptId2,
Collections.<ResourceRequest>singletonList(r2),
Collections.<ContainerId>emptyList(),
- null, null, null, null);
+ null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
}
assertEquals("A Used Resource should be 2 GB", 2 * GB,
@@ -3638,11 +3643,11 @@ public class TestCapacityScheduler {
ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory);
cs.allocate(appAttemptId1, Collections.<ResourceRequest>singletonList(r1),
Collections.<ContainerId>emptyList(),
- null, null, null, null).getContainers().size();
+ null, null, NULL_UPDATE_REQUESTS).getContainers().size();
CapacityScheduler.schedule(cs);
cs.allocate(appAttemptId2, Collections.<ResourceRequest>singletonList(r2),
- Collections.<ContainerId>emptyList(), null, null, null, null);
+ Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
//Check blocked Resource
assertEquals("A Used Resource should be 2 GB", 2 * GB,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index a6ae0c2..cf91841 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -55,6 +55,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preempti
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
+
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -280,7 +282,8 @@ public class TestChildQueueOrder {
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
Container container=TestUtils.getMockContainer(containerId,
node_0.getNodeID(), Resources.createResource(1*GB), priority);
- RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
+ RMContainer rmContainer = new RMContainerImpl(container,
+ SchedulerRequestKey.extractFrom(container), appAttemptId,
node_0.getNodeID(), "user", rmContext);
// Assign {1,2,3,4} 1GB containers respectively to queues
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index c9eb8b3..f9bf89d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -923,13 +923,15 @@ public class TestReservations {
Container container = TestUtils.getMockContainer(containerId,
node_1.getNodeID(), Resources.createResource(2*GB),
priorityMap.getPriority());
- RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
+ RMContainer rmContainer = new RMContainerImpl(container,
+ SchedulerRequestKey.extractFrom(container), appAttemptId,
node_1.getNodeID(), "user", rmContext);
Container container_1 = TestUtils.getMockContainer(containerId,
node_0.getNodeID(), Resources.createResource(1*GB),
priorityMap.getPriority());
- RMContainer rmContainer_1 = new RMContainerImpl(container_1, appAttemptId,
+ RMContainer rmContainer_1 = new RMContainerImpl(container_1,
+ SchedulerRequestKey.extractFrom(container_1), appAttemptId,
node_0.getNodeID(), "user", rmContext);
// no reserved containers
@@ -996,7 +998,8 @@ public class TestReservations {
Container container = TestUtils.getMockContainer(containerId,
node_1.getNodeID(), Resources.createResource(2*GB),
priorityMap.getPriority());
- RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
+ RMContainer rmContainer = new RMContainerImpl(container,
+ SchedulerRequestKey.extractFrom(container), appAttemptId,
node_1.getNodeID(), "user", rmContext);
// nothing reserved
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 992b75d..8e8267f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -41,6 +41,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
@@ -71,6 +74,8 @@ public class FairSchedulerTestBase {
public static final float TEST_RESERVATION_THRESHOLD = 0.09f;
private static final int SLEEP_DURATION = 10;
private static final int SLEEP_RETRIES = 1000;
+ final static ContainerUpdates NULL_UPDATE_REQUESTS =
+ new ContainerUpdates();
/**
* The list of nodes added to the cluster using the {@link #addNode} method.
@@ -181,7 +186,8 @@ public class FairSchedulerTestBase {
resourceManager.getRMContext().getRMApps()
.put(id.getApplicationId(), rmApp);
- scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null, null, null);
+ scheduler.allocate(id, ask, new ArrayList<ContainerId>(),
+ null, null, NULL_UPDATE_REQUESTS);
return id;
}
@@ -207,7 +213,8 @@ public class FairSchedulerTestBase {
resourceManager.getRMContext().getRMApps()
.put(id.getApplicationId(), rmApp);
- scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null, null, null);
+ scheduler.allocate(id, ask, new ArrayList<ContainerId>(),
+ null, null, NULL_UPDATE_REQUESTS);
return id;
}
@@ -229,7 +236,8 @@ public class FairSchedulerTestBase {
ResourceRequest request, ApplicationAttemptId attId) {
List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
ask.add(request);
- scheduler.allocate(attId, ask, new ArrayList<ContainerId>(), null, null, null, null);
+ scheduler.allocate(attId, ask, new ArrayList<ContainerId>(),
+ null, null, NULL_UPDATE_REQUESTS);
}
protected void createApplicationWithAMResource(ApplicationAttemptId attId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
index 4cc99b2..8bb06e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
@@ -119,7 +119,8 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
List<ResourceRequest> ask = new ArrayList<>();
ask.add(createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true));
scheduler.allocate(
- appAttemptId, ask, new ArrayList<ContainerId>(), null, null, null, null);
+ appAttemptId, ask, new ArrayList<ContainerId>(),
+ null, null, NULL_UPDATE_REQUESTS);
FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId);
triggerSchedulingAttempt();
@@ -157,7 +158,7 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true);
ask.add(request);
scheduler.allocate(appAttemptId, ask,
- new ArrayList<ContainerId>(), null, null, null, null);
+ new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
triggerSchedulingAttempt();
FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId);
@@ -169,7 +170,7 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
ask.clear();
ask.add(request);
scheduler.allocate(appAttemptId, ask,
- new ArrayList<ContainerId>(), null, null, null, null);
+ new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
triggerSchedulingAttempt();
checkAppConsumption(app, Resources.createResource(2048,2));
@@ -335,7 +336,7 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
ask1.add(request1);
ask1.add(request2);
scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null, null,
- null, null);
+ NULL_UPDATE_REQUESTS);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 5aa1e2d..b1e412b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -93,6 +93,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
@@ -124,6 +127,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
private final int GB = 1024;
private final static String ALLOC_FILE =
new File(TEST_DIR, "test-queues").getAbsolutePath();
+ private final static ContainerUpdates NULL_UPDATE_REQUESTS =
+ new ContainerUpdates();
@Before
public void setUp() throws IOException {
@@ -1257,7 +1262,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
asks.add(createResourceRequest(2048, node2.getRackName(), 1, 1, false));
scheduler.allocate(attemptId, asks, new ArrayList<ContainerId>(), null,
- null, null, null);
+ null, NULL_UPDATE_REQUESTS);
ApplicationAttemptId attId = createSchedulingRequest(2048, "queue1", "user1", 1);
scheduler.update();
@@ -2111,7 +2116,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ResourceRequest request1 =
createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1, true);
ask1.add(request1);
- scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null, null, null, null);
+ scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(),
+ null, null, NULL_UPDATE_REQUESTS);
// Second ask, queue2 requests 1 large.
List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
@@ -2121,7 +2127,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ResourceRequest.ANY, 1, 1, false);
ask2.add(request2);
ask2.add(request3);
- scheduler.allocate(id21, ask2, new ArrayList<ContainerId>(), null, null, null, null);
+ scheduler.allocate(id21, ask2, new ArrayList<ContainerId>(),
+ null, null, NULL_UPDATE_REQUESTS);
// Third ask, queue2 requests 2 small (minReqSize).
List<ResourceRequest> ask3 = new ArrayList<ResourceRequest>();
@@ -2131,7 +2138,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ResourceRequest.ANY, 2, 2, true);
ask3.add(request4);
ask3.add(request5);
- scheduler.allocate(id22, ask3, new ArrayList<ContainerId>(), null, null, null, null);
+ scheduler.allocate(id22, ask3, new ArrayList<ContainerId>(),
+ null, null, NULL_UPDATE_REQUESTS);
scheduler.update();
@@ -2665,7 +2673,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Complete container
scheduler.allocate(attId, new ArrayList<ResourceRequest>(),
- Arrays.asList(containerId), null, null, null, null);
+ Arrays.asList(containerId), null, null, NULL_UPDATE_REQUESTS);
assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB());
assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
@@ -2757,7 +2765,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
asks.add(createResourceRequest(1024, ResourceRequest.ANY, 1, 2, true));
scheduler.allocate(attemptId, asks, new ArrayList<ContainerId>(), null,
- null, null, null);
+ null, NULL_UPDATE_REQUESTS);
// node 1 checks in
scheduler.update();
@@ -3203,7 +3211,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
createResourceRequest(1024, node1.getHostName(), 1, 0, true),
createResourceRequest(1024, "rack1", 1, 0, true),
createResourceRequest(1024, ResourceRequest.ANY, 1, 1, true));
- scheduler.allocate(attId1, update, new ArrayList<ContainerId>(), null, null, null, null);
+ scheduler.allocate(attId1, update, new ArrayList<ContainerId>(),
+ null, null, NULL_UPDATE_REQUESTS);
// then node2 should get the container
scheduler.handle(node2UpdateEvent);
@@ -3250,7 +3259,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
anyRequest = createResourceRequest(1024, ResourceRequest.ANY,
1, 1, false);
scheduler.allocate(attId, Arrays.asList(rackRequest, anyRequest),
- new ArrayList<ContainerId>(), null, null, null, null);
+ new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
scheduler.handle(nodeUpdateEvent);
assertEquals(0, app.getReservedContainers().size());
@@ -4275,7 +4284,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ask1.add(request1);
scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null,
- null, null, null);
+ null, NULL_UPDATE_REQUESTS);
String hostName = "127.0.0.1";
RMNode node1 = MockNodes.newNodeInfo(1,
@@ -4351,11 +4360,11 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify the blacklist can be updated independent of requesting containers
scheduler.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(),
- Collections.singletonList(host), null, null, null);
+ Collections.singletonList(host), null, NULL_UPDATE_REQUESTS);
assertTrue(app.isPlaceBlacklisted(host));
scheduler.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(), null,
- Collections.singletonList(host), null, null);
+ Collections.singletonList(host), NULL_UPDATE_REQUESTS);
assertFalse(scheduler.getSchedulerApp(appAttemptId)
.isPlaceBlacklisted(host));
@@ -4365,7 +4374,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify a container does not actually get placed on the blacklisted host
scheduler.allocate(appAttemptId, update,
Collections.<ContainerId>emptyList(),
- Collections.singletonList(host), null, null, null);
+ Collections.singletonList(host), null, NULL_UPDATE_REQUESTS);
assertTrue(app.isPlaceBlacklisted(host));
scheduler.update();
scheduler.handle(updateEvent);
@@ -4375,7 +4384,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify a container gets placed on the empty blacklist
scheduler.allocate(appAttemptId, update,
Collections.<ContainerId>emptyList(), null,
- Collections.singletonList(host), null, null);
+ Collections.singletonList(host), NULL_UPDATE_REQUESTS);
assertFalse(app.isPlaceBlacklisted(host));
createSchedulingRequest(GB, "root.default", "user", 1);
scheduler.update();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a55bd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index bfbc7bd..3f97b59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
@@ -119,7 +120,10 @@ public class TestFifoScheduler {
private static Configuration conf;
private static final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
-
+
+ private final static ContainerUpdates NULL_UPDATE_REQUESTS =
+ new ContainerUpdates();
+
@Before
public void setUp() throws Exception {
conf = new Configuration();
@@ -274,7 +278,8 @@ public class TestFifoScheduler {
ask.add(nodeLocal);
ask.add(rackLocal);
ask.add(any);
- scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, null, null);
+ scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(),
+ null, null, NULL_UPDATE_REQUESTS);
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
@@ -368,7 +373,8 @@ public class TestFifoScheduler {
ask.add(nodeLocal);
ask.add(rackLocal);
ask.add(any);
- scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, null, null);
+ scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(),
+ null, null, NULL_UPDATE_REQUESTS);
// Before the node update event, there are one local request
Assert.assertEquals(1, nodeLocal.getNumContainers());
@@ -944,7 +950,7 @@ public class TestFifoScheduler {
ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1,
RMNodeLabelsManager.NO_LABEL));
fs.allocate(appAttemptId1, ask1, emptyId,
- Collections.singletonList(host_1_0), null, null, null);
+ Collections.singletonList(host_1_0), null, NULL_UPDATE_REQUESTS);
// Trigger container assignment
fs.handle(new NodeUpdateSchedulerEvent(n3));
@@ -952,14 +958,16 @@ public class TestFifoScheduler {
// Get the allocation for the application and verify no allocation on
// blacklist node
Allocation allocation1 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId,
+ null, null, NULL_UPDATE_REQUESTS);
Assert.assertEquals("allocation1", 0, allocation1.getContainers().size());
// verify host_1_1 can get allocated as not in blacklist
fs.handle(new NodeUpdateSchedulerEvent(n4));
Allocation allocation2 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId,
+ null, null, NULL_UPDATE_REQUESTS);
Assert.assertEquals("allocation2", 1, allocation2.getContainers().size());
List<Container> containerList = allocation2.getContainers();
for (Container container : containerList) {
@@ -974,29 +982,33 @@ public class TestFifoScheduler {
ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
fs.allocate(appAttemptId1, ask2, emptyId,
- Collections.singletonList("rack0"), null, null, null);
+ Collections.singletonList("rack0"), null, NULL_UPDATE_REQUESTS);
// verify n1 is not qualified to be allocated
fs.handle(new NodeUpdateSchedulerEvent(n1));
Allocation allocation3 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId,
+ null, null, NULL_UPDATE_REQUESTS);
Assert.assertEquals("allocation3", 0, allocation3.getContainers().size());
// verify n2 is not qualified to be allocated
fs.handle(new NodeUpdateSchedulerEvent(n2));
Allocation allocation4 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId,
+ null, null, NULL_UPDATE_REQUESTS);
Assert.assertEquals("allocation4", 0, allocation4.getContainers().size());
// verify n3 is not qualified to be allocated
fs.handle(new NodeUpdateSchedulerEvent(n3));
Allocation allocation5 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId,
+ null, null, NULL_UPDATE_REQUESTS);
Assert.assertEquals("allocation5", 0, allocation5.getContainers().size());
fs.handle(new NodeUpdateSchedulerEvent(n4));
Allocation allocation6 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId,
+ null, null, NULL_UPDATE_REQUESTS);
Assert.assertEquals("allocation6", 1, allocation6.getContainers().size());
containerList = allocation6.getContainers();
@@ -1055,25 +1067,29 @@ public class TestFifoScheduler {
List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
- fs.allocate(appAttemptId1, ask1, emptyId, null, null, null, null);
+ fs.allocate(appAttemptId1, ask1, emptyId,
+ null, null, NULL_UPDATE_REQUESTS);
// Ask for a 2 GB container for app 2
List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
- fs.allocate(appAttemptId2, ask2, emptyId, null, null, null, null);
+ fs.allocate(appAttemptId2, ask2, emptyId,
+ null, null, NULL_UPDATE_REQUESTS);
// Trigger container assignment
fs.handle(new NodeUpdateSchedulerEvent(n1));
// Get the allocation for the applications and verify headroom
Allocation allocation1 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId,
+ null, null, NULL_UPDATE_REQUESTS);
Assert.assertEquals("Allocation headroom", 1 * GB, allocation1
.getResourceLimit().getMemorySize());
Allocation allocation2 =
- fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null, null, null);
+ fs.allocate(appAttemptId2, emptyAsk, emptyId,
+ null, null, NULL_UPDATE_REQUESTS);
Assert.assertEquals("Allocation headroom", 1 * GB, allocation2
.getResourceLimit().getMemorySize());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-6079. Fix simple spelling
errors in yarn test code. Contributed by vijay.
Posted by st...@apache.org.
YARN-6079. Fix simple spelling errors in yarn test code. Contributed by vijay.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4db119b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4db119b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4db119b7
Branch: refs/heads/HADOOP-13345
Commit: 4db119b7b55213929e5b86f2abb0ed84a21719b5
Parents: e692316
Author: Junping Du <ju...@apache.org>
Authored: Tue Jan 10 18:20:27 2017 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Tue Jan 10 18:20:27 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java | 2 +-
.../java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java | 2 +-
.../hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java | 2 +-
.../hadoop/yarn/server/nodemanager/TestNodeHealthService.java | 2 +-
.../localizer/TestResourceLocalizationService.java | 2 +-
.../org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java | 4 ++--
.../yarn/server/resourcemanager/TestRMNodeTransitions.java | 2 +-
.../server/resourcemanager/scheduler/capacity/TestLeafQueue.java | 2 +-
.../hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java | 2 +-
.../resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java | 2 +-
10 files changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db119b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 8772fdc..8b1bbc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -247,7 +247,7 @@ public class TestAMRMClient {
Resource testCapability1 = Resource.newInstance(1024, 2);
List<? extends Collection<ContainerRequest>> matches =
amClient.getMatchingRequests(priority, node, testCapability1);
- assertEquals("Expected no macthing requests.", matches.size(), 0);
+ assertEquals("Expected no matching requests.", matches.size(), 0);
}
@Test (timeout=60000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db119b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
index 21a737d..82170b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
@@ -222,7 +222,7 @@ public class BasePBImplRecordsTest {
GetSetPair gsp = cur.getValue();
if ((gsp.getMethod == null) ||
(gsp.setMethod == null)) {
- LOG.info(String.format("Exclude protential property: %s\n", gsp.propertyName));
+ LOG.info(String.format("Exclude potential property: %s\n", gsp.propertyName));
itr.remove();
} else {
LOG.info(String.format("New property: %s type: %s", gsp.toString(), gsp.type));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db119b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
index 35bad4d..54e331b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
@@ -155,7 +155,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
} catch (IOException e) {
caught = true;
}
- Assert.assertTrue("invalid label charactor should not add to repo", caught);
+ Assert.assertTrue("invalid label character should not add to repo", caught);
caught = false;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db119b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
index 2e41dea..4b01349 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
@@ -163,7 +163,7 @@ public class TestNodeHealthService {
LOG.info("Checking Healthy--->timeout");
Assert.assertFalse("Node health status reported healthy even after timeout",
healthStatus.getIsNodeHealthy());
- Assert.assertTrue("Node script time out message not propogated",
+ Assert.assertTrue("Node script time out message not propagated",
healthStatus.getHealthReport().equals(
NodeHealthScriptRunner.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG
+ NodeHealthCheckerService.SEPARATOR
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db119b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index 42479ea..41cccc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -2102,7 +2102,7 @@ public class TestResourceLocalizationService {
Assert.assertEquals(userCachePath, destinationDirectory.getParent()
.toUri().toString());
} else {
- throw new Exception("Unexpected resource recevied.");
+ throw new Exception("Unexpected resource received.");
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db119b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
index 15b8f04..f807217 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
@@ -624,7 +624,7 @@ public class TestRMHA {
HAServiceProtocol.RequestSource.REQUEST_BY_USER);
FailFastDispatcher dispatcher =
((FailFastDispatcher) rm.rmContext.getDispatcher());
- // Verify transistion to transitionToStandby
+ // Verify transition to transitionToStandby
rm.adminService.transitionToStandby(requestInfo);
assertEquals("Fatal Event should be 0", 0, dispatcher.getEventCount());
assertEquals("HA state should be in standBy State", HAServiceState.STANDBY,
@@ -632,7 +632,7 @@ public class TestRMHA {
try {
// Verify refreshAll call failure and check fail Event is dispatched
rm.adminService.transitionToActive(requestInfo);
- Assert.fail("Transistion to Active should have failed for refreshAll()");
+ Assert.fail("Transition to Active should have failed for refreshAll()");
} catch (Exception e) {
assertTrue("Service fail Exception expected",
e instanceof ServiceFailedException);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db119b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index 6055afb..ba806ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -917,7 +917,7 @@ public class TestRMNodeTransitions {
Assert.assertEquals(NodeState.REBOOTED, node.getState());
Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
- Assert.assertEquals("Unhelathy Nodes", initialUnHealthy,
+ Assert.assertEquals("Unhealthy Nodes", initialUnHealthy,
cm.getUnhealthyNMs());
Assert.assertEquals("Decommissioning Nodes", initialDecommissioning,
cm.getNumDecommissioningNMs());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db119b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index b2695bc..0b5578f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -827,7 +827,7 @@ public class TestLeafQueue {
assertTrue("Verify user_1 got resources ", queueUser1.getUsed()
.getMemorySize() > 0);
assertTrue(
- "Exepected AbsoluteUsedCapacity > 0.95, got: "
+ "Expected AbsoluteUsedCapacity > 0.95, got: "
+ b.getAbsoluteUsedCapacity(), b.getAbsoluteUsedCapacity() > 0.95);
// Verify consumedRatio is based on dominant resources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db119b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index df8d67c..03fc206 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -268,7 +268,7 @@ public class TestRMWebApp {
when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
.thenReturn(response);
} catch (YarnException e) {
- Assert.fail("Exception is not expteced.");
+ Assert.fail("Exception is not expected.");
}
return clientRMService;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db119b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java
index 25a712c..c8afd4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java
@@ -156,7 +156,7 @@ public class TestRMWebServiceAppsNodelabel extends JerseyTestBase {
assertEquals("incorrect number of elements", 1, apps.length());
try {
apps.getJSONArray("app").getJSONObject(0).getJSONObject("resourceInfo");
- fail("resourceInfo object shouldnt be available for finished apps");
+ fail("resourceInfo object shouldn't be available for finished apps");
} catch (Exception e) {
assertTrue("resourceInfo shouldn't be available for finished apps",
true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HADOOP-13953. Make FTPFileSystem's
data connection mode and transfer mode configurable. Contributed by Xiao
Chen.
Posted by st...@apache.org.
HADOOP-13953. Make FTPFileSystem's data connection mode and transfer mode configurable. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a212a40
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a212a40
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a212a40
Branch: refs/heads/HADOOP-13345
Commit: 0a212a40fcbd12a11294bff7a31e7433111733c9
Parents: 91bf504
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Jan 9 15:18:26 2017 -0800
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Jan 9 15:18:26 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 70 +++++++++++++++++++-
.../src/main/resources/core-default.xml | 18 +++++
.../conf/TestCommonConfigurationFields.java | 2 +
.../apache/hadoop/fs/ftp/TestFTPFileSystem.java | 56 +++++++++++++++-
4 files changed, 143 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a212a40/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
index 28b07e8..25fec31 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
@@ -23,6 +23,7 @@ import java.io.InputStream;
import java.net.ConnectException;
import java.net.URI;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -65,6 +66,9 @@ public class FTPFileSystem extends FileSystem {
public static final String FS_FTP_HOST = "fs.ftp.host";
public static final String FS_FTP_HOST_PORT = "fs.ftp.host.port";
public static final String FS_FTP_PASSWORD_PREFIX = "fs.ftp.password.";
+ public static final String FS_FTP_DATA_CONNECTION_MODE =
+ "fs.ftp.data.connection.mode";
+ public static final String FS_FTP_TRANSFER_MODE = "fs.ftp.transfer.mode";
public static final String E_SAME_DIRECTORY_ONLY =
"only same directory renames are supported";
@@ -143,9 +147,10 @@ public class FTPFileSystem extends FileSystem {
NetUtils.UNKNOWN_HOST, 0,
new ConnectException("Server response " + reply));
} else if (client.login(user, password)) {
- client.setFileTransferMode(FTP.BLOCK_TRANSFER_MODE);
+ client.setFileTransferMode(getTransferMode(conf));
client.setFileType(FTP.BINARY_FILE_TYPE);
client.setBufferSize(DEFAULT_BUFFER_SIZE);
+ setDataConnectionMode(client, conf);
} else {
throw new IOException("Login failed on server - " + host + ", port - "
+ port + " as user '" + user + "'");
@@ -155,6 +160,69 @@ public class FTPFileSystem extends FileSystem {
}
/**
+ * Set FTP's transfer mode based on configuration. Valid values are
+ * STREAM_TRANSFER_MODE, BLOCK_TRANSFER_MODE and COMPRESSED_TRANSFER_MODE.
+ * <p/>
+ * Defaults to BLOCK_TRANSFER_MODE.
+ *
+ * @param conf
+ * @return
+ */
+ @VisibleForTesting
+ int getTransferMode(Configuration conf) {
+ final String mode = conf.get(FS_FTP_TRANSFER_MODE);
+ // FTP default is STREAM_TRANSFER_MODE, but Hadoop FTPFS's default is
+ // FTP.BLOCK_TRANSFER_MODE historically.
+ int ret = FTP.BLOCK_TRANSFER_MODE;
+ if (mode == null) {
+ return ret;
+ }
+ final String upper = mode.toUpperCase();
+ if (upper.equals("STREAM_TRANSFER_MODE")) {
+ ret = FTP.STREAM_TRANSFER_MODE;
+ } else if (upper.equals("COMPRESSED_TRANSFER_MODE")) {
+ ret = FTP.COMPRESSED_TRANSFER_MODE;
+ } else {
+ if (!upper.equals("BLOCK_TRANSFER_MODE")) {
+ LOG.warn("Cannot parse the value for " + FS_FTP_TRANSFER_MODE + ": "
+ + mode + ". Using default.");
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Set the FTPClient's data connection mode based on configuration. Valid
+ * values are ACTIVE_LOCAL_DATA_CONNECTION_MODE,
+ * PASSIVE_LOCAL_DATA_CONNECTION_MODE and PASSIVE_REMOTE_DATA_CONNECTION_MODE.
+ * <p/>
+ * Defaults to ACTIVE_LOCAL_DATA_CONNECTION_MODE.
+ *
+ * @param client
+ * @param conf
+ * @throws IOException
+ */
+ @VisibleForTesting
+ void setDataConnectionMode(FTPClient client, Configuration conf)
+ throws IOException {
+ final String mode = conf.get(FS_FTP_DATA_CONNECTION_MODE);
+ if (mode == null) {
+ return;
+ }
+ final String upper = mode.toUpperCase();
+ if (upper.equals("PASSIVE_LOCAL_DATA_CONNECTION_MODE")) {
+ client.enterLocalPassiveMode();
+ } else if (upper.equals("PASSIVE_REMOTE_DATA_CONNECTION_MODE")) {
+ client.enterRemotePassiveMode();
+ } else {
+ if (!upper.equals("ACTIVE_LOCAL_DATA_CONNECTION_MODE")) {
+ LOG.warn("Cannot parse the value for " + FS_FTP_DATA_CONNECTION_MODE
+ + ": " + mode + ". Using default.");
+ }
+ }
+ }
+
+ /**
* Logout and disconnect the given FTPClient. *
*
* @param client
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a212a40/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index ee2cc2e..9355a3c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -783,6 +783,24 @@
</property>
<property>
+ <name>fs.ftp.data.connection.mode</name>
+ <value>ACTIVE_LOCAL_DATA_CONNECTION_MODE</value>
+ <description>Set the FTPClient's data connection mode based on configuration.
+ Valid values are ACTIVE_LOCAL_DATA_CONNECTION_MODE,
+ PASSIVE_LOCAL_DATA_CONNECTION_MODE and PASSIVE_REMOTE_DATA_CONNECTION_MODE.
+ </description>
+</property>
+
+<property>
+ <name>fs.ftp.transfer.mode</name>
+ <value>BLOCK_TRANSFER_MODE</value>
+ <description>
+ Set FTP's transfer mode based on configuration. Valid values are
+ STREAM_TRANSFER_MODE, BLOCK_TRANSFER_MODE and COMPRESSED_TRANSFER_MODE.
+ </description>
+</property>
+
+<property>
<name>fs.df.interval</name>
<value>60000</value>
<description>Disk usage statistics refresh interval in msec.</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a212a40/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index a3a4026..966a8ac 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -89,6 +89,8 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
// Lots of properties not in the above classes
xmlPropsToSkipCompare.add("fs.ftp.password.localhost");
xmlPropsToSkipCompare.add("fs.ftp.user.localhost");
+ xmlPropsToSkipCompare.add("fs.ftp.data.connection.mode");
+ xmlPropsToSkipCompare.add("fs.ftp.transfer.mode");
xmlPropsToSkipCompare.add("hadoop.tmp.dir");
xmlPropsToSkipCompare.add("nfs3.mountd.port");
xmlPropsToSkipCompare.add("nfs3.server.port");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a212a40/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ftp/TestFTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ftp/TestFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ftp/TestFTPFileSystem.java
index 0ce2a9b..0604604 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ftp/TestFTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ftp/TestFTPFileSystem.java
@@ -19,15 +19,67 @@ package org.apache.hadoop.fs.ftp;
import org.apache.commons.net.ftp.FTP;
-import org.junit.Assert;
+import org.apache.commons.net.ftp.FTPClient;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.Timeout;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test basic @{link FTPFileSystem} class methods. Contract tests are in
+ * TestFTPContractXXXX.
+ */
public class TestFTPFileSystem {
+ @Rule
+ public Timeout testTimeout = new Timeout(180000);
+
@Test
public void testFTPDefaultPort() throws Exception {
FTPFileSystem ftp = new FTPFileSystem();
- Assert.assertEquals(FTP.DEFAULT_PORT, ftp.getDefaultPort());
+ assertEquals(FTP.DEFAULT_PORT, ftp.getDefaultPort());
}
+ @Test
+ public void testFTPTransferMode() throws Exception {
+ Configuration conf = new Configuration();
+ FTPFileSystem ftp = new FTPFileSystem();
+ assertEquals(FTP.BLOCK_TRANSFER_MODE, ftp.getTransferMode(conf));
+
+ conf.set(FTPFileSystem.FS_FTP_TRANSFER_MODE, "STREAM_TRANSFER_MODE");
+ assertEquals(FTP.STREAM_TRANSFER_MODE, ftp.getTransferMode(conf));
+
+ conf.set(FTPFileSystem.FS_FTP_TRANSFER_MODE, "COMPRESSED_TRANSFER_MODE");
+ assertEquals(FTP.COMPRESSED_TRANSFER_MODE, ftp.getTransferMode(conf));
+
+ conf.set(FTPFileSystem.FS_FTP_TRANSFER_MODE, "invalid");
+ assertEquals(FTPClient.BLOCK_TRANSFER_MODE, ftp.getTransferMode(conf));
+ }
+
+ @Test
+ public void testFTPDataConnectionMode() throws Exception {
+ Configuration conf = new Configuration();
+ FTPClient client = new FTPClient();
+ FTPFileSystem ftp = new FTPFileSystem();
+ assertEquals(FTPClient.ACTIVE_LOCAL_DATA_CONNECTION_MODE,
+ client.getDataConnectionMode());
+
+ ftp.setDataConnectionMode(client, conf);
+ assertEquals(FTPClient.ACTIVE_LOCAL_DATA_CONNECTION_MODE,
+ client.getDataConnectionMode());
+
+ conf.set(FTPFileSystem.FS_FTP_DATA_CONNECTION_MODE, "invalid");
+ ftp.setDataConnectionMode(client, conf);
+ assertEquals(FTPClient.ACTIVE_LOCAL_DATA_CONNECTION_MODE,
+ client.getDataConnectionMode());
+
+ conf.set(FTPFileSystem.FS_FTP_DATA_CONNECTION_MODE,
+ "PASSIVE_LOCAL_DATA_CONNECTION_MODE");
+ ftp.setDataConnectionMode(client, conf);
+ assertEquals(FTPClient.PASSIVE_LOCAL_DATA_CONNECTION_MODE,
+ client.getDataConnectionMode());
+
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-4148. When killing app,
RM releases app's resource before they are released by NM.
Contributed by Jason Lowe.
Posted by st...@apache.org.
YARN-4148. When killing app, RM releases app's resource before they are released by NM. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/945db55f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/945db55f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/945db55f
Branch: refs/heads/HADOOP-13345
Commit: 945db55f2e6521d33d4f90bbb09179b0feba5e7a
Parents: 7ec609b
Author: Junping Du <ju...@apache.org>
Authored: Mon Jan 9 18:14:46 2017 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Mon Jan 9 18:14:46 2017 -0800
----------------------------------------------------------------------
.../scheduler/AbstractYarnScheduler.java | 8 +-
.../scheduler/SchedulerNode.java | 82 +++++++++++--
.../scheduler/capacity/LeafQueue.java | 2 +-
.../common/fica/FiCaSchedulerNode.java | 4 +-
.../scheduler/fair/FairScheduler.java | 2 +-
.../scheduler/fifo/FifoScheduler.java | 2 +-
.../scheduler/TestAbstractYarnScheduler.java | 114 +++++++++++++++++++
.../scheduler/capacity/TestChildQueueOrder.java | 7 +-
8 files changed, 201 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/945db55f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 9ff0b24..64427b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -280,6 +280,7 @@ public abstract class AbstractYarnScheduler
}
application.containerLaunchedOnNode(containerId, node.getNodeID());
+ node.containerStarted(containerId);
} finally {
readLock.unlock();
}
@@ -607,7 +608,7 @@ public abstract class AbstractYarnScheduler
" in state: " + rmContainer.getState() + " event:" + event);
}
getSchedulerNode(rmContainer.getNodeId()).releaseContainer(
- rmContainer.getContainer());
+ rmContainer.getContainerId(), false);
}
// If the container is getting killed in ACQUIRED state, the requester (AM
@@ -941,6 +942,7 @@ public abstract class AbstractYarnScheduler
protected int updateCompletedContainers(List<ContainerStatus>
completedContainers, Resource releasedResources, NodeId nodeId) {
int releasedContainers = 0;
+ SchedulerNode node = getNode(nodeId);
List<ContainerId> untrackedContainerIdList = new ArrayList<ContainerId>();
for (ContainerStatus completedContainer : completedContainers) {
ContainerId containerId = completedContainer.getContainerId();
@@ -948,6 +950,10 @@ public abstract class AbstractYarnScheduler
RMContainer container = getRMContainer(containerId);
completedContainer(container,
completedContainer, RMContainerEventType.FINISHED);
+ if (node != null) {
+ node.releaseContainer(containerId, true);
+ }
+
if (container != null) {
releasedContainers++;
Resource ars = container.getAllocatedResource();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/945db55f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 759db05..15fd830 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -66,7 +66,7 @@ public abstract class SchedulerNode {
ResourceUtilization.newInstance(0, 0, 0f);
/* set of containers that are allocated containers */
- protected final Map<ContainerId, RMContainer> launchedContainers =
+ private final Map<ContainerId, ContainerInfo> launchedContainers =
new HashMap<>();
private final RMNode rmNode;
@@ -148,14 +148,26 @@ public abstract class SchedulerNode {
* application.
* @param rmContainer Allocated container
*/
- public synchronized void allocateContainer(RMContainer rmContainer) {
+ public void allocateContainer(RMContainer rmContainer) {
+ allocateContainer(rmContainer, false);
+ }
+
+ /**
+ * The Scheduler has allocated containers on this node to the given
+ * application.
+ * @param rmContainer Allocated container
+ * @param launchedOnNode True if the container has been launched
+ */
+ private synchronized void allocateContainer(RMContainer rmContainer,
+ boolean launchedOnNode) {
Container container = rmContainer.getContainer();
if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
deductUnallocatedResource(container.getResource());
++numContainers;
}
- launchedContainers.put(container.getId(), rmContainer);
+ launchedContainers.put(container.getId(),
+ new ContainerInfo(rmContainer, launchedOnNode));
if (LOG.isDebugEnabled()) {
LOG.debug("Assigned container " + container.getId() + " of capacity "
@@ -258,19 +270,25 @@ public abstract class SchedulerNode {
/**
* Release an allocated container on this node.
- * @param container Container to be released.
+ * @param containerId ID of container to be released.
+ * @param releasedByNode whether the release originates from a node update.
*/
- public synchronized void releaseContainer(Container container) {
- if (!isValidContainer(container.getId())) {
- LOG.error("Invalid container released " + container);
+ public synchronized void releaseContainer(ContainerId containerId,
+ boolean releasedByNode) {
+ ContainerInfo info = launchedContainers.get(containerId);
+ if (info == null) {
return;
}
- // Remove the containers from the nodemanger
- if (null != launchedContainers.remove(container.getId())) {
- updateResourceForReleasedContainer(container);
+ if (!releasedByNode && info.launchedOnNode) {
+ // wait until node reports container has completed
+ return;
}
+ launchedContainers.remove(containerId);
+ Container container = info.container.getContainer();
+ updateResourceForReleasedContainer(container);
+
if (LOG.isDebugEnabled()) {
LOG.debug("Released container " + container.getId() + " of capacity "
+ container.getResource() + " on host " + rmNode.getNodeAddress()
@@ -281,6 +299,17 @@ public abstract class SchedulerNode {
}
/**
+ * Inform the node that a container has launched.
+ * @param containerId ID of the launched container
+ */
+ public synchronized void containerStarted(ContainerId containerId) {
+ ContainerInfo info = launchedContainers.get(containerId);
+ if (info != null) {
+ info.launchedOnNode = true;
+ }
+ }
+
+ /**
* Add unallocated resources to the node. This is used when unallocating a
* container.
* @param resource Resources to add.
@@ -345,7 +374,25 @@ public abstract class SchedulerNode {
* @return List of running containers in the node.
*/
public synchronized List<RMContainer> getCopiedListOfRunningContainers() {
- return new ArrayList<RMContainer>(launchedContainers.values());
+ List<RMContainer> result = new ArrayList<>(launchedContainers.size());
+ for (ContainerInfo info : launchedContainers.values()) {
+ result.add(info.container);
+ }
+ return result;
+ }
+
+ /**
+ * Get the container for the specified container ID.
+ * @param containerId The container ID
+ * @return The container for the specified container ID
+ */
+ protected synchronized RMContainer getContainer(ContainerId containerId) {
+ RMContainer container = null;
+ ContainerInfo info = launchedContainers.get(containerId);
+ if (info != null) {
+ container = info.container;
+ }
+ return container;
}
/**
@@ -373,7 +420,7 @@ public abstract class SchedulerNode {
if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
return;
}
- allocateContainer(rmContainer);
+ allocateContainer(rmContainer, true);
}
/**
@@ -438,4 +485,15 @@ public abstract class SchedulerNode {
public ResourceUtilization getNodeUtilization() {
return this.nodeUtilization;
}
+
+
+ private static class ContainerInfo {
+ private final RMContainer container;
+ private boolean launchedOnNode;
+
+ public ContainerInfo(RMContainer container, boolean launchedOnNode) {
+ this.container = container;
+ this.launchedOnNode = launchedOnNode;
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/945db55f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 6d4dbe0..2df9407 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1703,7 +1703,7 @@ public class LeafQueue extends AbstractCSQueue {
removed = application.containerCompleted(rmContainer, containerStatus,
event, node.getPartition());
- node.releaseContainer(container);
+ node.releaseContainer(rmContainer.getContainerId(), false);
}
// Book-keeping
http://git-wip-us.apache.org/repos/asf/hadoop/blob/945db55f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
index 344daf2..8eac929 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
@@ -126,7 +126,7 @@ public class FiCaSchedulerNode extends SchedulerNode {
// According to decisions from preemption policy, mark the container to killable
public synchronized void markContainerToKillable(ContainerId containerId) {
- RMContainer c = launchedContainers.get(containerId);
+ RMContainer c = getContainer(containerId);
if (c != null && !killableContainers.containsKey(containerId)) {
killableContainers.put(containerId, c);
Resources.addTo(totalKillableResources, c.getAllocatedResource());
@@ -136,7 +136,7 @@ public class FiCaSchedulerNode extends SchedulerNode {
// According to decisions from preemption policy, mark the container to
// non-killable
public synchronized void markContainerToNonKillable(ContainerId containerId) {
- RMContainer c = launchedContainers.get(containerId);
+ RMContainer c = getContainer(containerId);
if (c != null && killableContainers.containsKey(containerId)) {
killableContainers.remove(containerId);
Resources.subtractFrom(totalKillableResources, c.getAllocatedResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/945db55f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 6bbccc6..97871e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -719,7 +719,7 @@ public class FairScheduler extends
application.unreserve(rmContainer.getReservedSchedulerKey(), node);
} else{
application.containerCompleted(rmContainer, containerStatus, event);
- node.releaseContainer(container);
+ node.releaseContainer(rmContainer.getContainerId(), false);
updateRootQueueMetrics();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/945db55f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index d52f538..63e7f40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -843,7 +843,7 @@ public class FifoScheduler extends
RMNodeLabelsManager.NO_LABEL);
// Inform the node
- node.releaseContainer(container);
+ node.releaseContainer(rmContainer.getContainerId(), false);
// Update total usage
Resources.subtractFrom(usedResource, container.getResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/945db55f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index b32f138..778d2a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
@@ -424,6 +425,119 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
}
}
+ @Test(timeout=60000)
+ public void testContainerReleasedByNode() throws Exception {
+ System.out.println("Starting testContainerReleasedByNode");
+ configureScheduler();
+ YarnConfiguration conf = getConf();
+ MockRM rm1 = new MockRM(conf);
+ try {
+ rm1.start();
+ RMApp app1 =
+ rm1.submitApp(200, "name", "user",
+ new HashMap<ApplicationAccessType, String>(), false, "default",
+ -1, null, "Test", false, true);
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // allocate a container that fills more than half the node
+ am1.allocate("127.0.0.1", 8192, 1, new ArrayList<ContainerId>());
+ nm1.nodeHeartbeat(true);
+
+ // wait for containers to be allocated.
+ List<Container> containers =
+ am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ while (containers.isEmpty()) {
+ Thread.sleep(10);
+ nm1.nodeHeartbeat(true);
+ containers = am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ }
+
+ // release the container from the AM
+ ContainerId cid = containers.get(0).getId();
+ List<ContainerId> releasedContainers = new ArrayList<>(1);
+ releasedContainers.add(cid);
+ List<ContainerStatus> completedContainers = am1.allocate(
+ new ArrayList<ResourceRequest>(), releasedContainers)
+ .getCompletedContainersStatuses();
+ while (completedContainers.isEmpty()) {
+ Thread.sleep(10);
+ completedContainers = am1.allocate(
+ new ArrayList<ResourceRequest>(), releasedContainers)
+ .getCompletedContainersStatuses();
+ }
+
+ // verify new container can be allocated immediately because container
+ // never launched on the node
+ containers = am1.allocate("127.0.0.1", 8192, 1,
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ nm1.nodeHeartbeat(true);
+ while (containers.isEmpty()) {
+ Thread.sleep(10);
+ nm1.nodeHeartbeat(true);
+ containers = am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ }
+
+ // launch the container on the node
+ cid = containers.get(0).getId();
+ nm1.nodeHeartbeat(cid.getApplicationAttemptId(), cid.getContainerId(),
+ ContainerState.RUNNING);
+ rm1.waitForState(nm1, cid, RMContainerState.RUNNING);
+
+ // release the container from the AM
+ releasedContainers.clear();
+ releasedContainers.add(cid);
+ completedContainers = am1.allocate(
+ new ArrayList<ResourceRequest>(), releasedContainers)
+ .getCompletedContainersStatuses();
+ while (completedContainers.isEmpty()) {
+ Thread.sleep(10);
+ completedContainers = am1.allocate(
+ new ArrayList<ResourceRequest>(), releasedContainers)
+ .getCompletedContainersStatuses();
+ }
+
+ // verify new container cannot be allocated immediately because container
+ // has not been released by the node
+ containers = am1.allocate("127.0.0.1", 8192, 1,
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ nm1.nodeHeartbeat(true);
+ Assert.assertTrue("new container allocated before node freed old",
+ containers.isEmpty());
+ for (int i = 0; i < 10; ++i) {
+ Thread.sleep(10);
+ containers = am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ nm1.nodeHeartbeat(true);
+ Assert.assertTrue("new container allocated before node freed old",
+ containers.isEmpty());
+ }
+
+ // free the old container from the node
+ nm1.nodeHeartbeat(cid.getApplicationAttemptId(), cid.getContainerId(),
+ ContainerState.COMPLETE);
+
+ // verify new container is now allocated
+ containers = am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ while (containers.isEmpty()) {
+ Thread.sleep(10);
+ nm1.nodeHeartbeat(true);
+ containers = am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ }
+ } finally {
+ rm1.stop();
+ System.out.println("Stopping testContainerReleasedByNode");
+ }
+ }
+
@Test(timeout = 60000)
public void testResourceRequestRestoreWhenRMContainerIsAtAllocated()
throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/945db55f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index cf91841..3c1f676 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
@@ -164,7 +165,8 @@ public class TestChildQueueOrder {
}).
when(queue).assignContainers(eq(clusterResource), any(PlacementSet.class),
any(ResourceLimits.class), any(SchedulingMode.class));
- doNothing().when(node).releaseContainer(any(Container.class));
+ doNothing().when(node).releaseContainer(any(ContainerId.class),
+ anyBoolean());
}
@@ -236,7 +238,8 @@ public class TestChildQueueOrder {
FiCaSchedulerNode node_0 =
TestUtils.getMockNode("host_0", DEFAULT_RACK, 0, memoryPerNode*GB);
- doNothing().when(node_0).releaseContainer(any(Container.class));
+ doNothing().when(node_0).releaseContainer(any(ContainerId.class),
+ anyBoolean());
final Resource clusterResource =
Resources.createResource(numNodes * (memoryPerNode*GB),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org