You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by fa...@apache.org on 2016/11/07 00:54:12 UTC
[01/50] [abbrv] hadoop git commit: HDFS-11030.
TestDataNodeVolumeFailure#testVolumeFailure is flaky (though passing).
Contributed by Mingliang Liu [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 5d280b903 -> 56b715f8e (forced update)
HDFS-11030. TestDataNodeVolumeFailure#testVolumeFailure is flaky (though passing). Contributed by Mingliang Liu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c49f73a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c49f73a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c49f73a
Branch: refs/heads/HADOOP-13345
Commit: 0c49f73a6c19ce0d0cd59cf8dfaa9a35f67f47ab
Parents: 90dd3a8
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Oct 20 13:44:25 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Oct 31 16:00:33 2016 -0700
----------------------------------------------------------------------
.../datanode/TestDataNodeVolumeFailure.java | 64 ++++++++++----------
1 file changed, 33 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c49f73a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
index 9ffe7b6..4aba4e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
@@ -29,6 +29,7 @@ import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -52,7 +53,6 @@ import org.apache.hadoop.hdfs.RemotePeerFactory;
import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -66,14 +66,16 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
-import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import com.google.common.base.Supplier;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -108,6 +110,7 @@ public class TestDataNodeVolumeFailure {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, block_size);
// Allow a single volume failure (there are two volumes)
conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 30);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dn_num).build();
cluster.waitActive();
fs = cluster.getFileSystem();
@@ -135,7 +138,7 @@ public class TestDataNodeVolumeFailure {
* and that we can replicate to both datanodes even after a single volume
* failure if the configuration parameter allows this.
*/
- @Test
+ @Test(timeout = 120000)
public void testVolumeFailure() throws Exception {
System.out.println("Data dir: is " + dataDir.getPath());
@@ -155,7 +158,7 @@ public class TestDataNodeVolumeFailure {
// fail the volume
// delete/make non-writable one of the directories (failed volume)
data_fail = new File(dataDir, "data3");
- failedDir = MiniDFSCluster.getFinalizedDir(dataDir,
+ failedDir = MiniDFSCluster.getFinalizedDir(data_fail,
cluster.getNamesystem().getBlockPoolId());
if (failedDir.exists() &&
//!FileUtil.fullyDelete(failedDir)
@@ -171,29 +174,26 @@ public class TestDataNodeVolumeFailure {
// we need to make sure that the "failed" volume is being accessed -
// and that will cause failure, blocks removal, "emergency" block report
triggerFailure(filename, filesize);
-
- // make sure a block report is sent
- DataNode dn = cluster.getDataNodes().get(1); //corresponds to dir data3
- String bpid = cluster.getNamesystem().getBlockPoolId();
- DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid);
-
- Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
- dn.getFSDataset().getBlockReports(bpid);
-
- // Send block report
- StorageBlockReport[] reports =
- new StorageBlockReport[perVolumeBlockLists.size()];
-
- int reportIndex = 0;
- for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
- DatanodeStorage dnStorage = kvPair.getKey();
- BlockListAsLongs blockList = kvPair.getValue();
- reports[reportIndex++] =
- new StorageBlockReport(dnStorage, blockList);
- }
-
- cluster.getNameNodeRpc().blockReport(dnR, bpid, reports,
- new BlockReportContext(1, 0, System.nanoTime(), 0, true));
+ // DN eventually have latest volume failure information for next heartbeat
+ final DataNode dn = cluster.getDataNodes().get(1);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ final VolumeFailureSummary summary =
+ dn.getFSDataset().getVolumeFailureSummary();
+ return summary != null &&
+ summary.getFailedStorageLocations() != null &&
+ summary.getFailedStorageLocations().length == 1;
+ }
+ }, 10, 30 * 1000);
+
+ // trigger DN to send heartbeat
+ DataNodeTestUtils.triggerHeartbeat(dn);
+ final BlockManager bm = cluster.getNamesystem().getBlockManager();
+ // trigger NN handel heartbeat
+ BlockManagerTestUtil.checkHeartbeat(bm);
+ // NN now should have latest volume failure
+ assertEquals(1, cluster.getNamesystem().getVolumeFailuresTotal());
// verify number of blocks and files...
verify(filename, filesize);
@@ -492,9 +492,11 @@ public class TestDataNodeVolumeFailure {
* @throws IOException
*/
private boolean deteteBlocks(File dir) {
- File [] fileList = dir.listFiles();
+ Collection<File> fileList = FileUtils.listFiles(dir,
+ TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE);
for(File f : fileList) {
if(f.getName().startsWith(Block.BLOCK_FILE_PREFIX)) {
+ System.out.println("Deleting file " + f);
if(!f.delete())
return false;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDFS-10566. Submit plan request
should throw exception if Datanode is in non-REGULAR status. Contributed by
Xiaobing Zhou.
Posted by fa...@apache.org.
HDFS-10566. Submit plan request should throw exception if Datanode is in non-REGULAR status. Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/730cb0cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/730cb0cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/730cb0cf
Branch: refs/heads/HADOOP-13345
Commit: 730cb0cff6a6e2f1a6eef3593568e8a1b5172cf7
Parents: b592061
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Nov 2 17:46:56 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Nov 2 17:46:56 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/DataNode.java | 6 ++
.../diskbalancer/DiskBalancerException.java | 1 +
.../diskbalancer/DiskBalancerTestUtil.java | 19 ++++
.../command/TestDiskBalancerCommand.java | 103 ++++++++++++++-----
4 files changed, 105 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730cb0cf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index f89d38c..de9e48e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -3414,6 +3414,12 @@ public class DataNode extends ReconfigurableBase
String planFile, String planData, boolean skipDateCheck)
throws IOException {
checkSuperuserPrivilege();
+ if (getStartupOption(getConf()) != StartupOption.REGULAR) {
+ throw new DiskBalancerException(
+ "Datanode is in special state, e.g. Upgrade/Rollback etc."
+ + " Disk balancing not permitted.",
+ DiskBalancerException.Result.DATANODE_STATUS_NOT_REGULAR);
+ }
// TODO : Support force option
this.diskBalancer.submitPlan(planID, planVersion, planFile, planData,
skipDateCheck);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730cb0cf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
index 95ff722..a730a57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
@@ -40,6 +40,7 @@ public class DiskBalancerException extends IOException {
NO_SUCH_PLAN,
UNKNOWN_KEY,
INVALID_NODE,
+ DATANODE_STATUS_NOT_REGULAR,
}
private final Result result;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730cb0cf/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
index c60fe21..bc4181b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
@@ -262,6 +263,23 @@ public class DiskBalancerTestUtil {
final int defaultBlockSize,
final int fileLen)
throws IOException, InterruptedException, TimeoutException {
+ return newImbalancedCluster(
+ conf,
+ numDatanodes,
+ storageCapacities,
+ defaultBlockSize,
+ fileLen,
+ null);
+ }
+
+ public static MiniDFSCluster newImbalancedCluster(
+ final Configuration conf,
+ final int numDatanodes,
+ final long[] storageCapacities,
+ final int defaultBlockSize,
+ final int fileLen,
+ final StartupOption dnOption)
+ throws IOException, InterruptedException, TimeoutException {
conf.setBoolean(DFSConfigKeys.DFS_DISK_BALANCER_ENABLED, true);
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultBlockSize);
conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, defaultBlockSize);
@@ -281,6 +299,7 @@ public class DiskBalancerTestUtil {
.storageCapacities(storageCapacities)
.storageTypes(new StorageType[]{StorageType.DISK, StorageType.DISK})
.storagesPerDatanode(2)
+ .dnStartupOption(dnOption)
.build();
FsVolumeImpl source = null;
FsVolumeImpl dest = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730cb0cf/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
index 0f65f25..ad16cfa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerTestUtil;
import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ClusterConnector;
@@ -53,6 +54,7 @@ import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ConnectorFactory;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerDataNode;
import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.util.Tool;
@@ -104,6 +106,50 @@ public class TestDiskBalancerCommand {
}
/**
+ * Tests if it's allowed to submit and execute plan when Datanode is in status
+ * other than REGULAR.
+ */
+ @Test(timeout = 60000)
+ public void testSubmitPlanInNonRegularStatus() throws Exception {
+ final int numDatanodes = 1;
+ MiniDFSCluster miniCluster = null;
+ final Configuration hdfsConf = new HdfsConfiguration();
+
+ try {
+ /* new cluster with imbalanced capacity */
+ miniCluster = DiskBalancerTestUtil.newImbalancedCluster(
+ hdfsConf,
+ numDatanodes,
+ CAPACITIES,
+ DEFAULT_BLOCK_SIZE,
+ FILE_LEN,
+ StartupOption.ROLLBACK);
+
+ /* get full path of plan */
+ final String planFileFullName = runAndVerifyPlan(miniCluster, hdfsConf);
+
+ try {
+ /* run execute command */
+ final String cmdLine = String.format(
+ "hdfs diskbalancer -%s %s",
+ EXECUTE,
+ planFileFullName);
+ runCommand(cmdLine, hdfsConf, miniCluster);
+ } catch(RemoteException e) {
+ assertThat(e.getClassName(), containsString("DiskBalancerException"));
+ assertThat(e.toString(),
+ is(allOf(
+ containsString("Datanode is in special state"),
+ containsString("Disk balancing not permitted."))));
+ }
+ } finally {
+ if (miniCluster != null) {
+ miniCluster.shutdown();
+ }
+ }
+ }
+
+ /**
* Tests running multiple commands under on setup. This mainly covers
* {@link org.apache.hadoop.hdfs.server.diskbalancer.command.Command#close}
*/
@@ -122,36 +168,16 @@ public class TestDiskBalancerCommand {
CAPACITIES,
DEFAULT_BLOCK_SIZE,
FILE_LEN);
- String cmdLine = "";
- List<String> outputs = null;
- final DataNode dn = miniCluster.getDataNodes().get(0);
-
- /* run plan command */
- cmdLine = String.format(
- "hdfs diskbalancer -%s %s",
- PLAN,
- dn.getDatanodeUuid());
- outputs = runCommand(cmdLine, hdfsConf, miniCluster);
- /* get path of plan file*/
- final String planFileName = dn.getDatanodeUuid();
-
- /* verify plan command */
- assertEquals(
- "There must be two lines: the 1st is writing plan to...,"
- + " the 2nd is actual full path of plan file.",
- 2, outputs.size());
- assertThat(outputs.get(1), containsString(planFileName));
-
- /* get full path of plan file*/
- final String planFileFullName = outputs.get(1);
+ /* get full path of plan */
+ final String planFileFullName = runAndVerifyPlan(miniCluster, hdfsConf);
/* run execute command */
- cmdLine = String.format(
+ final String cmdLine = String.format(
"hdfs diskbalancer -%s %s",
EXECUTE,
planFileFullName);
- outputs = runCommand(cmdLine, hdfsConf, miniCluster);
+ runCommand(cmdLine, hdfsConf, miniCluster);
} finally {
if (miniCluster != null) {
miniCluster.shutdown();
@@ -159,6 +185,35 @@ public class TestDiskBalancerCommand {
}
}
+ private String runAndVerifyPlan(
+ final MiniDFSCluster miniCluster,
+ final Configuration hdfsConf) throws Exception {
+ String cmdLine = "";
+ List<String> outputs = null;
+ final DataNode dn = miniCluster.getDataNodes().get(0);
+
+ /* run plan command */
+ cmdLine = String.format(
+ "hdfs diskbalancer -%s %s",
+ PLAN,
+ dn.getDatanodeUuid());
+ outputs = runCommand(cmdLine, hdfsConf, miniCluster);
+
+ /* get path of plan file*/
+ final String planFileName = dn.getDatanodeUuid();
+
+ /* verify plan command */
+ assertEquals(
+ "There must be two lines: the 1st is writing plan to...,"
+ + " the 2nd is actual full path of plan file.",
+ 2, outputs.size());
+ assertThat(outputs.get(1), containsString(planFileName));
+
+ /* get full path of plan file*/
+ final String planFileFullName = outputs.get(1);
+ return planFileFullName;
+ }
+
/* test basic report */
@Test(timeout = 60000)
public void testReportSimple() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HADOOP-13787. Azure
testGlobStatusThrowsExceptionForUnreadableDir fails. Contributed by John
Zhuge.
Posted by fa...@apache.org.
HADOOP-13787. Azure testGlobStatusThrowsExceptionForUnreadableDir fails. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20c4d8ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20c4d8ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20c4d8ef
Branch: refs/heads/HADOOP-13345
Commit: 20c4d8efab425fe3893ef9a5bec85d4a3b64d5a6
Parents: b71907b
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Nov 3 11:15:26 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Nov 3 11:16:07 2016 -0700
----------------------------------------------------------------------
.../fs/azure/TestNativeAzureFileSystemOperationsMocked.java | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20c4d8ef/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemOperationsMocked.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemOperationsMocked.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemOperationsMocked.java
index 69c942d..89ed381 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemOperationsMocked.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemOperationsMocked.java
@@ -52,6 +52,15 @@ public class TestNativeAzureFileSystemOperationsMocked extends
}
@Override
+ public void testGlobStatusThrowsExceptionForUnreadableDir()
+ throws Exception {
+ System.out.println(
+ "Skipping testGlobStatusThrowsExceptionForUnreadableDir since WASB"
+ + " doesn't honor directory permissions.");
+ assumeNotWindows();
+ }
+
+ @Override
public String getTestRootDir() {
return TEST_ROOT_DIR;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: Revert "HADOOP-13717. Normalize
daemonization behavior of the diskbalancer with balancer and mover."
Posted by fa...@apache.org.
Revert "HADOOP-13717. Normalize daemonization behavior of the diskbalancer with balancer and mover."
This reverts commit bd7f5911c726c505a654d5d30b7dc5a92dca05bd.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2800f52
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2800f52
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2800f52
Branch: refs/heads/HADOOP-13345
Commit: f2800f523d4a4f9ff9c3286f9a839d081dcb5b96
Parents: 3779190
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Nov 2 13:46:33 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Nov 2 13:46:33 2016 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2800f52/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 2c59004..2299980 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -116,7 +116,6 @@ function hdfscmd_case
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSAdmin
;;
diskbalancer)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DiskBalancerCLI
;;
envvars)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDFS-11085. Add unit test for
NameNode failing to start when name dir is unwritable. Contributed by
Xiaobing Zhou
Posted by fa...@apache.org.
HDFS-11085. Add unit test for NameNode failing to start when name dir is unwritable. Contributed by Xiaobing Zhou
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c0ab102
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c0ab102
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c0ab102
Branch: refs/heads/HADOOP-13345
Commit: 0c0ab102ab392ba07ed2aa8d8a67eef4c20cad9b
Parents: de01327
Author: Mingliang Liu <li...@apache.org>
Authored: Fri Nov 4 14:34:40 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Fri Nov 4 14:34:49 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/TestStartup.java | 55 +++++++++++++++++++-
1 file changed, 54 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c0ab102/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
index 4b6c0bd..5da19a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
@@ -19,8 +19,12 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption.IMPORT;
import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
+import static org.hamcrest.CoreMatchers.allOf;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -29,6 +33,8 @@ import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.URI;
+import java.nio.file.Paths;
+import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -49,6 +55,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
@@ -108,7 +115,7 @@ public class TestStartup {
fileAsURI(new File(hdfsDir, "secondary")).toString());
config.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
WILDCARD_HTTP_HOST + "0");
-
+
FileSystem.setDefaultUri(config, "hdfs://"+NAME_NODE_HOST + "0");
}
@@ -645,6 +652,52 @@ public class TestStartup {
}
}
+ @Test(timeout = 30000)
+ public void testNNFailToStartOnReadOnlyNNDir() throws Exception {
+ /* set NN dir */
+ final String nnDirStr = Paths.get(
+ hdfsDir.toString(),
+ GenericTestUtils.getMethodName(), "name").toString();
+ config.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nnDirStr);
+
+ try(MiniDFSCluster cluster = new MiniDFSCluster.Builder(config)
+ .numDataNodes(1)
+ .manageNameDfsDirs(false)
+ .build()) {
+ cluster.waitActive();
+
+ /* get and verify NN dir */
+ final Collection<URI> nnDirs = FSNamesystem.getNamespaceDirs(config);
+ assertNotNull(nnDirs);
+ assertTrue(nnDirs.iterator().hasNext());
+ assertEquals(
+ "NN dir should be created after NN startup.",
+ nnDirStr,
+ nnDirs.iterator().next().getPath());
+ final File nnDir = new File(nnDirStr);
+ assertTrue(nnDir.exists());
+ assertTrue(nnDir.isDirectory());
+
+ try {
+ /* set read only */
+ assertTrue(
+ "Setting NN dir read only should succeed.",
+ nnDir.setReadOnly());
+ cluster.restartNameNodes();
+ fail("Restarting NN should fail on read only NN dir.");
+ } catch (InconsistentFSStateException e) {
+ assertThat(e.toString(), is(allOf(
+ containsString("InconsistentFSStateException"),
+ containsString(nnDirStr),
+ containsString("in an inconsistent state"),
+ containsString(
+ "storage directory does not exist or is not accessible."))));
+ } finally {
+ /* set back to writable in order to clean it */
+ assertTrue("Setting NN dir should succeed.", nnDir.setWritable(true));
+ }
+ }
+ }
/**
* Verify the following scenario.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: MAPREDUCE-6765. MR should not
schedule container requests in cases where reducer or mapper containers
demand resource larger than the maximum supported (haibochen via rkanter)
Posted by fa...@apache.org.
MAPREDUCE-6765. MR should not schedule container requests in cases where reducer or mapper containers demand resource larger than the maximum supported (haibochen via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc2b69eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc2b69eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc2b69eb
Branch: refs/heads/HADOOP-13345
Commit: fc2b69eba1c5df59f6175205c27dc7b584df50c0
Parents: 1b6ecaf
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Nov 1 20:47:25 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Nov 1 20:47:25 2016 -0700
----------------------------------------------------------------------
.../v2/app/rm/RMContainerAllocator.java | 171 +++++++++++--------
.../v2/app/rm/TestRMContainerAllocator.java | 96 ++++++++++-
2 files changed, 195 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc2b69eb/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index e3b673a..db8f337 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -151,10 +151,10 @@ public class RMContainerAllocator extends RMContainerRequestor
//holds information about the assigned containers to task attempts
private final AssignedRequests assignedRequests;
-
+
//holds scheduled requests to be fulfilled by RM
private final ScheduledRequests scheduledRequests = new ScheduledRequests();
-
+
private int containersAllocated = 0;
private int containersReleased = 0;
private int hostLocalAssigned = 0;
@@ -370,76 +370,16 @@ public class RMContainerAllocator extends RMContainerRequestor
}
}
- @SuppressWarnings({ "unchecked" })
protected synchronized void handleEvent(ContainerAllocatorEvent event) {
recalculateReduceSchedule = true;
if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
ContainerRequestEvent reqEvent = (ContainerRequestEvent) event;
- JobId jobId = getJob().getID();
- Resource supportedMaxContainerCapability = getMaxContainerCapability();
- if (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP)) {
- if (mapResourceRequest.equals(Resources.none())) {
- mapResourceRequest = reqEvent.getCapability();
- eventHandler.handle(new JobHistoryEvent(jobId,
- new NormalizedResourceEvent(
- org.apache.hadoop.mapreduce.TaskType.MAP, mapResourceRequest
- .getMemorySize())));
- LOG.info("mapResourceRequest:" + mapResourceRequest);
- if (mapResourceRequest.getMemorySize() > supportedMaxContainerCapability
- .getMemorySize()
- || mapResourceRequest.getVirtualCores() > supportedMaxContainerCapability
- .getVirtualCores()) {
- String diagMsg =
- "MAP capability required is more than the supported "
- + "max container capability in the cluster. Killing the Job. mapResourceRequest: "
- + mapResourceRequest + " maxContainerCapability:"
- + supportedMaxContainerCapability;
- LOG.info(diagMsg);
- eventHandler.handle(new JobDiagnosticsUpdateEvent(jobId, diagMsg));
- eventHandler.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
- }
- }
- // set the resources
- reqEvent.getCapability().setMemorySize(mapResourceRequest.getMemorySize());
- reqEvent.getCapability().setVirtualCores(
- mapResourceRequest.getVirtualCores());
- scheduledRequests.addMap(reqEvent);//maps are immediately scheduled
+ boolean isMap = reqEvent.getAttemptID().getTaskId().getTaskType().
+ equals(TaskType.MAP);
+ if (isMap) {
+ handleMapContainerRequest(reqEvent);
} else {
- if (reduceResourceRequest.equals(Resources.none())) {
- reduceResourceRequest = reqEvent.getCapability();
- eventHandler.handle(new JobHistoryEvent(jobId,
- new NormalizedResourceEvent(
- org.apache.hadoop.mapreduce.TaskType.REDUCE,
- reduceResourceRequest.getMemorySize())));
- LOG.info("reduceResourceRequest:" + reduceResourceRequest);
- if (reduceResourceRequest.getMemorySize() > supportedMaxContainerCapability
- .getMemorySize()
- || reduceResourceRequest.getVirtualCores() > supportedMaxContainerCapability
- .getVirtualCores()) {
- String diagMsg =
- "REDUCE capability required is more than the "
- + "supported max container capability in the cluster. Killing the "
- + "Job. reduceResourceRequest: " + reduceResourceRequest
- + " maxContainerCapability:"
- + supportedMaxContainerCapability;
- LOG.info(diagMsg);
- eventHandler.handle(new JobDiagnosticsUpdateEvent(jobId, diagMsg));
- eventHandler.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
- }
- }
- // set the resources
- reqEvent.getCapability().setMemorySize(reduceResourceRequest.getMemorySize());
- reqEvent.getCapability().setVirtualCores(
- reduceResourceRequest.getVirtualCores());
- if (reqEvent.getEarlierAttemptFailed()) {
- //add to the front of queue for fail fast
- pendingReduces.addFirst(new ContainerRequest(reqEvent,
- PRIORITY_REDUCE, reduceNodeLabelExpression));
- } else {
- pendingReduces.add(new ContainerRequest(reqEvent, PRIORITY_REDUCE,
- reduceNodeLabelExpression));
- //reduces are added to pending and are slowly ramped up
- }
+ handleReduceContainerRequest(reqEvent);
}
} else if (
@@ -476,6 +416,103 @@ public class RMContainerAllocator extends RMContainerRequestor
}
}
+ @SuppressWarnings({ "unchecked" })
+ private void handleReduceContainerRequest(ContainerRequestEvent reqEvent) {
+ assert(reqEvent.getAttemptID().getTaskId().getTaskType().equals(
+ TaskType.REDUCE));
+
+ Resource supportedMaxContainerCapability = getMaxContainerCapability();
+ JobId jobId = getJob().getID();
+
+ if (reduceResourceRequest.equals(Resources.none())) {
+ reduceResourceRequest = reqEvent.getCapability();
+ eventHandler.handle(new JobHistoryEvent(jobId,
+ new NormalizedResourceEvent(
+ org.apache.hadoop.mapreduce.TaskType.REDUCE,
+ reduceResourceRequest.getMemorySize())));
+ LOG.info("reduceResourceRequest:" + reduceResourceRequest);
+ }
+
+ boolean reduceContainerRequestAccepted = true;
+ if (reduceResourceRequest.getMemorySize() >
+ supportedMaxContainerCapability.getMemorySize()
+ ||
+ reduceResourceRequest.getVirtualCores() >
+ supportedMaxContainerCapability.getVirtualCores()) {
+ reduceContainerRequestAccepted = false;
+ }
+
+ if (reduceContainerRequestAccepted) {
+ // set the resources
+ reqEvent.getCapability().setVirtualCores(
+ reduceResourceRequest.getVirtualCores());
+ reqEvent.getCapability().setMemorySize(
+ reduceResourceRequest.getMemorySize());
+
+ if (reqEvent.getEarlierAttemptFailed()) {
+ //previously failed reducers are added to the front for fail fast
+ pendingReduces.addFirst(new ContainerRequest(reqEvent,
+ PRIORITY_REDUCE, reduceNodeLabelExpression));
+ } else {
+ //reduces are added to pending queue and are slowly ramped up
+ pendingReduces.add(new ContainerRequest(reqEvent,
+ PRIORITY_REDUCE, reduceNodeLabelExpression));
+ }
+ } else {
+ String diagMsg = "REDUCE capability required is more than the " +
+ "supported max container capability in the cluster. Killing" +
+ " the Job. reduceResourceRequest: " + reduceResourceRequest +
+ " maxContainerCapability:" + supportedMaxContainerCapability;
+ LOG.info(diagMsg);
+ eventHandler.handle(new JobDiagnosticsUpdateEvent(jobId, diagMsg));
+ eventHandler.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
+ }
+ }
+
+ @SuppressWarnings({ "unchecked" })
+ private void handleMapContainerRequest(ContainerRequestEvent reqEvent) {
+ assert(reqEvent.getAttemptID().getTaskId().getTaskType().equals(
+ TaskType.MAP));
+
+ Resource supportedMaxContainerCapability = getMaxContainerCapability();
+ JobId jobId = getJob().getID();
+
+ if (mapResourceRequest.equals(Resources.none())) {
+ mapResourceRequest = reqEvent.getCapability();
+ eventHandler.handle(new JobHistoryEvent(jobId,
+ new NormalizedResourceEvent(
+ org.apache.hadoop.mapreduce.TaskType.MAP,
+ mapResourceRequest.getMemorySize())));
+ LOG.info("mapResourceRequest:" + mapResourceRequest);
+ }
+
+ boolean mapContainerRequestAccepted = true;
+ if (mapResourceRequest.getMemorySize() >
+ supportedMaxContainerCapability.getMemorySize()
+ ||
+ mapResourceRequest.getVirtualCores() >
+ supportedMaxContainerCapability.getVirtualCores()) {
+ mapContainerRequestAccepted = false;
+ }
+
+ if(mapContainerRequestAccepted) {
+ // set the resources
+ reqEvent.getCapability().setMemorySize(
+ mapResourceRequest.getMemorySize());
+ reqEvent.getCapability().setVirtualCores(
+ mapResourceRequest.getVirtualCores());
+ scheduledRequests.addMap(reqEvent); //maps are immediately scheduled
+ } else {
+ String diagMsg = "The required MAP capability is more than the " +
+ "supported max container capability in the cluster. Killing" +
+ " the Job. mapResourceRequest: " + mapResourceRequest +
+ " maxContainerCapability:" + supportedMaxContainerCapability;
+ LOG.info(diagMsg);
+ eventHandler.handle(new JobDiagnosticsUpdateEvent(jobId, diagMsg));
+ eventHandler.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
+ }
+ }
+
private static String getHost(String contMgrAddress) {
String host = contMgrAddress;
String[] hostport = host.split(":");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc2b69eb/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index bcce793..f9ee9cc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -1792,12 +1792,18 @@ public class TestRMContainerAllocator {
private ContainerRequestEvent createReq(JobId jobId, int taskAttemptId,
int memory, String[] hosts) {
- return createReq(jobId, taskAttemptId, memory, hosts, false, false);
+ return createReq(jobId, taskAttemptId, memory, 1, hosts, false, false);
}
- private ContainerRequestEvent
- createReq(JobId jobId, int taskAttemptId, int memory, String[] hosts,
- boolean earlierFailedAttempt, boolean reduce) {
+ private ContainerRequestEvent createReq(JobId jobId, int taskAttemptId,
+ int mem, String[] hosts, boolean earlierFailedAttempt, boolean reduce) {
+ return createReq(jobId, taskAttemptId, mem,
+ 1, hosts, earlierFailedAttempt, reduce);
+ }
+
+ private ContainerRequestEvent createReq(JobId jobId, int taskAttemptId,
+ int memory, int vcore, String[] hosts, boolean earlierFailedAttempt,
+ boolean reduce) {
TaskId taskId;
if (reduce) {
taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
@@ -1806,7 +1812,7 @@ public class TestRMContainerAllocator {
}
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
taskAttemptId);
- Resource containerNeed = Resource.newInstance(memory, 1);
+ Resource containerNeed = Resource.newInstance(memory, vcore);
if (earlierFailedAttempt) {
return ContainerRequestEvent
.createContainerRequestEventForFailedContainer(attemptId,
@@ -2608,6 +2614,86 @@ public class TestRMContainerAllocator {
}
@Test
+ public void testUnsupportedMapContainerRequirement() throws Exception {
+ final Resource maxContainerSupported = Resource.newInstance(1, 1);
+
+ final ApplicationId appId = ApplicationId.newInstance(1, 1);
+ final ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ final JobId jobId =
+ MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
+
+ final MockScheduler mockScheduler = new MockScheduler(appAttemptId);
+ final Configuration conf = new Configuration();
+
+ final MyContainerAllocator allocator = new MyContainerAllocator(null,
+ conf, appAttemptId, mock(Job.class), SystemClock.getInstance()) {
+ @Override
+ protected void register() {
+ }
+ @Override
+ protected ApplicationMasterProtocol createSchedulerProxy() {
+ return mockScheduler;
+ }
+ @Override
+ protected Resource getMaxContainerCapability() {
+ return maxContainerSupported;
+ }
+ };
+
+ ContainerRequestEvent mapRequestEvt = createReq(jobId, 0,
+ (int) (maxContainerSupported.getMemorySize() + 10),
+ maxContainerSupported.getVirtualCores(),
+ new String[0], false, false);
+ allocator.sendRequests(Arrays.asList(mapRequestEvt));
+ allocator.schedule();
+
+ Assert.assertEquals(0, mockScheduler.lastAnyAskMap);
+ }
+
+ @Test
+ public void testUnsupportedReduceContainerRequirement() throws Exception {
+ final Resource maxContainerSupported = Resource.newInstance(1, 1);
+
+ final ApplicationId appId = ApplicationId.newInstance(1, 1);
+ final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
+ appId, 1);
+ final JobId jobId =
+ MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
+
+ final MockScheduler mockScheduler = new MockScheduler(appAttemptId);
+ final Configuration conf = new Configuration();
+
+ final MyContainerAllocator allocator = new MyContainerAllocator(null,
+ conf, appAttemptId, mock(Job.class), SystemClock.getInstance()) {
+ @Override
+ protected void register() {
+ }
+ @Override
+ protected ApplicationMasterProtocol createSchedulerProxy() {
+ return mockScheduler;
+ }
+ @Override
+ protected Resource getMaxContainerCapability() {
+ return maxContainerSupported;
+ }
+ };
+
+ ContainerRequestEvent reduceRequestEvt = createReq(jobId, 0,
+ (int) (maxContainerSupported.getMemorySize() + 10),
+ maxContainerSupported.getVirtualCores(),
+ new String[0], false, true);
+ allocator.sendRequests(Arrays.asList(reduceRequestEvt));
+ // Reducer container requests are added to the pending queue upon request,
+ // schedule all reducers here so that we can observe if reducer requests
+ // are accepted by RMContainerAllocator on RM side.
+ allocator.scheduleAllReduces();
+ allocator.schedule();
+
+ Assert.assertEquals(0, mockScheduler.lastAnyAskReduce);
+ }
+
+ @Test
public void testRMUnavailable()
throws Exception {
Configuration conf = new Configuration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HADOOP-13452. S3Guard: Implement
access policy for intra-client consistency with in-memory metadata store.
Contributed by Aaron Fabbri.
Posted by fa...@apache.org.
HADOOP-13452. S3Guard: Implement access policy for intra-client consistency with in-memory metadata store. Contributed by Aaron Fabbri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05767451
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05767451
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05767451
Branch: refs/heads/HADOOP-13345
Commit: 05767451d147dfdd333d56322ec1fc5d9adb1e12
Parents: 2a734dc
Author: Chris Nauroth <cn...@apache.org>
Authored: Wed Oct 19 16:32:46 2016 -0700
Committer: Aaron Fabbri <fa...@apache.org>
Committed: Fri Nov 4 20:30:59 2016 -0700
----------------------------------------------------------------------
.../fs/s3a/s3guard/DirListingMetadata.java | 9 +
.../fs/s3a/s3guard/LocalMetadataStore.java | 216 +++++++++++++++++++
.../hadoop/fs/s3a/s3guard/LruHashMap.java | 50 +++++
.../hadoop/fs/s3a/s3guard/MetadataStore.java | 14 +-
.../hadoop/fs/s3a/s3guard/PathMetadata.java | 7 +-
.../fs/s3a/s3guard/AbstractMSContract.java | 5 +-
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 8 +-
.../fs/s3a/s3guard/TestLocalMetadataStore.java | 65 ++++++
8 files changed, 359 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05767451/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
index acb3e51..1838f42 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
@@ -38,6 +38,13 @@ import com.google.common.base.Preconditions;
@InterfaceStability.Evolving
public class DirListingMetadata {
+ /**
+ * Convenience parameter for passing into
+ * {@link DirListingMetadata#DirListingMetadata(Path, Collection, boolean)}.
+ */
+ public static final Collection<PathMetadata> EMPTY_DIR =
+ Collections.emptyList();
+
private final Path path;
/** Using a map for fast find / remove with large directories. */
@@ -151,6 +158,8 @@ public class DirListingMetadata {
*/
private void checkChildPath(Path childPath) {
Preconditions.checkNotNull(childPath, "childPath must be non-null");
+ Preconditions.checkArgument(childPath.isAbsolute(), "childPath must be " +
+ "absolute");
Preconditions.checkArgument(!childPath.isRoot(),
"childPath cannot be the root path");
Preconditions.checkArgument(childPath.getParent().equals(path),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05767451/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
new file mode 100644
index 0000000..d47d85e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * This is a local, in-memory, implementation of MetadataStore.
+ * This is *not* a coherent cache across processes. It is only
+ * locally-coherent.
+ *
+ * The purpose of this is for unit testing. It could also be used to accelerate
+ * local-only operations where only one process is operating on a given object
+ * store, or multiple processes are accessing a read-only storage bucket.
+ *
+ * This MetadataStore does not enforce filesystem rules such as disallowing
+ * non-recursive removal of non-empty directories. It is assumed the caller
+ * already has to perform these sorts of checks.
+ */
+public class LocalMetadataStore implements MetadataStore {
+
+ public static final Logger LOG = LoggerFactory.getLogger(MetadataStore.class);
+ public static final int DEFAULT_MAX_RECORDS = 128;
+ public static final String CONF_MAX_RECORDS =
+ "fs.metadatastore.local.max_records";
+
+ /** Contains directories and files. */
+ private LruHashMap<Path, PathMetadata> fileHash;
+
+ /** Contains directory listings. */
+ private LruHashMap<Path, DirListingMetadata> dirHash;
+
+ private FileSystem fs;
+
+ @Override
+ public void initialize(FileSystem fileSystem) throws IOException {
+ Preconditions.checkNotNull(fileSystem);
+ fs = fileSystem;
+ Configuration conf = fs.getConf();
+ Preconditions.checkNotNull(conf);
+ int maxRecords = conf.getInt(CONF_MAX_RECORDS, DEFAULT_MAX_RECORDS);
+ if (maxRecords < 4) {
+ maxRecords = 4;
+ }
+ // Start w/ less than max capacity. Space / time trade off.
+ fileHash = new LruHashMap<>(maxRecords/2, maxRecords);
+ dirHash = new LruHashMap<>(maxRecords/4, maxRecords);
+ }
+
+ @Override
+ public void delete(Path path) throws IOException {
+ doDelete(path, false);
+ }
+
+ @Override
+ public void deleteSubtree(Path path) throws IOException {
+ doDelete(path, true);
+ }
+
+ private synchronized void doDelete(Path path, boolean recursive) {
+
+ // We could implement positive hit for 'deleted' files. For now we
+ // do not track them.
+
+ // Delete entry from file cache, then from cached parent directory, if any
+
+ // Remove target file/dir
+ fileHash.remove(path);
+
+ // Update parent dir listing, if any
+ dirHashDeleteFile(path);
+
+ if (recursive) {
+ // Remove all entries that have this dir as path prefix.
+ clearHashByAncestor(path, dirHash);
+ clearHashByAncestor(path, fileHash);
+ }
+ }
+
+ @Override
+ public synchronized PathMetadata get(Path path) throws IOException {
+ return fileHash.mruGet(path);
+ }
+
+ @Override
+ public synchronized DirListingMetadata listChildren(Path path) throws
+ IOException {
+ return dirHash.mruGet(path);
+ }
+
+ @Override
+ public void move(Path src, Path dst) throws IOException {
+ // TODO implement me
+ throw new IOException("LocalMetadataStore#move() not implemented yet");
+ }
+
+ @Override
+ public void put(PathMetadata meta) throws IOException {
+
+ FileStatus status = meta.getFileStatus();
+ Path path = status.getPath();
+ synchronized (this) {
+
+ /* Add entry for this file. */
+ fileHash.put(path, meta);
+
+ /* Directory case:
+ * We also make sure we have an entry in the dirHash, so subsequent
+ * listStatus(path) at least see the directory.
+ *
+ * If we had a boolean flag argument "isNew", we would know whether this
+ * is an existing directory the client discovered via getFileStatus(),
+ * or if it is a newly-created directory. In the latter case, we would
+ * be able to mark the directory as authoritative (fully-cached),
+ * saving round trips to underlying store for subsequent listStatus()
+ */
+
+ /* If directory, go ahead and cache the fact that it is empty. */
+ if (status.isDirectory()) {
+ DirListingMetadata dir = dirHash.mruGet(path);
+ if (dir == null) {
+ dirHash.put(path, new DirListingMetadata(path, DirListingMetadata
+ .EMPTY_DIR, false));
+ }
+ }
+
+ /* Update cached parent dir. */
+ Path parentPath = path.getParent();
+ DirListingMetadata parent = dirHash.mruGet(parentPath);
+ if (parent == null) {
+ /* Track this new file's listing in parent. Parent is not
+ * authoritative, since there may be other items in it we don't know
+ * about. */
+ parent = new DirListingMetadata(parentPath,
+ DirListingMetadata.EMPTY_DIR, false);
+ dirHash.put(parentPath, parent);
+ }
+ parent.put(status);
+ }
+ }
+
+ @Override
+ public synchronized void put(DirListingMetadata meta) throws IOException {
+ dirHash.put(meta.getPath(), meta);
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ @VisibleForTesting
+ static <T> void clearHashByAncestor(Path ancestor, Map<Path, T> hash) {
+ for (Iterator<Map.Entry<Path, T>> it = hash.entrySet().iterator();
+ it.hasNext();) {
+ Map.Entry<Path, T> entry = it.next();
+ Path f = entry.getKey();
+ if (isAncestorOf(ancestor, f)) {
+ it.remove();
+ }
+ }
+ }
+
+ /**
+ * @return true iff 'ancestor' is ancestor dir in path 'f'.
+ * All paths here are absolute. Dir does not count as its own ancestor.
+ */
+ private static boolean isAncestorOf(Path ancestor, Path f) {
+ String aStr = ancestor.toString();
+ if (!ancestor.isRoot()) {
+ aStr += "/";
+ }
+ String fStr = f.toString();
+ return (fStr.startsWith(aStr));
+ }
+
+ /**
+ * Update dirHash to reflect deletion of file 'f'. Call with lock held.
+ */
+ private void dirHashDeleteFile(Path path) {
+ Path parent = path.getParent();
+ if (parent != null) {
+ DirListingMetadata dir = dirHash.get(parent);
+ if (dir != null) {
+ dir.remove(path);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05767451/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LruHashMap.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LruHashMap.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LruHashMap.java
new file mode 100644
index 0000000..e355095
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LruHashMap.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * LinkedHashMap that implements a maximum size and LRU eviction policy.
+ */
+public class LruHashMap<K, V> extends LinkedHashMap<K, V> {
+ private final int maxSize;
+ public LruHashMap(int initialCapacity, int maxSize) {
+ super(initialCapacity);
+ this.maxSize = maxSize;
+ }
+
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+ return size() > maxSize;
+ }
+
+ /**
+ * get() plus side-effect of making the element Most Recently Used.
+ * @param key lookup key
+ * @return value
+ */
+
+ public V mruGet(K key) {
+ V val = remove(key);
+ if (val != null) {
+ put(key, val);
+ }
+ return val;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05767451/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
index c034b9a..a6d29da 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
@@ -21,15 +21,15 @@ package org.apache.hadoop.fs.s3a.s3guard;
import java.io.Closeable;
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
* {@code MetadataStore} defines the set of operations that any metadata store
- * implementation must provide.
+ * implementation must provide. Note that all {@link Path} objects provided
+ * to methods must be absolute, not relative paths.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -38,10 +38,10 @@ public interface MetadataStore extends Closeable {
/**
* Performs one-time initialization of the metadata store.
*
- * @param conf {@code Configuration}
+ * @param fs {@code FileSystem} associated with the MetadataStore
* @throws IOException if there is an error
*/
- void initialize(Configuration conf) throws IOException;
+ void initialize(FileSystem fs) throws IOException;
/**
* Deletes exactly one path.
@@ -88,7 +88,7 @@ public interface MetadataStore extends Closeable {
* recursively.
*
* @param src the source path
- * @param dst the destination path
+ * @param dst the new path of the file/dir after the move
* @throws IOException if there is an error
*/
void move(Path src, Path dst) throws IOException;
@@ -99,7 +99,7 @@ public interface MetadataStore extends Closeable {
* saving the full path ancestry.
*
* Implementations must update any {@code DirListingMetadata} objects which
- * track the parent of this file.
+ * track the immediate parent of this file.
*
* @param meta the metadata to save
* @throws IOException if there is an error
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05767451/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
index 37fbfec..eb2e520 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
@@ -35,11 +35,14 @@ class PathMetadata {
/**
* Creates a new {@code PathMetadata} containing given {@code FileStatus}.
- *
- * @param fileStatus file status
+ * @param fileStatus file status containing an absolute path.
*/
public PathMetadata(FileStatus fileStatus) {
Preconditions.checkNotNull(fileStatus, "fileStatus must be non-null");
+ Preconditions.checkNotNull(fileStatus.getPath(), "fileStatus path must be" +
+ " non-null");
+ Preconditions.checkArgument(fileStatus.getPath().isAbsolute(), "path must" +
+ " be absolute");
this.fileStatus = fileStatus;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05767451/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java
index 7cb540a..9a1b590 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a.s3guard;
-import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
import java.io.IOException;
@@ -26,7 +26,8 @@ import java.io.IOException;
* Test specification for MetadataStore contract tests. Supplies configuration
* and MetadataStore instance.
*/
-public abstract class AbstractMSContract extends Configured {
+public abstract class AbstractMSContract {
+ public abstract FileSystem getFileSystem();
public abstract MetadataStore getMetadataStore() throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05767451/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index ac27696..701f355 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -72,7 +72,8 @@ public abstract class MetadataStoreTestBase extends Assert {
contract = createContract();
ms = contract.getMetadataStore();
assertNotNull("null MetadataStore", ms);
- ms.initialize(contract.getConf());
+ assertNotNull("null FileSystem", contract.getFileSystem());
+ ms.initialize(contract.getFileSystem());
}
@After
@@ -146,9 +147,8 @@ public abstract class MetadataStoreTestBase extends Assert {
assertFalse("Root not fully cached", dir.isAuthoritative());
assertNotNull("have root dir file listing", dir.getListing());
assertEquals("One file in root dir", 1, dir.getListing().size());
- assertEquals("file1 in root dir", "/file1",
- dir.getListing().iterator().next().getFileStatus().getPath()
- .toString());
+ assertEquals("file1 in root dir", "/file1", dir.getListing().iterator()
+ .next().getFileStatus().getPath().toString());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05767451/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
new file mode 100644
index 0000000..c8b3a85
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+
+import java.io.IOException;
+
+/**
+ * MetadataStore unit test for {@link LocalMetadataStore}.
+ */
+public class TestLocalMetadataStore extends MetadataStoreTestBase {
+
+ private static final String MAX_ENTRIES_STR = "16";
+
+ private static class LocalMSContract extends AbstractMSContract {
+
+ private FileSystem fs;
+
+ public LocalMSContract() {
+ fs = new RawLocalFileSystem();
+ Configuration config = fs.getConf();
+ if (config == null) {
+ config = new Configuration();
+ }
+ config.set(LocalMetadataStore.CONF_MAX_RECORDS, MAX_ENTRIES_STR);
+ fs.setConf(config);
+ }
+
+ @Override
+ public FileSystem getFileSystem() {
+ return fs;
+ }
+
+ @Override
+ public MetadataStore getMetadataStore() throws IOException {
+ LocalMetadataStore lms = new LocalMetadataStore();
+ return lms;
+ }
+ }
+
+ @Override
+ public AbstractMSContract createContract() {
+ return new LocalMSContract();
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HADOOP-13667. Fix typing mistake
of inline document in hadoop-metrics2.properties (Contributed by Rui Gao via
Daniel Templeton)
Posted by fa...@apache.org.
HADOOP-13667. Fix typing mistake of inline document in hadoop-metrics2.properties (Contributed by Rui Gao via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0dc2a6a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0dc2a6a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0dc2a6a3
Branch: refs/heads/HADOOP-13345
Commit: 0dc2a6a3a50add18e959475032264b04aff18fc4
Parents: cb5cc0d
Author: Daniel Templeton <te...@apache.org>
Authored: Wed Nov 2 06:52:27 2016 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Wed Nov 2 06:53:33 2016 -0700
----------------------------------------------------------------------
.../src/main/conf/hadoop-metrics2.properties | 22 +++++++++++++++++---
1 file changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0dc2a6a3/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
index 0c09228..16fdcf0 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
@@ -50,9 +50,25 @@
# If '*' all tags are used. If specifiying multiple tags separate them with
# commas. Note that the last segment of the property name is the context name.
#
-#*.sink.ganglia.tagsForPrefix.jvm=ProcesName
-#*.sink.ganglia.tagsForPrefix.dfs=
-#*.sink.ganglia.tagsForPrefix.rpc=
+# A typical use of tags is separating the metrics by the HDFS rpc port
+# and HDFS service rpc port.
+# For example:
+# With following HDFS configuration:
+# dfs.namenode.rpc-address is set as namenodeAddress:9110
+# dfs.namenode.servicerpc-address is set as namenodeAddress:9111
+# If no tags are used, following metric would be gathered:
+# rpc.rpc.NumOpenConnections
+# If using "*.sink.ganglia.tagsForPrefix.rpc=port",
+# following metrics would be gathered:
+# rpc.rpc.port=9110.NumOpenConnections
+# rpc.rpc.port=9111.NumOpenConnections
+#
+#*.sink.ganglia.tagsForPrefix.jvm=ProcessName
+#*.sink.ganglia.tagsForPrefix.dfs=HAState,IsOutOfSync
+#*.sink.ganglia.tagsForPrefix.rpc=port
+#*.sink.ganglia.tagsForPrefix.rpcdetailed=port
+#*.sink.ganglia.tagsForPrefix.metricssystem=*
+#*.sink.ganglia.tagsForPrefix.ugi=*
#*.sink.ganglia.tagsForPrefix.mapred=
#namenode.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-4862. Handle duplicate
completed containers in RMNodeImpl. Contributed by Rohith Sharma K S
Posted by fa...@apache.org.
YARN-4862. Handle duplicate completed containers in RMNodeImpl. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/352cbaa7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/352cbaa7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/352cbaa7
Branch: refs/heads/HADOOP-13345
Commit: 352cbaa7a54a94bad2bed131d6a250c5b21a7701
Parents: 0e75496
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Nov 3 13:54:31 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Nov 3 13:54:31 2016 +0000
----------------------------------------------------------------------
.../resourcemanager/rmnode/RMNodeImpl.java | 32 +++++++++---
.../scheduler/AbstractYarnScheduler.java | 21 ++++++--
.../resourcemanager/TestRMNodeTransitions.java | 40 ++++++++++++++
.../TestResourceTrackerService.java | 55 ++++++++++++++++++++
4 files changed, 139 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/352cbaa7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 375b4cf..d3d115f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -141,6 +141,10 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private final Set<ContainerId> launchedContainers =
new HashSet<ContainerId>();
+ /* track completed container globally */
+ private final Set<ContainerId> completedContainers =
+ new HashSet<ContainerId>();
+
/* set of containers that need to be cleaned */
private final Set<ContainerId> containersToClean = new TreeSet<ContainerId>(
new ContainerIdComparator());
@@ -578,6 +582,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
response.addContainersToBeRemovedFromNM(
new ArrayList<ContainerId>(this.containersToBeRemovedFromNM));
response.addAllContainersToSignal(this.containersToSignal);
+ this.completedContainers.removeAll(this.containersToBeRemovedFromNM);
this.containersToClean.clear();
this.finishedApplications.clear();
this.containersToSignal.clear();
@@ -1287,6 +1292,11 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
return this.launchedContainers;
}
+ @VisibleForTesting
+ public Set<ContainerId> getCompletedContainers() {
+ return this.completedContainers;
+ }
+
@Override
public Set<String> getNodeLabels() {
RMNodeLabelsManager nlm = context.getNodeLabelManager();
@@ -1329,7 +1339,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
// containers.
List<ContainerStatus> newlyLaunchedContainers =
new ArrayList<ContainerStatus>();
- List<ContainerStatus> completedContainers =
+ List<ContainerStatus> newlyCompletedContainers =
new ArrayList<ContainerStatus>();
int numRemoteRunningContainers = 0;
for (ContainerStatus remoteContainer : containerStatuses) {
@@ -1385,15 +1395,25 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
// Completed containers should also include the OPPORTUNISTIC containers
// so that the AM gets properly notified.
- completedContainers.add(remoteContainer);
+ if (completedContainers.add(containerId)) {
+ newlyCompletedContainers.add(remoteContainer);
+ }
+ }
+ }
+
+ List<ContainerStatus> lostContainers =
+ findLostContainers(numRemoteRunningContainers, containerStatuses);
+ for (ContainerStatus remoteContainer : lostContainers) {
+ ContainerId containerId = remoteContainer.getContainerId();
+ if (completedContainers.add(containerId)) {
+ newlyCompletedContainers.add(remoteContainer);
}
}
- completedContainers.addAll(findLostContainers(
- numRemoteRunningContainers, containerStatuses));
- if (newlyLaunchedContainers.size() != 0 || completedContainers.size() != 0) {
+ if (newlyLaunchedContainers.size() != 0
+ || newlyCompletedContainers.size() != 0) {
nodeUpdateQueue.add(new UpdatedContainerInfo(newlyLaunchedContainers,
- completedContainers));
+ newlyCompletedContainers));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/352cbaa7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index df59556..5e6a726 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRecoverEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
@@ -909,16 +910,18 @@ public abstract class AbstractYarnScheduler
* Process completed container list.
* @param completedContainers Extracted list of completed containers
* @param releasedResources Reference resource object for completed containers
+ * @param nodeId NodeId corresponding to the NodeManager
* @return The total number of released containers
*/
protected int updateCompletedContainers(List<ContainerStatus>
- completedContainers, Resource releasedResources) {
+ completedContainers, Resource releasedResources, NodeId nodeId) {
int releasedContainers = 0;
+ List<ContainerId> untrackedContainerIdList = new ArrayList<ContainerId>();
for (ContainerStatus completedContainer : completedContainers) {
ContainerId containerId = completedContainer.getContainerId();
LOG.debug("Container FINISHED: " + containerId);
RMContainer container = getRMContainer(containerId);
- completedContainer(getRMContainer(containerId),
+ completedContainer(container,
completedContainer, RMContainerEventType.FINISHED);
if (container != null) {
releasedContainers++;
@@ -930,8 +933,19 @@ public abstract class AbstractYarnScheduler
if (rrs != null) {
Resources.addTo(releasedResources, rrs);
}
+ } else {
+ // Add containers which are untracked by RM.
+ untrackedContainerIdList.add(containerId);
}
}
+
+ // Acknowledge NM to remove RM-untracked-containers from NM context.
+ if (!untrackedContainerIdList.isEmpty()) {
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMNodeFinishedContainersPulledByAMEvent(nodeId,
+ untrackedContainerIdList));
+ }
+
return releasedContainers;
}
@@ -977,7 +991,7 @@ public abstract class AbstractYarnScheduler
// Process completed containers
Resource releasedResources = Resource.newInstance(0, 0);
int releasedContainers = updateCompletedContainers(completedContainers,
- releasedResources);
+ releasedResources, nm.getNodeID());
// If the node is decommissioning, send an update to have the total
// resource equal to the used resource, so no available resource to
@@ -1004,4 +1018,5 @@ public abstract class AbstractYarnScheduler
" availableResource: " + node.getUnallocatedResource());
}
}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/352cbaa7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index 6038b31..6055afb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeReconnectEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
@@ -1065,4 +1066,43 @@ public class TestRMNodeTransitions {
Assert.assertTrue("second container not running",
node.getLaunchedContainers().contains(cid2));
}
+
+ @Test
+ public void testForHandlingDuplicatedCompltedContainers() {
+ // Start the node
+ node.handle(new RMNodeStartedEvent(null, null, null));
+ // Add info to the queue first
+ node.setNextHeartBeat(false);
+
+ ContainerId completedContainerId1 = BuilderUtils.newContainerId(
+ BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
+
+ RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent(null);
+
+ ContainerStatus containerStatus1 = mock(ContainerStatus.class);
+
+ doReturn(completedContainerId1).when(containerStatus1).getContainerId();
+ doReturn(Collections.singletonList(containerStatus1)).when(statusEvent1)
+ .getContainers();
+
+ verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
+ node.handle(statusEvent1);
+ verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
+ Assert.assertEquals(1, node.getQueueSize());
+ Assert.assertEquals(1, node.getCompletedContainers().size());
+
+ // test for duplicate entries
+ node.handle(statusEvent1);
+ Assert.assertEquals(1, node.getQueueSize());
+
+ // send clean up container event
+ node.handle(new RMNodeFinishedContainersPulledByAMEvent(node.getNodeID(),
+ Collections.singletonList(completedContainerId1)));
+
+ NodeHeartbeatResponse hbrsp =
+ Records.newRecord(NodeHeartbeatResponse.class);
+ node.updateNodeHeartbeatResponseForCleanup(hbrsp);
+ Assert.assertEquals(1, hbrsp.getContainersToBeRemovedFromNM().size());
+ Assert.assertEquals(0, node.getCompletedContainers().size());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/352cbaa7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 916c942..42fd49e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -19,9 +19,11 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileOutputStream;
@@ -2003,4 +2005,57 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
DefaultMetricsSystem.shutdown();
}
}
+
+ @Test(timeout = 60000)
+ public void testNodeHeartBeatResponseForUnknownContainerCleanUp()
+ throws Exception {
+ Configuration conf = new Configuration();
+ rm = new MockRM(conf);
+ rm.init(conf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ rm.drainEvents();
+
+ // send 1st heartbeat
+ nm1.nodeHeartbeat(true);
+
+ // Create 2 unknown containers tracked by NM
+ ApplicationId applicationId = BuilderUtils.newApplicationId(1, 1);
+ ApplicationAttemptId applicationAttemptId = BuilderUtils
+ .newApplicationAttemptId(applicationId, 1);
+ ContainerId cid1 = BuilderUtils.newContainerId(applicationAttemptId, 2);
+ ContainerId cid2 = BuilderUtils.newContainerId(applicationAttemptId, 3);
+ ArrayList<ContainerStatus> containerStats =
+ new ArrayList<ContainerStatus>();
+ containerStats.add(
+ ContainerStatus.newInstance(cid1, ContainerState.COMPLETE, "", -1));
+ containerStats.add(
+ ContainerStatus.newInstance(cid2, ContainerState.COMPLETE, "", -1));
+
+ Map<ApplicationId, List<ContainerStatus>> conts =
+ new HashMap<ApplicationId, List<ContainerStatus>>();
+ conts.put(applicationAttemptId.getApplicationId(), containerStats);
+
+ // add RMApp into context.
+ RMApp app1 = mock(RMApp.class);
+ when(app1.getApplicationId()).thenReturn(applicationId);
+ rm.getRMContext().getRMApps().put(applicationId, app1);
+
+ // Send unknown container status in heartbeat
+ nm1.nodeHeartbeat(conts, true);
+ rm.drainEvents();
+
+ int containersToBeRemovedFromNM = 0;
+ while (true) {
+ NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ rm.drainEvents();
+ containersToBeRemovedFromNM +=
+ nodeHeartbeat.getContainersToBeRemovedFromNM().size();
+ // asserting for 2 since two unknown containers status has been sent
+ if (containersToBeRemovedFromNM == 2) {
+ break;
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HADOOP-13738. DiskChecker should
perform some disk IO.
Posted by fa...@apache.org.
HADOOP-13738. DiskChecker should perform some disk IO.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b6ecaf0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b6ecaf0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b6ecaf0
Branch: refs/heads/HADOOP-13345
Commit: 1b6ecaf016aaf7f6a09a4d576294b5e0a6850a1f
Parents: bd7f591
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Nov 1 18:02:23 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Nov 1 18:02:23 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/util/DiskChecker.java | 159 ++++++++++++++++-
.../org/apache/hadoop/util/TestDiskChecker.java | 178 ++++++++++++++++---
2 files changed, 309 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b6ecaf0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
index 2c73af8..a4fa8fd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
@@ -19,14 +19,23 @@
package org.apache.hadoop.util;
import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Class that provides utility functions for checking disk problem
@@ -34,6 +43,8 @@ import org.apache.hadoop.fs.permission.FsPermission;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class DiskChecker {
+ public static final Logger LOG = LoggerFactory.getLogger(DiskChecker.class);
+
public static class DiskErrorException extends IOException {
public DiskErrorException(String msg) {
super(msg);
@@ -49,7 +60,12 @@ public class DiskChecker {
super(msg);
}
}
-
+
+ // Provider that abstracts some FileOutputStream operations for
+ // testability.
+ private static AtomicReference<FileIoProvider> fileIoProvider =
+ new AtomicReference<>(new DefaultFileIoProvider());
+
/**
* Create the directory if it doesn't exist and check that dir is readable,
* writable and executable
@@ -63,6 +79,7 @@ public class DiskChecker {
+ dir.toString());
}
checkAccessByFileMethods(dir);
+ doDiskIo(dir);
}
/**
@@ -80,6 +97,7 @@ public class DiskChecker {
throws DiskErrorException, IOException {
mkdirsWithExistsAndPermissionCheck(localFS, dir, expected);
checkAccessByFileMethods(localFS.pathToFile(dir));
+ doDiskIo(localFS.pathToFile(dir));
}
/**
@@ -173,4 +191,143 @@ public class DiskChecker {
if (created || !localFS.getFileStatus(dir).getPermission().equals(expected))
localFS.setPermission(dir, expected);
}
+
+ // State related to running disk IO checks.
+ private static final String DISK_IO_FILE_PREFIX =
+ "DiskChecker.OK_TO_DELETE_.";
+
+ @VisibleForTesting
+ static final int DISK_IO_MAX_ITERATIONS = 3;
+
+ /**
+ * Performs some disk IO by writing to a new file in the given directory
+ * and sync'ing file contents to disk.
+ *
+ * This increases the likelihood of catching catastrophic disk/controller
+ * failures sooner.
+ *
+ * @param dir directory to be checked.
+ * @throws DiskErrorException if we hit an error while trying to perform
+ * disk IO against the file.
+ */
+ private static void doDiskIo(File dir) throws DiskErrorException {
+ try {
+ IOException ioe = null;
+
+ for (int i = 0; i < DISK_IO_MAX_ITERATIONS; ++i) {
+ final File file = getFileNameForDiskIoCheck(dir, i+1);
+ try {
+ diskIoCheckWithoutNativeIo(file);
+ return;
+ } catch (IOException e) {
+ // Let's retry a few times before we really give up and
+ // declare the disk as bad.
+ ioe = e;
+ }
+ }
+ throw ioe; // Just rethrow the last exception to signal failure.
+ } catch(IOException e) {
+ throw new DiskErrorException("Error checking directory " + dir, e);
+ }
+ }
+
+ /**
+ * Try to perform some disk IO by writing to the given file
+ * without using Native IO.
+ *
+ * @param file
+ * @throws IOException if there was a non-retriable error.
+ */
+ private static void diskIoCheckWithoutNativeIo(File file)
+ throws IOException {
+ FileOutputStream fos = null;
+
+ try {
+ final FileIoProvider provider = fileIoProvider.get();
+ fos = provider.get(file);
+ provider.write(fos, new byte[1]);
+ fos.getFD().sync();
+ fos.close();
+ fos = null;
+ if (!file.delete() && file.exists()) {
+ throw new IOException("Failed to delete " + file);
+ }
+ file = null;
+ } finally {
+ IOUtils.cleanup(null, fos);
+ FileUtils.deleteQuietly(file);
+ }
+ }
+
+ /**
+ * Generate a path name for a test file under the given directory.
+ *
+ * @return file object.
+ */
+ @VisibleForTesting
+ static File getFileNameForDiskIoCheck(File dir, int iterationCount) {
+ if (iterationCount < DISK_IO_MAX_ITERATIONS) {
+ // Use file names of the format prefix.001 by default.
+ return new File(dir,
+ DISK_IO_FILE_PREFIX + String.format("%03d", iterationCount));
+ } else {
+ // If the first few checks then fail, try using a randomly generated
+ // file name.
+ return new File(dir, DISK_IO_FILE_PREFIX + UUID.randomUUID());
+ }
+ }
+
+ /**
+ * An interface that abstracts operations on {@link FileOutputStream}
+ * objects for testability.
+ */
+ interface FileIoProvider {
+ FileOutputStream get(File f) throws FileNotFoundException;
+ void write(FileOutputStream fos, byte[] data) throws IOException;
+ }
+
+ /**
+ * The default implementation of {@link FileIoProvider}.
+ */
+ private static class DefaultFileIoProvider implements FileIoProvider {
+ /**
+ * See {@link FileOutputStream#FileOutputStream(File)}.
+ */
+ @Override
+ public FileOutputStream get(File f) throws FileNotFoundException {
+ return new FileOutputStream(f);
+ }
+
+ /**
+ * See {@link FileOutputStream#write(byte[])}.
+ */
+ @Override
+ public void write(FileOutputStream fos, byte[] data) throws IOException {
+ fos.write(data);
+ }
+ }
+
+ /**
+ * Replace the {@link FileIoProvider} for tests.
+ * This method MUST NOT be used outside of unit tests.
+ *
+ * @param newFosProvider
+ * @return the old FileIoProvider.
+ */
+ @VisibleForTesting
+ static FileIoProvider replaceFileOutputStreamProvider(
+ FileIoProvider newFosProvider) {
+ return fileIoProvider.getAndSet(newFosProvider);
+ }
+
+ /**
+ * Retrieve the current {@link FileIoProvider}.
+ * This method MUST NOT be used outside of unit tests.
+ *
+ * @return the current FileIoProvider.
+ */
+ @VisibleForTesting
+ static FileIoProvider getFileOutputStreamProvider() {
+ return fileIoProvider.get();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b6ecaf0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
index e2e152a..40b8d0d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
@@ -19,7 +19,11 @@ package org.apache.hadoop.util;
import java.io.*;
import java.nio.file.Files;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.util.DiskChecker.FileIoProvider;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -33,27 +37,46 @@ import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestDiskChecker {
- final FsPermission defaultPerm = new FsPermission("755");
- final FsPermission invalidPerm = new FsPermission("000");
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestDiskChecker.class);
- @Test (timeout = 30000)
+ private final FsPermission defaultPerm = new FsPermission("755");
+ private final FsPermission invalidPerm = new FsPermission("000");
+
+ private FileIoProvider fileIoProvider = null;
+
+ @Before
+ public void setup() {
+ // Some tests replace the static field DiskChecker#fileIoProvider.
+ // Cache it so we can restore it after each test completes.
+ fileIoProvider = DiskChecker.getFileOutputStreamProvider();
+ }
+
+ @After
+ public void cleanup() {
+ DiskChecker.replaceFileOutputStreamProvider(fileIoProvider);
+ }
+
+ @Test(timeout = 30000)
public void testMkdirs_dirExists() throws Throwable {
_mkdirs(true, defaultPerm, defaultPerm);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testMkdirs_noDir() throws Throwable {
_mkdirs(false, defaultPerm, defaultPerm);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testMkdirs_dirExists_badUmask() throws Throwable {
_mkdirs(true, defaultPerm, invalidPerm);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testMkdirs_noDir_badUmask() throws Throwable {
_mkdirs(false, defaultPerm, invalidPerm);
}
@@ -78,34 +101,33 @@ public class TestDiskChecker {
verify(fs).getFileStatus(dir);
verify(stat).getPermission();
}
- }
- catch (DiskErrorException e) {
+ } catch (DiskErrorException e) {
if (before != after)
assertTrue(e.getMessage().startsWith("Incorrect permission"));
}
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testCheckDir_normal() throws Throwable {
_checkDirs(true, new FsPermission("755"), true);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testCheckDir_notDir() throws Throwable {
_checkDirs(false, new FsPermission("000"), false);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testCheckDir_notReadable() throws Throwable {
_checkDirs(true, new FsPermission("000"), false);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testCheckDir_notWritable() throws Throwable {
_checkDirs(true, new FsPermission("444"), false);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testCheckDir_notListable() throws Throwable {
_checkDirs(true, new FsPermission("666"), false); // not listable
}
@@ -134,13 +156,15 @@ public class TestDiskChecker {
throws Throwable {
File localDir = isDir ? createTempDir() : createTempFile();
Shell.execCommand(Shell.getSetPermissionCommand(String.format("%04o",
- perm.toShort()), false, localDir.getAbsolutePath()));
+ perm.toShort()), false, localDir.getAbsolutePath()));
try {
DiskChecker.checkDir(FileSystem.getLocal(new Configuration()),
- new Path(localDir.getAbsolutePath()), perm);
- assertTrue("checkDir success", success);
+ new Path(localDir.getAbsolutePath()), perm);
+ assertTrue("checkDir success, expected failure", success);
} catch (DiskErrorException e) {
- assertFalse("checkDir success", success);
+ if (success) {
+ throw e; // Unexpected exception!
+ }
}
localDir.delete();
}
@@ -150,27 +174,27 @@ public class TestDiskChecker {
* permission for result of mapper.
*/
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testCheckDir_normal_local() throws Throwable {
checkDirs(true, "755", true);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testCheckDir_notDir_local() throws Throwable {
checkDirs(false, "000", false);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testCheckDir_notReadable_local() throws Throwable {
checkDirs(true, "000", false);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testCheckDir_notWritable_local() throws Throwable {
checkDirs(true, "444", false);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testCheckDir_notListable_local() throws Throwable {
checkDirs(true, "666", false);
}
@@ -179,16 +203,116 @@ public class TestDiskChecker {
throws Throwable {
File localDir = isDir ? createTempDir() : createTempFile();
Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
- localDir.getAbsolutePath()));
+ localDir.getAbsolutePath()));
try {
DiskChecker.checkDir(localDir);
- assertTrue("checkDir success", success);
+ assertTrue("checkDir success, expected failure", success);
} catch (DiskErrorException e) {
- e.printStackTrace();
- assertFalse("checkDir success", success);
+ if (success) {
+ throw e; // Unexpected exception!
+ }
}
localDir.delete();
- System.out.println("checkDir success: " + success);
+ }
+ /**
+ * Verify DiskChecker ignores at least 2 transient file creation errors.
+ */
+ @Test(timeout = 30000)
+ public void testDiskIoIgnoresTransientCreateErrors() throws Throwable {
+ DiskChecker.replaceFileOutputStreamProvider(new TestFileIoProvider(
+ DiskChecker.DISK_IO_MAX_ITERATIONS - 1, 0));
+ checkDirs(true, "755", true);
+ }
+
+ /**
+ * Verify DiskChecker bails after 3 file creation errors.
+ */
+ @Test(timeout = 30000)
+ public void testDiskIoDetectsCreateErrors() throws Throwable {
+ DiskChecker.replaceFileOutputStreamProvider(new TestFileIoProvider(
+ DiskChecker.DISK_IO_MAX_ITERATIONS, 0));
+ checkDirs(true, "755", false);
+ }
+
+ /**
+ * Verify DiskChecker ignores at least 2 transient file write errors.
+ */
+ @Test(timeout = 30000)
+ public void testDiskIoIgnoresTransientWriteErrors() throws Throwable {
+ DiskChecker.replaceFileOutputStreamProvider(new TestFileIoProvider(
+ 0, DiskChecker.DISK_IO_MAX_ITERATIONS - 1));
+ checkDirs(true, "755", true);
+ }
+
+ /**
+ * Verify DiskChecker bails after 3 file write errors.
+ */
+ @Test(timeout = 30000)
+ public void testDiskIoDetectsWriteErrors() throws Throwable {
+ DiskChecker.replaceFileOutputStreamProvider(new TestFileIoProvider(
+ 0, DiskChecker.DISK_IO_MAX_ITERATIONS));
+ checkDirs(true, "755", false);
+ }
+
+ /**
+ * Verify DiskChecker's test file naming scheme.
+ */
+ @Test(timeout = 30000)
+ public void testDiskIoFileNaming() throws Throwable {
+ final File rootDir = new File("/");
+ assertTrue(".001".matches("\\.00\\d$"));
+ for (int i = 1; i < DiskChecker.DISK_IO_MAX_ITERATIONS; ++i) {
+ final File file = DiskChecker.getFileNameForDiskIoCheck(rootDir, i);
+ assertTrue(
+ "File name does not match expected pattern: " + file,
+ file.toString().matches("^.*\\.[0-9]+$"));
+ }
+ final File guidFile = DiskChecker.getFileNameForDiskIoCheck(
+ rootDir, DiskChecker.DISK_IO_MAX_ITERATIONS);
+ assertTrue(
+ "File name does not match expected pattern: " + guidFile,
+ guidFile.toString().matches("^.*\\.[A-Za-z0-9-]+$"));
+ }
+
+ /**
+ * A dummy {@link DiskChecker#FileIoProvider} that can throw a programmable
+ * number of times.
+ */
+ private static class TestFileIoProvider implements FileIoProvider {
+ private final AtomicInteger numCreateCalls = new AtomicInteger(0);
+ private final AtomicInteger numWriteCalls = new AtomicInteger(0);
+
+ private final int numTimesToThrowOnCreate;
+ private final int numTimesToThrowOnWrite;
+
+ public TestFileIoProvider(
+ int numTimesToThrowOnCreate, int numTimesToThrowOnWrite) {
+ this.numTimesToThrowOnCreate = numTimesToThrowOnCreate;
+ this.numTimesToThrowOnWrite = numTimesToThrowOnWrite;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public FileOutputStream get(File f) throws FileNotFoundException {
+ if (numCreateCalls.getAndIncrement() < numTimesToThrowOnCreate) {
+ throw new FileNotFoundException("Dummy exception for testing");
+ }
+ // Can't mock final class FileOutputStream.
+ return new FileOutputStream(f);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void write(FileOutputStream fos, byte[] data) throws IOException {
+ if (numWriteCalls.getAndIncrement() < numTimesToThrowOnWrite) {
+ throw new IOException("Dummy exception for testing");
+ }
+ fos.write(data);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-5802.
updateApplicationPriority api in scheduler should ensure to re-insert app to
correct ordering policy. Contributred by Bibin A Chundatt
Posted by fa...@apache.org.
YARN-5802. updateApplicationPriority api in scheduler should ensure to re-insert app to correct ordering policy. Contributred by Bibin A Chundatt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19b3779a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19b3779a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19b3779a
Branch: refs/heads/HADOOP-13345
Commit: 19b3779ae7455230ed89971141b2667eae624aab
Parents: 69dd5fa
Author: Sunil <su...@apache.org>
Authored: Fri Nov 4 16:07:28 2016 +0530
Committer: Sunil <su...@apache.org>
Committed: Fri Nov 4 16:07:28 2016 +0530
----------------------------------------------------------------------
.../scheduler/capacity/LeafQueue.java | 12 +++-
.../resourcemanager/TestClientRMService.java | 19 +++++
.../capacity/TestApplicationPriority.java | 74 ++++++++++++++++++++
3 files changed, 102 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19b3779a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 214c6e7..8941fdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -2266,12 +2266,18 @@ public class LeafQueue extends AbstractCSQueue {
try {
writeLock.lock();
FiCaSchedulerApp attempt = app.getCurrentAppAttempt();
- getOrderingPolicy().removeSchedulableEntity(attempt);
-
+ boolean isActive = orderingPolicy.removeSchedulableEntity(attempt);
+ if (!isActive) {
+ pendingOrderingPolicy.removeSchedulableEntity(attempt);
+ }
// Update new priority in SchedulerApplication
attempt.setPriority(newAppPriority);
- getOrderingPolicy().addSchedulableEntity(attempt);
+ if (isActive) {
+ orderingPolicy.addSchedulableEntity(attempt);
+ } else {
+ pendingOrderingPolicy.addSchedulableEntity(attempt);
+ }
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19b3779a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 706c274..884e236 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -1642,6 +1642,25 @@ public class TestClientRMService {
}
@Test(timeout = 120000)
+ public void testUpdatePriorityAndKillAppWithZeroClusterResource()
+ throws Exception {
+ int maxPriority = 10;
+ int appPriority = 5;
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY,
+ maxPriority);
+ MockRM rm = new MockRM(conf);
+ rm.init(conf);
+ rm.start();
+ RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority));
+ ClientRMService rmService = rm.getClientRMService();
+ testApplicationPriorityUpdation(rmService, app1, appPriority, appPriority);
+ rm.killApp(app1.getApplicationId());
+ rm.waitForState(app1.getApplicationId(), RMAppState.KILLED);
+ rm.stop();
+ }
+
+ @Test(timeout = 120000)
public void testUpdateApplicationPriorityRequest() throws Exception {
int maxPriority = 10;
int appPriority = 5;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19b3779a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index 523d49d..2a346f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -23,6 +23,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
@@ -54,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeRepo
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -733,4 +736,75 @@ public class TestApplicationPriority {
rm2.stop();
rm1.stop();
}
+
+ @Test(timeout = 120000)
+ public void testUpdatePriorityOnPendingAppAndKillAttempt() throws Exception {
+ int maxPriority = 10;
+ int appPriority = 5;
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY,
+ maxPriority);
+ MockRM rm = new MockRM(conf);
+ rm.init(conf);
+ rm.start();
+
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ CSQueue defaultQueue = (LeafQueue) cs.getQueue("default");
+
+ // Update priority and kill application with no resource
+ RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority));
+ Collection<FiCaSchedulerApp> appsPending =
+ ((LeafQueue) defaultQueue).getPendingApplications();
+ Collection<FiCaSchedulerApp> activeApps =
+ ((LeafQueue) defaultQueue).getOrderingPolicy().getSchedulableEntities();
+
+ // Verify app is in pending state
+ Assert.assertEquals("Pending apps should be 1", 1, appsPending.size());
+ Assert.assertEquals("Active apps should be 0", 0, activeApps.size());
+
+ // kill app1 which is pending
+ killAppAndVerifyOrderingPolicy(rm, defaultQueue, 0, 0, app1);
+
+ // Check ordering policy size when resource is added
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 8096, rm.getResourceTrackerService());
+ nm1.registerNode();
+ RMApp app2 = rm.submitApp(1024, Priority.newInstance(appPriority));
+ Assert.assertEquals("Pending apps should be 0", 0, appsPending.size());
+ Assert.assertEquals("Active apps should be 1", 1, activeApps.size());
+ RMApp app3 = rm.submitApp(1024, Priority.newInstance(appPriority));
+ RMApp app4 = rm.submitApp(1024, Priority.newInstance(appPriority));
+ Assert.assertEquals("Pending apps should be 2", 2, appsPending.size());
+ Assert.assertEquals("Active apps should be 1", 1, activeApps.size());
+ // kill app3, pending apps should reduce to 1
+ killAppAndVerifyOrderingPolicy(rm, defaultQueue, 1, 1, app3);
+ // kill app2, running apps is killed and pending added to running
+ killAppAndVerifyOrderingPolicy(rm, defaultQueue, 0, 1, app2);
+ // kill app4, all apps are killed and both policy size should be zero
+ killAppAndVerifyOrderingPolicy(rm, defaultQueue, 0, 0, app4);
+ rm.stop();
+ }
+
+ private void killAppAndVerifyOrderingPolicy(MockRM rm, CSQueue defaultQueue,
+ int appsPendingExpected, int activeAppsExpected, RMApp app)
+ throws YarnException {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ cs.updateApplicationPriority(Priority.newInstance(2),
+ app.getApplicationId());
+ SchedulerEvent removeAttempt;
+ removeAttempt = new AppAttemptRemovedSchedulerEvent(
+ app.getCurrentAppAttempt().getAppAttemptId(), RMAppAttemptState.KILLED,
+ false);
+ cs.handle(removeAttempt);
+ rm.drainEvents();
+ Collection<FiCaSchedulerApp> appsPending =
+ ((LeafQueue) defaultQueue).getPendingApplications();
+ Collection<FiCaSchedulerApp> activeApps =
+ ((LeafQueue) defaultQueue).getApplications();
+ Assert.assertEquals("Pending apps should be " + appsPendingExpected,
+ appsPendingExpected, appsPending.size());
+ Assert.assertEquals("Active apps should be " + activeAppsExpected,
+ activeAppsExpected, activeApps.size());
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-5837. NPE when getting node
status of a decommissioned node after an RM restart. Contributed by Robert
Kanter
Posted by fa...@apache.org.
YARN-5837. NPE when getting node status of a decommissioned node after an RM restart. Contributed by Robert Kanter
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bb741ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bb741ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bb741ff
Branch: refs/heads/HADOOP-13345
Commit: 6bb741ff0ef208a8628bc64d6537999d4cd67955
Parents: 0c0ab10
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Nov 4 22:20:21 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Nov 4 22:20:21 2016 +0000
----------------------------------------------------------------------
.../server/resourcemanager/NodesListManager.java | 4 +++-
.../yarn/server/resourcemanager/MockNM.java | 4 ++++
.../server/resourcemanager/TestRMRestart.java | 19 +++++++++++++++++++
3 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb741ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index 99413bc..7d69f93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -242,7 +243,8 @@ public class NodesListManager extends CompositeService implements
for (final String host : excludeList) {
NodeId nodeId = createUnknownNodeId(host);
RMNodeImpl rmNode = new RMNodeImpl(nodeId,
- rmContext, host, -1, -1, new UnknownNode(host), null, null);
+ rmContext, host, -1, -1, new UnknownNode(host),
+ Resource.newInstance(0, 0), "unknown");
rmContext.getInactiveRMNodes().put(nodeId, rmNode);
rmNode.handle(new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb741ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index d4aa113..32cdb1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -256,4 +256,8 @@ public class MockNM {
public int getvCores() {
return vCores;
}
+
+ public String getVersion() {
+ return version;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb741ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 38ef59b..a98a124 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -40,6 +40,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
@@ -103,6 +104,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@@ -1957,6 +1959,9 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
rm1.start();
MockNM nm1 = rm1.registerNode("localhost:1234", 8000);
MockNM nm2 = rm1.registerNode("host2:1234", 8000);
+ Resource expectedCapability =
+ Resource.newInstance(nm1.getMemory(), nm1.getvCores());
+ String expectedVersion = nm1.getVersion();
Assert
.assertEquals(0,
ClusterMetrics.getMetrics().getNumDecommisionedNMs());
@@ -1978,6 +1983,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
Assert
.assertEquals(2,
ClusterMetrics.getMetrics().getNumDecommisionedNMs());
+ verifyNodesAfterDecom(rm1, 2, expectedCapability, expectedVersion);
rm1.stop();
rm1 = null;
Assert
@@ -1991,6 +1997,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
Assert
.assertEquals(2,
ClusterMetrics.getMetrics().getNumDecommisionedNMs());
+ verifyNodesAfterDecom(rm2, 2, Resource.newInstance(0, 0), "unknown");
} finally {
if (rm1 != null) {
rm1.stop();
@@ -2001,6 +2008,18 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
}
}
+ private void verifyNodesAfterDecom(MockRM rm, int numNodes,
+ Resource expectedCapability,
+ String expectedVersion) {
+ ConcurrentMap<NodeId, RMNode> inactiveRMNodes =
+ rm.getRMContext().getInactiveRMNodes();
+ Assert.assertEquals(numNodes, inactiveRMNodes.size());
+ for (RMNode rmNode : inactiveRMNodes.values()) {
+ Assert.assertEquals(expectedCapability, rmNode.getTotalCapability());
+ Assert.assertEquals(expectedVersion, rmNode.getNodeManagerVersion());
+ }
+ }
+
// Test Delegation token is renewed synchronously so that recover events
// can be processed before any other external incoming events, specifically
// the ContainerFinished event on NM re-registraton.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDFS-11080. Update HttpFS to use
ConfigRedactor. Contributed by Sean Mackrory.
Posted by fa...@apache.org.
HDFS-11080. Update HttpFS to use ConfigRedactor. Contributed by Sean Mackrory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e521c5a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e521c5a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e521c5a
Branch: refs/heads/HADOOP-13345
Commit: 7e521c5a49fbcf88285c102051ea2522edc847b9
Parents: dcc07ad
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Nov 2 19:11:05 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Nov 2 19:11:05 2016 -0700
----------------------------------------------------------------------
.../hadoop/fs/CommonConfigurationKeysPublic.java | 2 ++
.../src/main/resources/core-default.xml | 2 +-
.../org/apache/hadoop/conf/TestConfigRedactor.java | 2 ++
.../java/org/apache/hadoop/lib/server/Server.java | 15 ++++++---------
4 files changed, 11 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index b5b107c..f23dd51 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -770,7 +770,9 @@ public class CommonConfigurationKeysPublic {
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
"hadoop.security.sensitive-config-keys";
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
+ "secret$" + "," +
"password$" + "," +
+ "ssl.keystore.pass$" + "," +
"fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
"fs.azure\\.account.key.*" + "," +
"dfs.webhdfs.oauth2.[a-z]+.token" + "," +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 1e15b8e..327acfa 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -504,7 +504,7 @@
<property>
<name>hadoop.security.sensitive-config-keys</name>
- <value>password$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
+ <value>secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
<description>A comma-separated list of regular expressions to match against
configuration keys that should be redacted where appropriate, for
example, when logging modified properties during a reconfiguration,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
index 81f8f71..eedb9b2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
@@ -47,6 +47,7 @@ public class TestConfigRedactor {
"dfs.webhdfs.oauth2.refresh.token",
"ssl.server.keystore.keypassword",
"ssl.server.keystore.password",
+ "httpfs.ssl.keystore.pass",
"hadoop.security.sensitive-config-keys"
);
for (String key : sensitiveKeys) {
@@ -60,6 +61,7 @@ public class TestConfigRedactor {
"fs.defaultFS",
"dfs.replication",
"ssl.server.keystore.location",
+ "httpfs.config.dir",
"hadoop.security.credstore.java-keystore-provider.password-file"
);
for (String key : normalKeys) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
index 1a0f9ff..82be027 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.lib.server;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.ConfigRedactor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.lib.util.Check;
import org.apache.hadoop.lib.util.ConfigurationUtils;
@@ -482,15 +483,13 @@ public class Server {
}
ConfigurationUtils.injectDefaults(defaultConf, config);
-
+ ConfigRedactor redactor = new ConfigRedactor(config);
for (String name : System.getProperties().stringPropertyNames()) {
String value = System.getProperty(name);
if (name.startsWith(getPrefix() + ".")) {
config.set(name, value);
- if (name.endsWith(".password") || name.endsWith(".secret")) {
- value = "*MASKED*";
- }
- log.info("System property sets {}: {}", name, value);
+ String redacted = redactor.redact(name, value);
+ log.info("System property sets {}: {}", name, redacted);
}
}
@@ -499,10 +498,8 @@ public class Server {
for (Map.Entry<String, String> entry : config) {
String name = entry.getKey();
String value = config.get(entry.getKey());
- if (name.endsWith(".password") || name.endsWith(".secret")) {
- value = "*MASKED*";
- }
- log.debug(" {}: {}", entry.getKey(), value);
+ String redacted = redactor.redact(name, value);
+ log.debug(" {}: {}", entry.getKey(), redacted);
}
log.debug("------------------------------------------------------");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HADOOP-13583. Incorporate
checkcompatibility script which runs Java API Compliance Checker.
Posted by fa...@apache.org.
HADOOP-13583. Incorporate checkcompatibility script which runs Java API Compliance Checker.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cd22d66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cd22d66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cd22d66
Branch: refs/heads/HADOOP-13345
Commit: 0cd22d66691fdbc3fb3e0a35ad1625236c3ae3f7
Parents: aacf214
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Nov 1 16:36:10 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Nov 1 16:36:10 2016 -0700
----------------------------------------------------------------------
BUILDING.txt | 11 +
dev-support/bin/checkcompatibility.py | 345 +++++++++++++++++++++++++++++
2 files changed, 356 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd22d66/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 1fbf8bb..d6f6db7 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -389,3 +389,14 @@ http://www.zlib.net/
Building distributions:
* Build distribution with native code : mvn package [-Pdist][-Pdocs][-Psrc][-Dtar]
+
+----------------------------------------------------------------------------------
+Running compatibility checks with checkcompatibility.py
+
+Invoke `./dev-support/bin/checkcompatibility.py` to run Java API Compliance Checker
+to compare the public Java APIs of two git objects. This can be used by release
+managers to compare the compatibility of a previous and current release.
+
+As an example, this invocation will check the compatibility of interfaces annotated as Public or LimitedPrivate:
+
+./dev-support/bin/checkcompatibility.py --annotation org.apache.hadoop.classification.InterfaceAudience.Public --annotation org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate --include "hadoop.*" branch-2.7.2 trunk
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd22d66/dev-support/bin/checkcompatibility.py
----------------------------------------------------------------------
diff --git a/dev-support/bin/checkcompatibility.py b/dev-support/bin/checkcompatibility.py
new file mode 100755
index 0000000..ad1e9cb
--- /dev/null
+++ b/dev-support/bin/checkcompatibility.py
@@ -0,0 +1,345 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Script which checks Java API compatibility between two revisions of the
+# Java client.
+#
+# Originally sourced from Apache Kudu, which was based on the
+# compatibility checker from the Apache HBase project, but ported to
+# Python for better readability.
+
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+import urllib2
+try:
+ import argparse
+except ImportError:
+ sys.stderr.write("Please install argparse, e.g. via `pip install argparse`.")
+ sys.exit(2)
+
+# Various relative paths
+REPO_DIR = os.getcwd()
+
+def check_output(*popenargs, **kwargs):
+ r"""Run command with arguments and return its output as a byte string.
+ Backported from Python 2.7 as it's implemented as pure python on stdlib.
+ >>> check_output(['/usr/bin/python', '--version'])
+ Python 2.6.2
+ """
+ process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+ output, _ = process.communicate()
+ retcode = process.poll()
+ if retcode:
+ cmd = kwargs.get("args")
+ if cmd is None:
+ cmd = popenargs[0]
+ error = subprocess.CalledProcessError(retcode, cmd)
+ error.output = output
+ raise error
+ return output
+
+def get_repo_dir():
+ """ Return the path to the top of the repo. """
+ dirname, _ = os.path.split(os.path.abspath(__file__))
+ return os.path.join(dirname, "../..")
+
+def get_scratch_dir():
+ """ Return the path to the scratch dir that we build within. """
+ scratch_dir = os.path.join(get_repo_dir(), "target", "compat-check")
+ if not os.path.exists(scratch_dir):
+ os.makedirs(scratch_dir)
+ return scratch_dir
+
+def get_java_acc_dir():
+ """ Return the path where we check out the Java API Compliance Checker. """
+ return os.path.join(get_repo_dir(), "target", "java-acc")
+
+
+def clean_scratch_dir(scratch_dir):
+ """ Clean up and re-create the scratch directory. """
+ if os.path.exists(scratch_dir):
+ logging.info("Removing scratch dir %s...", scratch_dir)
+ shutil.rmtree(scratch_dir)
+ logging.info("Creating empty scratch dir %s...", scratch_dir)
+ os.makedirs(scratch_dir)
+
+
+def checkout_java_tree(rev, path):
+ """ Check out the Java source tree for the given revision into
+ the given path. """
+ logging.info("Checking out %s in %s", rev, path)
+ os.makedirs(path)
+ # Extract java source
+ subprocess.check_call(["bash", '-o', 'pipefail', "-c",
+ ("git archive --format=tar %s | " +
+ "tar -C \"%s\" -xf -") % (rev, path)],
+ cwd=get_repo_dir())
+
+def get_git_hash(revname):
+ """ Convert 'revname' to its SHA-1 hash. """
+ return check_output(["git", "rev-parse", revname],
+ cwd=get_repo_dir()).strip()
+
+def get_repo_name():
+ """Get the name of the repo based on the git remote."""
+ remotes = check_output(["git", "remote", "-v"],
+ cwd=get_repo_dir()).strip().split("\n")
+ # Example output:
+ # origin https://github.com/apache/hadoop.git (fetch)
+ # origin https://github.com/apache/hadoop.git (push)
+ remote_url = remotes[0].split("\t")[1].split(" ")[0]
+ remote = remote_url.split("/")[-1]
+ if remote.endswith(".git"):
+ remote = remote[:-4]
+ return remote
+
+def build_tree(java_path):
+ """ Run the Java build within 'path'. """
+ logging.info("Building in %s...", java_path)
+ subprocess.check_call(["mvn", "-DskipTests", "-Dmaven.javadoc.skip=true",
+ "package"],
+ cwd=java_path)
+
+
+def checkout_java_acc(force):
+ """
+ Check out the Java API Compliance Checker. If 'force' is true, will
+ re-download even if the directory exists.
+ """
+ acc_dir = get_java_acc_dir()
+ if os.path.exists(acc_dir):
+ logging.info("Java ACC is already downloaded.")
+ if not force:
+ return
+ logging.info("Forcing re-download.")
+ shutil.rmtree(acc_dir)
+
+ logging.info("Downloading Java ACC...")
+
+ url = "https://github.com/lvc/japi-compliance-checker/archive/1.8.tar.gz"
+ scratch_dir = get_scratch_dir()
+ path = os.path.join(scratch_dir, os.path.basename(url))
+ jacc = urllib2.urlopen(url)
+ with open(path, 'wb') as w:
+ w.write(jacc.read())
+
+ subprocess.check_call(["tar", "xzf", path],
+ cwd=scratch_dir)
+
+ shutil.move(os.path.join(scratch_dir, "japi-compliance-checker-1.8"),
+ os.path.join(acc_dir))
+
+
+def find_jars(path):
+ """ Return a list of jars within 'path' to be checked for compatibility. """
+ all_jars = set(check_output(["find", path, "-name", "*.jar"]).splitlines())
+
+ return [j for j in all_jars if (
+ "-tests" not in j and
+ "-sources" not in j and
+ "-with-dependencies" not in j)]
+
+def write_xml_file(path, version, jars):
+ """Write the XML manifest file for JACC."""
+ with open(path, "wt") as f:
+ f.write("<version>" + version + "</version>\n")
+ f.write("<archives>")
+ for j in jars:
+ f.write(j + "\n")
+ f.write("</archives>")
+
+def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations):
+ """ Run the compliance checker to compare 'src' and 'dst'. """
+ logging.info("Will check compatibility between original jars:\n\t%s\n" +
+ "and new jars:\n\t%s",
+ "\n\t".join(src_jars),
+ "\n\t".join(dst_jars))
+
+ java_acc_path = os.path.join(get_java_acc_dir(), "japi-compliance-checker.pl")
+
+ src_xml_path = os.path.join(get_scratch_dir(), "src.xml")
+ dst_xml_path = os.path.join(get_scratch_dir(), "dst.xml")
+ write_xml_file(src_xml_path, src_name, src_jars)
+ write_xml_file(dst_xml_path, dst_name, dst_jars)
+
+ out_path = os.path.join(get_scratch_dir(), "report.html")
+
+ args = ["perl", java_acc_path,
+ "-l", get_repo_name(),
+ "-d1", src_xml_path,
+ "-d2", dst_xml_path,
+ "-report-path", out_path]
+
+ if annotations is not None:
+ annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
+ with file(annotations_path, "w") as f:
+ for ann in annotations:
+ print >>f, ann
+ args += ["-annotations-list", annotations_path]
+
+ subprocess.check_call(args)
+
+def filter_jars(jars, include_filters, exclude_filters):
+ """Filter the list of JARs based on include and exclude filters."""
+ filtered = []
+ # Apply include filters
+ for j in jars:
+ found = False
+ basename = os.path.basename(j)
+ for f in include_filters:
+ if f.match(basename):
+ found = True
+ break
+ if found:
+ filtered += [j]
+ else:
+ logging.debug("Ignoring JAR %s", j)
+ # Apply exclude filters
+ exclude_filtered = []
+ for j in filtered:
+ basename = os.path.basename(j)
+ found = False
+ for f in exclude_filters:
+ if f.match(basename):
+ found = True
+ break
+ if found:
+ logging.debug("Ignoring JAR %s", j)
+ else:
+ exclude_filtered += [j]
+
+ return exclude_filtered
+
+
+def main():
+ """Main function."""
+ logging.basicConfig(level=logging.INFO)
+ parser = argparse.ArgumentParser(
+ description="Run Java API Compliance Checker.")
+ parser.add_argument("-f", "--force-download",
+ action="store_true",
+ help="Download dependencies (i.e. Java JAVA_ACC) " +
+ "even if they are already present")
+ parser.add_argument("-i", "--include-file",
+ action="append",
+ dest="include_files",
+ help="Regex filter for JAR files to be included. " +
+ "Applied before the exclude filters. " +
+ "Can be specified multiple times.")
+ parser.add_argument("-e", "--exclude-file",
+ action="append",
+ dest="exclude_files",
+ help="Regex filter for JAR files to be excluded. " +
+ "Applied after the include filters. " +
+ "Can be specified multiple times.")
+ parser.add_argument("-a", "--annotation",
+ action="append",
+ dest="annotations",
+ help="Fully-qualified Java annotation. " +
+ "Java ACC will only check compatibility of " +
+ "annotated classes. Can be specified multiple times.")
+ parser.add_argument("--skip-clean",
+ action="store_true",
+ help="Skip cleaning the scratch directory.")
+ parser.add_argument("--skip-build",
+ action="store_true",
+ help="Skip building the projects.")
+ parser.add_argument("src_rev", nargs=1, help="Source revision.")
+ parser.add_argument("dst_rev", nargs="?", default="HEAD",
+ help="Destination revision. " +
+ "If not specified, will use HEAD.")
+
+ if len(sys.argv) == 1:
+ parser.print_help()
+ sys.exit(1)
+
+ args = parser.parse_args()
+
+ src_rev, dst_rev = args.src_rev[0], args.dst_rev
+
+ logging.info("Source revision: %s", src_rev)
+ logging.info("Destination revision: %s", dst_rev)
+
+ # Construct the JAR regex patterns for filtering.
+ include_filters = []
+ if args.include_files is not None:
+ for f in args.include_files:
+ logging.info("Applying JAR filename include filter: %s", f)
+ include_filters += [re.compile(f)]
+ else:
+ include_filters = [re.compile(".*")]
+
+ exclude_filters = []
+ if args.exclude_files is not None:
+ for f in args.exclude_files:
+ logging.info("Applying JAR filename exclude filter: %s", f)
+ exclude_filters += [re.compile(f)]
+
+ # Construct the annotation list
+ annotations = args.annotations
+ if annotations is not None:
+ logging.info("Filtering classes using %d annotation(s):", len(annotations))
+ for a in annotations:
+ logging.info("\t%s", a)
+
+ # Download deps.
+ checkout_java_acc(args.force_download)
+
+ # Set up the build.
+ scratch_dir = get_scratch_dir()
+ src_dir = os.path.join(scratch_dir, "src")
+ dst_dir = os.path.join(scratch_dir, "dst")
+
+ if args.skip_clean:
+ logging.info("Skipping cleaning the scratch directory")
+ else:
+ clean_scratch_dir(scratch_dir)
+ # Check out the src and dst source trees.
+ checkout_java_tree(get_git_hash(src_rev), src_dir)
+ checkout_java_tree(get_git_hash(dst_rev), dst_dir)
+
+ # Run the build in each.
+ if args.skip_build:
+ logging.info("Skipping the build")
+ else:
+ build_tree(src_dir)
+ build_tree(dst_dir)
+
+ # Find the JARs.
+ src_jars = find_jars(src_dir)
+ dst_jars = find_jars(dst_dir)
+
+ # Filter the JARs.
+ src_jars = filter_jars(src_jars, include_filters, exclude_filters)
+ dst_jars = filter_jars(dst_jars, include_filters, exclude_filters)
+
+ if len(src_jars) == 0 or len(dst_jars) == 0:
+ logging.error("No JARs found! Are your filters too strong?")
+ sys.exit(1)
+
+ run_java_acc(src_rev, src_jars,
+ dst_rev, dst_jars, annotations)
+
+
+if __name__ == "__main__":
+ main()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-4998. Minor cleanup to UGI
use in AdminService. (Daniel Templeton via kasha)
Posted by fa...@apache.org.
YARN-4998. Minor cleanup to UGI use in AdminService. (Daniel Templeton via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/733aa993
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/733aa993
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/733aa993
Branch: refs/heads/HADOOP-13345
Commit: 733aa993134ba324c712590fa92b8ef230b0839a
Parents: 07ab89e
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Mon Oct 31 16:02:50 2016 -0700
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Mon Oct 31 16:26:01 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/server/resourcemanager/AdminService.java | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/733aa993/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 33daf7f..c060659 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -154,8 +154,7 @@ public class AdminService extends CompositeService implements
YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
daemonUser = UserGroupInformation.getCurrentUser();
authorizer = YarnAuthorizationProvider.getInstance(conf);
- authorizer.setAdmins(getAdminAclList(conf), UserGroupInformation
- .getCurrentUser());
+ authorizer.setAdmins(getAdminAclList(conf), daemonUser);
rmId = conf.get(YarnConfiguration.RM_HA_ID);
isCentralizedNodeLabelConfiguration =
@@ -542,8 +541,7 @@ public class AdminService extends CompositeService implements
Configuration conf =
getConfiguration(new Configuration(false),
YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
- authorizer.setAdmins(getAdminAclList(conf), UserGroupInformation
- .getCurrentUser());
+ authorizer.setAdmins(getAdminAclList(conf), daemonUser);
RMAuditLogger.logSuccess(user.getShortUserName(), operation,
"AdminService");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-5809. AsyncDispatcher
possibly invokes multiple shutdown threads when handling exception (Jian He
via Varun Saxena)
Posted by fa...@apache.org.
YARN-5809. AsyncDispatcher possibly invokes multiple shutdown threads when handling exception (Jian He via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07ab89e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07ab89e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07ab89e8
Branch: refs/heads/HADOOP-13345
Commit: 07ab89e8bb3f647cef4f80f39237169a0c6a8520
Parents: 0c49f73
Author: Varun Saxena <va...@apache.org>
Authored: Tue Nov 1 04:19:00 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Tue Nov 1 04:37:03 2016 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07ab89e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index df542ed..42a6819 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -195,6 +195,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
if (exitOnDispatchException
&& (ShutdownHookManager.get().isShutdownInProgress()) == false
&& stopped == false) {
+ stopped = true;
Thread shutDownThread = new Thread(createShutDownThread());
shutDownThread.setName("AsyncDispatcher ShutDown handler");
shutDownThread.start();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-5788. Apps not activiated and
AM limit resource in UI and REST not updated after -replaceLabelsOnNode
(Bibin A Chundatt via Varun Saxena)
Posted by fa...@apache.org.
YARN-5788. Apps not activiated and AM limit resource in UI and REST not updated after -replaceLabelsOnNode (Bibin A Chundatt via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d2d8d25
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d2d8d25
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d2d8d25
Branch: refs/heads/HADOOP-13345
Commit: 7d2d8d25ba0cb10a3c6192d4123f27ede5ef2ba6
Parents: 310aa46
Author: Varun Saxena <va...@apache.org>
Authored: Tue Nov 1 15:32:04 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Tue Nov 1 15:32:04 2016 +0530
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 105 ++++++++++---------
.../TestCapacitySchedulerNodeLabelUpdate.java | 74 ++++++++++++-
2 files changed, 128 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d8d25/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index cfdcb10..d759d47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1126,57 +1126,52 @@ public class CapacityScheduler extends
writeLock.unlock();
}
}
-
+
/**
* Process node labels update on a node.
*/
private void updateLabelsOnNode(NodeId nodeId,
Set<String> newLabels) {
- try {
- writeLock.lock();
- FiCaSchedulerNode node = nodeTracker.getNode(nodeId);
- if (null == node) {
- return;
- }
-
- // Get new partition, we have only one partition per node
- String newPartition;
- if (newLabels.isEmpty()) {
- newPartition = RMNodeLabelsManager.NO_LABEL;
- } else{
- newPartition = newLabels.iterator().next();
- }
+ FiCaSchedulerNode node = nodeTracker.getNode(nodeId);
+ if (null == node) {
+ return;
+ }
- // old partition as well
- String oldPartition = node.getPartition();
+ // Get new partition, we have only one partition per node
+ String newPartition;
+ if (newLabels.isEmpty()) {
+ newPartition = RMNodeLabelsManager.NO_LABEL;
+ } else{
+ newPartition = newLabels.iterator().next();
+ }
- // Update resources of these containers
- for (RMContainer rmContainer : node.getCopiedListOfRunningContainers()) {
- FiCaSchedulerApp application = getApplicationAttempt(
- rmContainer.getApplicationAttemptId());
- if (null != application) {
- application.nodePartitionUpdated(rmContainer, oldPartition,
- newPartition);
- } else{
- LOG.warn("There's something wrong, some RMContainers running on"
- + " a node, but we cannot find SchedulerApplicationAttempt "
- + "for it. Node=" + node.getNodeID() + " applicationAttemptId="
- + rmContainer.getApplicationAttemptId());
- continue;
- }
- }
+ // old partition as well
+ String oldPartition = node.getPartition();
- // Unreserve container on this node
- RMContainer reservedContainer = node.getReservedContainer();
- if (null != reservedContainer) {
- killReservedContainer(reservedContainer);
+ // Update resources of these containers
+ for (RMContainer rmContainer : node.getCopiedListOfRunningContainers()) {
+ FiCaSchedulerApp application = getApplicationAttempt(
+ rmContainer.getApplicationAttemptId());
+ if (null != application) {
+ application.nodePartitionUpdated(rmContainer, oldPartition,
+ newPartition);
+ } else{
+ LOG.warn("There's something wrong, some RMContainers running on"
+ + " a node, but we cannot find SchedulerApplicationAttempt "
+ + "for it. Node=" + node.getNodeID() + " applicationAttemptId="
+ + rmContainer.getApplicationAttemptId());
+ continue;
}
+ }
- // Update node labels after we've done this
- node.updateLabels(newLabels);
- } finally {
- writeLock.unlock();
+ // Unreserve container on this node
+ RMContainer reservedContainer = node.getReservedContainer();
+ if (null != reservedContainer) {
+ killReservedContainer(reservedContainer);
}
+
+ // Update node labels after we've done this
+ node.updateLabels(newLabels);
}
private void updateSchedulerHealth(long now, FiCaSchedulerNode node,
@@ -1371,13 +1366,8 @@ public class CapacityScheduler extends
{
NodeLabelsUpdateSchedulerEvent labelUpdateEvent =
(NodeLabelsUpdateSchedulerEvent) event;
-
- for (Entry<NodeId, Set<String>> entry : labelUpdateEvent
- .getUpdatedNodeToLabels().entrySet()) {
- NodeId id = entry.getKey();
- Set<String> labels = entry.getValue();
- updateLabelsOnNode(id, labels);
- }
+
+ updateNodeLabelsAndQueueResource(labelUpdateEvent);
}
break;
case NODE_UPDATE:
@@ -1482,6 +1472,27 @@ public class CapacityScheduler extends
}
}
+ /**
+ * Process node labels update.
+ */
+ private void updateNodeLabelsAndQueueResource(
+ NodeLabelsUpdateSchedulerEvent labelUpdateEvent) {
+ try {
+ writeLock.lock();
+ for (Entry<NodeId, Set<String>> entry : labelUpdateEvent
+ .getUpdatedNodeToLabels().entrySet()) {
+ NodeId id = entry.getKey();
+ Set<String> labels = entry.getValue();
+ updateLabelsOnNode(id, labels);
+ }
+ Resource clusterResource = getClusterResource();
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
private void addNode(RMNode nodeManager) {
try {
writeLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d8d25/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index 0ae77f2..439e9df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -413,7 +413,7 @@ public class TestCapacitySchedulerNodeLabelUpdate {
rm.close();
}
- @Test(timeout = 3000000)
+ @Test(timeout = 300000)
public void testMoveApplicationWithLabel() throws Exception {
// set node -> label
mgr.addToCluserNodeLabelsWithDefaultExclusivity(
@@ -589,7 +589,49 @@ public class TestCapacitySchedulerNodeLabelUpdate {
rm.close();
}
- @Test (timeout = 60000)
+ @Test
+ public void testAMResourceLimitNodeUpdatePartition() throws Exception {
+ conf.setInt("yarn.scheduler.minimum-allocation-mb", 64);
+ // inject node label manager
+ MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm.getRMContext().setNodeLabelManager(mgr);
+ rm.start();
+ rm.registerNode("h1:1234", 6400);
+ mgr.addToCluserNodeLabelsWithDefaultExclusivity(
+ ImmutableSet.of("x", "y", "z"));
+
+ // .1 percentage of 6400 will be for am
+ checkAMResourceLimit(rm, "a", 640, "");
+ checkAMResourceLimit(rm, "a", 0, "x");
+ checkAMResourceLimit(rm, "a", 0, "y");
+ checkAMResourceLimit(rm, "a", 0, "z");
+
+ mgr.replaceLabelsOnNode(
+ ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+ rm.drainEvents();
+
+ checkAMResourceLimit(rm, "a", 640, "x");
+ checkAMResourceLimit(rm, "a", 0, "y");
+ checkAMResourceLimit(rm, "a", 0, "z");
+ checkAMResourceLimit(rm, "a", 0, "");
+
+ // Switch
+ mgr.replaceLabelsOnNode(
+ ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("y")));
+ rm.drainEvents();
+
+ checkAMResourceLimit(rm, "a", 0, "x");
+ checkAMResourceLimit(rm, "a", 640, "y");
+ checkAMResourceLimit(rm, "a", 0, "z");
+ checkAMResourceLimit(rm, "a", 0, "");
+ }
+
+ @Test(timeout = 60000)
public void testAMResourceUsageWhenNodeUpdatesPartition()
throws Exception {
// set node -> label
@@ -638,8 +680,8 @@ public class TestCapacitySchedulerNodeLabelUpdate {
FiCaSchedulerApp app = cs.getApplicationAttempt(am1.getApplicationAttemptId());
// change h1's label to z
- cs.handle(new NodeLabelsUpdateSchedulerEvent(ImmutableMap.of(nm1.getNodeId(),
- toSet("z"))));
+ cs.handle(new NodeLabelsUpdateSchedulerEvent(
+ ImmutableMap.of(nm1.getNodeId(), toSet("z"))));
// Now the resources also should change from x to z. Verify AM and normal
// used resource are successfully changed.
@@ -677,4 +719,28 @@ public class TestCapacitySchedulerNodeLabelUpdate {
rm.close();
}
+
+ private void checkAMResourceLimit(MockRM rm, String queuename, int memory,
+ String label) throws InterruptedException {
+ Assert.assertEquals(memory,
+ waitForResourceUpdate(rm, queuename, memory, label, 3000L));
+ }
+
+ private long waitForResourceUpdate(MockRM rm, String queuename, long memory,
+ String label, long timeout) throws InterruptedException {
+ long start = System.currentTimeMillis();
+ long memorySize = 0;
+ while (System.currentTimeMillis() - start < timeout) {
+ CapacityScheduler scheduler =
+ (CapacityScheduler) rm.getResourceScheduler();
+ CSQueue queue = scheduler.getQueue(queuename);
+ memorySize =
+ queue.getQueueResourceUsage().getAMLimit(label).getMemorySize();
+ if (memory == memorySize) {
+ return memorySize;
+ }
+ Thread.sleep(100);
+ }
+ return memorySize;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDFS-10499.
TestNameNodeMetadataConsistency#testGenerationStampInFuture Fails
Intermittently. Contributed by Yiqun Lin.
Posted by fa...@apache.org.
HDFS-10499. TestNameNodeMetadataConsistency#testGenerationStampInFuture Fails Intermittently. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/808f5d89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/808f5d89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/808f5d89
Branch: refs/heads/HADOOP-13345
Commit: 808f5d8985323bc1afa79a1c5d24b229f535ec3b
Parents: 5b577f4
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Tue Nov 1 21:32:52 2016 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Tue Nov 1 21:32:52 2016 +0530
----------------------------------------------------------------------
.../TestNameNodeMetadataConsistency.java | 45 +++++++++++---------
1 file changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808f5d89/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
index 6c2832e..fb9705d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
@@ -27,6 +27,9 @@ import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import com.google.common.base.Supplier;
import org.junit.After;
import org.junit.Before;
@@ -34,7 +37,6 @@ import org.junit.Test;
import java.io.IOException;
import java.io.OutputStream;
-import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -73,8 +75,7 @@ public class TestNameNodeMetadataConsistency {
* safe mode while it is in startup mode.
*/
@Test
- public void testGenerationStampInFuture() throws
- IOException, InterruptedException {
+ public void testGenerationStampInFuture() throws Exception {
cluster.waitActive();
FileSystem fs = cluster.getFileSystem();
@@ -105,9 +106,7 @@ public class TestNameNodeMetadataConsistency {
cluster.getNameNode().getNamesystem().getBlockManager());
cluster.restartDataNode(dnProps);
- waitTil(TimeUnit.SECONDS.toMillis(SCAN_WAIT));
- cluster.triggerBlockReports();
- waitTil(TimeUnit.SECONDS.toMillis(SCAN_WAIT));
+ waitForNumBytes(TEST_DATA_IN_FUTURE.length());
// Make sure that we find all written bytes in future block
assertEquals(TEST_DATA_IN_FUTURE.length(),
@@ -122,9 +121,7 @@ public class TestNameNodeMetadataConsistency {
* hence we should not have positive count of Blocks in future.
*/
@Test
- public void testEnsureGenStampsIsStartupOnly() throws
- IOException, InterruptedException {
-
+ public void testEnsureGenStampsIsStartupOnly() throws Exception {
String testData = " This is test data";
cluster.restartDataNodes();
cluster.restartNameNodes();
@@ -153,21 +150,31 @@ public class TestNameNodeMetadataConsistency {
cluster.getNameNode().getNamesystem().writeUnlock();
cluster.restartDataNode(dnProps);
- waitTil(TimeUnit.SECONDS.toMillis(SCAN_WAIT));
- cluster.triggerBlockReports();
- waitTil(TimeUnit.SECONDS.toMillis(SCAN_WAIT));
-
+ waitForNumBytes(0);
// Make sure that there are no bytes in future since isInStartupSafe
// mode is not true.
assertEquals(0, cluster.getNameNode().getBytesWithFutureGenerationStamps());
}
- private void waitTil(long waitPeriod) {
- try {
- Thread.sleep(waitPeriod);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
+ private void waitForNumBytes(final int numBytes) throws Exception {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+
+ @Override
+ public Boolean get() {
+ try {
+ cluster.triggerBlockReports();
+ // Compare the number of bytes
+ if (cluster.getNameNode().getBytesWithFutureGenerationStamps()
+ == numBytes) {
+ return true;
+ }
+ } catch (Exception e) {
+ // Ignore the exception
+ }
+
+ return false;
+ }
+ }, SCAN_WAIT * 1000, 60000);
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-11088. Quash unnecessary
safemode WARN message during NameNode startup. Contributed by Yiqun Lin.
Posted by fa...@apache.org.
HADOOP-11088. Quash unnecessary safemode WARN message during NameNode startup. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cad93d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cad93d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cad93d5
Branch: refs/heads/HADOOP-13345
Commit: 5cad93d5c753112931ce97dba966d1ce6b53724b
Parents: 7534aee
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Nov 3 15:30:57 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Nov 3 15:31:07 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/blockmanagement/BlockManagerSafeMode.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cad93d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
index a0017a1..ed304f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.util.Daemon;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -127,8 +128,8 @@ class BlockManagerSafeMode {
this.threshold = conf.getFloat(DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY,
DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_DEFAULT);
if (this.threshold > 1.0) {
- LOG.warn("The threshold value should't be greater than 1, threshold: {}",
- threshold);
+ LOG.warn("The threshold value shouldn't be greater than 1, " +
+ "threshold: {}", threshold);
}
this.datanodeThreshold = conf.getInt(
DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY,
@@ -171,7 +172,8 @@ class BlockManagerSafeMode {
startTime = monotonicNow();
setBlockTotal(total);
if (areThresholdsMet()) {
- leaveSafeMode(true);
+ boolean exitResult = leaveSafeMode(false);
+ Preconditions.checkState(exitResult, "Failed to leave safe mode.");
} else {
// enter safe mode
status = BMSafeModeStatus.PENDING_THRESHOLD;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HDFS-11031. Add additional unit
test for DataNode startup behavior when volumes fail. Contributed by
Mingliang Liu.
Posted by fa...@apache.org.
HDFS-11031. Add additional unit test for DataNode startup behavior when volumes fail. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb5cc0dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb5cc0dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb5cc0dc
Branch: refs/heads/HADOOP-13345
Commit: cb5cc0dc53d3543e13b7b7cf9425780ded0538cc
Parents: fc2b69e
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Wed Nov 2 10:35:47 2016 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Wed Nov 2 10:35:47 2016 +0530
----------------------------------------------------------------------
.../datanode/TestDataNodeVolumeFailure.java | 111 ++++++++++++++++++-
1 file changed, 110 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb5cc0dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
index 4aba4e3..49cb513 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
@@ -73,17 +73,24 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
-import com.google.common.base.Supplier;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.TrueFileFilter;
+
+import com.google.common.base.Supplier;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Fine-grain testing of block files and locations after volume failure.
*/
public class TestDataNodeVolumeFailure {
+ private final static Logger LOG = LoggerFactory.getLogger(
+ TestDataNodeVolumeFailure.class);
final private int block_size = 512;
MiniDFSCluster cluster = null;
private Configuration conf;
@@ -414,6 +421,108 @@ public class TestDataNodeVolumeFailure {
}
/**
+ * Test if there is volume failure, the DataNode will fail to start.
+ *
+ * We fail a volume by setting the parent directory non-writable.
+ */
+ @Test (timeout = 120000)
+ public void testDataNodeFailToStartWithVolumeFailure() throws Exception {
+ // Method to simulate volume failures is currently not supported on Windows.
+ assumeNotWindows();
+
+ failedDir = new File(dataDir, "failedDir");
+ assertTrue("Failed to fail a volume by setting it non-writable",
+ failedDir.mkdir() && failedDir.setReadOnly());
+
+ startNewDataNodeWithDiskFailure(new File(failedDir, "newDir1"), false);
+ }
+
+ /**
+ * DataNode will start and tolerate one failing disk according to config.
+ *
+ * We fail a volume by setting the parent directory non-writable.
+ */
+ @Test (timeout = 120000)
+ public void testDNStartAndTolerateOneVolumeFailure() throws Exception {
+ // Method to simulate volume failures is currently not supported on Windows.
+ assumeNotWindows();
+
+ failedDir = new File(dataDir, "failedDir");
+ assertTrue("Failed to fail a volume by setting it non-writable",
+ failedDir.mkdir() && failedDir.setReadOnly());
+
+ startNewDataNodeWithDiskFailure(new File(failedDir, "newDir1"), true);
+ }
+
+ /**
+ * Test if data directory is not readable/writable, DataNode won't start.
+ */
+ @Test (timeout = 120000)
+ public void testDNFailToStartWithDataDirNonWritable() throws Exception {
+ // Method to simulate volume failures is currently not supported on Windows.
+ assumeNotWindows();
+
+ final File readOnlyDir = new File(dataDir, "nonWritable");
+ assertTrue("Set the data dir permission non-writable",
+ readOnlyDir.mkdir() && readOnlyDir.setReadOnly());
+
+ startNewDataNodeWithDiskFailure(new File(readOnlyDir, "newDir1"), false);
+ }
+
+ /**
+ * DataNode will start and tolerate one non-writable data directory
+ * according to config.
+ */
+ @Test (timeout = 120000)
+ public void testDNStartAndTolerateOneDataDirNonWritable() throws Exception {
+ // Method to simulate volume failures is currently not supported on Windows.
+ assumeNotWindows();
+
+ final File readOnlyDir = new File(dataDir, "nonWritable");
+ assertTrue("Set the data dir permission non-writable",
+ readOnlyDir.mkdir() && readOnlyDir.setReadOnly());
+ startNewDataNodeWithDiskFailure(new File(readOnlyDir, "newDir1"), true);
+ }
+
+ /**
+ * @param badDataDir bad data dir, either disk failure or non-writable
+ * @param tolerated true if one volume failure is allowed else false
+ */
+ private void startNewDataNodeWithDiskFailure(File badDataDir,
+ boolean tolerated) throws Exception {
+ final File data5 = new File(dataDir, "data5");
+ final String newDirs = badDataDir.toString() + "," + data5.toString();
+ final Configuration newConf = new Configuration(conf);
+ newConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
+ LOG.info("Setting dfs.datanode.data.dir for new DataNode as {}", newDirs);
+ newConf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,
+ tolerated ? 1 : 0);
+
+ // bring up one more DataNode
+ assertEquals(repl, cluster.getDataNodes().size());
+ cluster.startDataNodes(newConf, 1, false, null, null);
+ assertEquals(repl + 1, cluster.getDataNodes().size());
+
+ if (tolerated) {
+ // create new file and it should be able to replicate to 3 nodes
+ final Path p = new Path("/test1.txt");
+ DFSTestUtil.createFile(fs, p, block_size * blocks_num, (short) 3, 1L);
+ DFSTestUtil.waitReplication(fs, p, (short) (repl + 1));
+ } else {
+ // DataNode should stop soon if it does not tolerate disk failure
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ final String bpid = cluster.getNamesystem().getBlockPoolId();
+ final BPOfferService bpos = cluster.getDataNodes().get(2)
+ .getBPOfferService(bpid);
+ return !bpos.isAlive();
+ }
+ }, 100, 30 * 1000);
+ }
+ }
+
+ /**
* verifies two things:
* 1. number of locations of each block in the name node
* matches number of actual files
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDFS-11097. Fix warnings for
deprecated StorageReceivedDeletedBlocks constructor. Contributed by Yiqun
Lin.
Posted by fa...@apache.org.
HDFS-11097. Fix warnings for deprecated StorageReceivedDeletedBlocks constructor. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b71907b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b71907b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b71907b2
Branch: refs/heads/HADOOP-13345
Commit: b71907b2ae3d8f05b4332e06d52ec2096681ea6b
Parents: 9ee0e31
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Nov 3 10:27:52 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Nov 3 10:27:52 2016 -0700
----------------------------------------------------------------------
.../DatanodeProtocolServerSideTranslatorPB.java | 4 +++-
.../server/protocol/StorageReceivedDeletedBlocks.java | 7 +++++++
.../blockmanagement/TestPendingReconstruction.java | 12 ++++++++----
.../server/datanode/TestIncrementalBrVariations.java | 4 +++-
.../hdfs/server/namenode/NNThroughputBenchmark.java | 5 +++--
.../hadoop/hdfs/server/namenode/TestDeadDatanode.java | 3 ++-
6 files changed, 26 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71907b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
index b3d5f0d..257adf9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionResponsePro
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
@@ -230,7 +231,8 @@ public class DatanodeProtocolServerSideTranslatorPB implements
info[i] = new StorageReceivedDeletedBlocks(
PBHelperClient.convert(sBlock.getStorage()), rdBlocks);
} else {
- info[i] = new StorageReceivedDeletedBlocks(sBlock.getStorageUuid(), rdBlocks);
+ info[i] = new StorageReceivedDeletedBlocks(
+ new DatanodeStorage(sBlock.getStorageUuid()), rdBlocks);
}
}
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71907b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java
index 29216cf..12b0351 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java
@@ -28,6 +28,9 @@ public class StorageReceivedDeletedBlocks {
final DatanodeStorage storage;
private final ReceivedDeletedBlockInfo[] blocks;
+ /**
+ * @deprecated Use {@link #getStorage()} instead
+ */
@Deprecated
public String getStorageID() {
return storage.getStorageID();
@@ -41,6 +44,10 @@ public class StorageReceivedDeletedBlocks {
return blocks;
}
+ /**
+ * @deprecated Use {@link #StorageReceivedDeletedBlocks(
+ * DatanodeStorage, ReceivedDeletedBlockInfo[])} instead
+ */
@Deprecated
public StorageReceivedDeletedBlocks(final String storageID,
final ReceivedDeletedBlockInfo[] blocks) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71907b2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java
index c30f630..ba3be91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
@@ -312,7 +313,8 @@ public class TestPendingReconstruction {
DatanodeRegistration dnR = datanodes.get(i).getDNRegistrationForBP(
poolId);
StorageReceivedDeletedBlocks[] report = {
- new StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored",
+ new StorageReceivedDeletedBlocks(
+ new DatanodeStorage("Fake-storage-ID-Ignored"),
new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo(
blocks[0], BlockStatus.RECEIVED_BLOCK, "") }) };
cluster.getNameNodeRpc().blockReceivedAndDeleted(dnR, poolId, report);
@@ -330,9 +332,11 @@ public class TestPendingReconstruction {
DatanodeRegistration dnR = datanodes.get(i).getDNRegistrationForBP(
poolId);
StorageReceivedDeletedBlocks[] report =
- { new StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored",
- new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo(
- blocks[0], BlockStatus.RECEIVED_BLOCK, "") }) };
+ { new StorageReceivedDeletedBlocks(
+ new DatanodeStorage("Fake-storage-ID-Ignored"),
+ new ReceivedDeletedBlockInfo[] {
+ new ReceivedDeletedBlockInfo(
+ blocks[0], BlockStatus.RECEIVED_BLOCK, "")}) };
cluster.getNameNodeRpc().blockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71907b2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
index f2570c2..78b5cf9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.*;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
@@ -172,7 +173,8 @@ public class TestIncrementalBrVariations {
assertTrue(foundBlockOnStorage);
reports[i] =
- new StorageReceivedDeletedBlocks(volume.getStorageID(), rdbi);
+ new StorageReceivedDeletedBlocks(
+ new DatanodeStorage(volume.getStorageID()), rdbi);
if (splitReports) {
// If we are splitting reports then send the report for this storage now.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71907b2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index ad336ef..4b9db07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -1037,7 +1037,7 @@ public class NNThroughputBenchmark implements Tool {
blocks[i], ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,
null) };
StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks(
- targetStorageID, rdBlocks) };
+ new DatanodeStorage(targetStorageID), rdBlocks) };
dataNodeProto.blockReceivedAndDeleted(receivedDNReg, bpid, report);
}
}
@@ -1157,7 +1157,8 @@ public class NNThroughputBenchmark implements Tool {
loc.getBlock().getLocalBlock(),
ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) };
StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks(
- datanodes[dnIdx].storage.getStorageID(), rdBlocks) };
+ new DatanodeStorage(datanodes[dnIdx].storage.getStorageID()),
+ rdBlocks) };
dataNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration,
bpid, report);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71907b2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index 14b14b2..6998934 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -102,7 +102,8 @@ public class TestDeadDatanode {
ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,
null) };
StorageReceivedDeletedBlocks[] storageBlocks = {
- new StorageReceivedDeletedBlocks(reg.getDatanodeUuid(), blocks) };
+ new StorageReceivedDeletedBlocks(
+ new DatanodeStorage(reg.getDatanodeUuid()), blocks) };
// Ensure blockReceived call from dead datanode is not rejected with
// IOException, since it's async, but the node remains unregistered.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDFS-11076. Add unit test for
extended Acls. Contributed by Chen Liang
Posted by fa...@apache.org.
HDFS-11076. Add unit test for extended Acls. Contributed by Chen Liang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7534aee0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7534aee0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7534aee0
Branch: refs/heads/HADOOP-13345
Commit: 7534aee09af47c6961100588312da8d133be1b27
Parents: 7154a20
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Nov 3 13:49:10 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Nov 3 13:49:10 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/TestExtendedAcls.java | 441 +++++++++++++++++++
1 file changed, 441 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7534aee0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestExtendedAcls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestExtendedAcls.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestExtendedAcls.java
new file mode 100644
index 0000000..b983a8e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestExtendedAcls.java
@@ -0,0 +1,441 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
+
+import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;
+import static org.apache.hadoop.fs.permission.AclEntryType.MASK;
+import static org.apache.hadoop.fs.permission.AclEntryType.USER;
+import static org.apache.hadoop.fs.permission.FsAction.NONE;
+import static org.apache.hadoop.fs.permission.FsAction.READ;
+import static org.apache.hadoop.fs.permission.FsAction.READ_EXECUTE;
+import static org.apache.hadoop.fs.permission.FsAction.READ_WRITE;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry;
+
+import static org.apache.hadoop.fs.permission.AclEntryScope.DEFAULT;
+import static org.apache.hadoop.fs.permission.FsAction.ALL;
+import static org.apache.hadoop.fs.permission.AclEntryType.GROUP;
+import static org.apache.hadoop.fs.permission.AclEntryType.OTHER;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * A class for testing the behavior of HDFS directory and file ACL.
+ */
+public class TestExtendedAcls {
+
+ private static MiniDFSCluster cluster;
+ private static Configuration conf;
+
+ private static final short REPLICATION = 3;
+
+ private static DistributedFileSystem hdfs;
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ conf = new Configuration();
+ conf.setBoolean(DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(REPLICATION)
+ .build();
+ cluster.waitActive();
+ hdfs = cluster.getFileSystem();
+ }
+
+ @AfterClass
+ public static void shutdown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ /**
+ * Set default ACL to a directory.
+ * Create subdirectory, it must have default acls set.
+ * Create sub file and it should have default acls.
+ * @throws IOException
+ */
+ @Test
+ public void testDefaultAclNewChildDirFile() throws IOException {
+ Path parent = new Path("/testDefaultAclNewChildDirFile");
+ List<AclEntry> acls = Lists.newArrayList(
+ aclEntry(DEFAULT, USER, "foo", ALL));
+
+ hdfs.mkdirs(parent);
+ hdfs.setAcl(parent, acls);
+
+ // create sub directory
+ Path childDir = new Path(parent, "childDir");
+ hdfs.mkdirs(childDir);
+ // the sub directory should have the default acls
+ AclEntry[] childDirExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, READ_EXECUTE)
+ };
+ AclStatus childDirAcl = hdfs.getAclStatus(childDir);
+ assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());
+
+ // create sub file
+ Path childFile = new Path(parent, "childFile");
+ hdfs.create(childFile).close();
+ // the sub file should have the default acls
+ AclEntry[] childFileExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_EXECUTE)
+ };
+ AclStatus childFileAcl = hdfs.getAclStatus(childFile);
+ assertArrayEquals(
+ childFileExpectedAcl, childFileAcl.getEntries().toArray());
+
+ hdfs.delete(parent, true);
+ }
+
+ /**
+ * Set default ACL to a directory and make sure existing sub dirs/files
+ * does not have default acl.
+ * @throws IOException
+ */
+ @Test
+ public void testDefaultAclExistingDirFile() throws Exception {
+ Path parent = new Path("/testDefaultAclExistingDirFile");
+ hdfs.mkdirs(parent);
+ // the old acls
+ List<AclEntry> acls1 = Lists.newArrayList(
+ aclEntry(DEFAULT, USER, "foo", ALL));
+ // the new acls
+ List<AclEntry> acls2 = Lists.newArrayList(
+ aclEntry(DEFAULT, USER, "foo", READ_EXECUTE));
+ // set parent to old acl
+ hdfs.setAcl(parent, acls1);
+
+ Path childDir = new Path(parent, "childDir");
+ hdfs.mkdirs(childDir);
+ // the sub directory should also have the old acl
+ AclEntry[] childDirExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, READ_EXECUTE)
+ };
+ AclStatus childDirAcl = hdfs.getAclStatus(childDir);
+ assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());
+
+ Path childFile = new Path(childDir, "childFile");
+ // the sub file should also have the old acl
+ hdfs.create(childFile).close();
+ AclEntry[] childFileExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_EXECUTE)
+ };
+ AclStatus childFileAcl = hdfs.getAclStatus(childFile);
+ assertArrayEquals(
+ childFileExpectedAcl, childFileAcl.getEntries().toArray());
+
+ // now change parent to new acls
+ hdfs.setAcl(parent, acls2);
+
+ // sub directory and sub file should still have the old acls
+ childDirAcl = hdfs.getAclStatus(childDir);
+ assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());
+ childFileAcl = hdfs.getAclStatus(childFile);
+ assertArrayEquals(
+ childFileExpectedAcl, childFileAcl.getEntries().toArray());
+
+ // now remove the parent acls
+ hdfs.removeAcl(parent);
+
+ // sub directory and sub file should still have the old acls
+ childDirAcl = hdfs.getAclStatus(childDir);
+ assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());
+ childFileAcl = hdfs.getAclStatus(childFile);
+ assertArrayEquals(
+ childFileExpectedAcl, childFileAcl.getEntries().toArray());
+
+ // check changing the access mode of the file
+ // mask out the access of group other for testing
+ hdfs.setPermission(childFile, new FsPermission((short)0640));
+ boolean canAccess =
+ tryAccess(childFile, "other", new String[]{"other"}, READ);
+ assertFalse(canAccess);
+ hdfs.delete(parent, true);
+ }
+
+ /**
+ * Verify that access acl does not get inherited on newly created subdir/file.
+ * @throws IOException
+ */
+ @Test
+ public void testAccessAclNotInherited() throws IOException {
+ Path parent = new Path("/testAccessAclNotInherited");
+ hdfs.mkdirs(parent);
+ // parent have both access acl and default acl
+ List<AclEntry> acls = Lists.newArrayList(
+ aclEntry(DEFAULT, USER, "foo", READ_EXECUTE),
+ aclEntry(ACCESS, USER, READ_WRITE),
+ aclEntry(ACCESS, GROUP, READ),
+ aclEntry(ACCESS, OTHER, READ),
+ aclEntry(ACCESS, USER, "bar", ALL));
+ hdfs.setAcl(parent, acls);
+
+ Path childDir = new Path(parent, "childDir");
+ hdfs.mkdirs(childDir);
+ // subdirectory should only have the default acl inherited
+ AclEntry[] childDirExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, READ),
+ aclEntry(DEFAULT, USER, READ_WRITE),
+ aclEntry(DEFAULT, USER, "foo", READ_EXECUTE),
+ aclEntry(DEFAULT, GROUP, READ),
+ aclEntry(DEFAULT, MASK, READ_EXECUTE),
+ aclEntry(DEFAULT, OTHER, READ)
+ };
+ AclStatus childDirAcl = hdfs.getAclStatus(childDir);
+ assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());
+
+ Path childFile = new Path(parent, "childFile");
+ hdfs.create(childFile).close();
+ // sub file should only have the default acl inherited
+ AclEntry[] childFileExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, READ)
+ };
+ AclStatus childFileAcl = hdfs.getAclStatus(childFile);
+ assertArrayEquals(
+ childFileExpectedAcl, childFileAcl.getEntries().toArray());
+
+ hdfs.delete(parent, true);
+ }
+
+ /**
+ * Create a parent dir and set default acl to allow foo read/write access.
+ * Create a sub dir and set default acl to allow bar group read/write access.
+ * parent dir/file can not be viewed/appended by bar group.
+ * parent dir/child dir/file can be viewed/appended by bar group.
+ * @throws Exception
+ */
+ @Test
+ public void testGradSubdirMoreAccess() throws Exception {
+ Path parent = new Path("/testGradSubdirMoreAccess");
+ hdfs.mkdirs(parent);
+ List<AclEntry> aclsParent = Lists.newArrayList(
+ aclEntry(DEFAULT, USER, "foo", READ_EXECUTE));
+ List<AclEntry> aclsChild = Lists.newArrayList(
+ aclEntry(DEFAULT, GROUP, "bar", READ_WRITE));
+
+ hdfs.setAcl(parent, aclsParent);
+ AclEntry[] parentDirExpectedAcl = new AclEntry[] {
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", READ_EXECUTE),
+ aclEntry(DEFAULT, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, MASK, READ_EXECUTE),
+ aclEntry(DEFAULT, OTHER, READ_EXECUTE)
+ };
+ AclStatus parentAcl = hdfs.getAclStatus(parent);
+ assertArrayEquals(parentDirExpectedAcl, parentAcl.getEntries().toArray());
+
+ Path childDir = new Path(parent, "childDir");
+ hdfs.mkdirs(childDir);
+ hdfs.modifyAclEntries(childDir, aclsChild);
+ // child dir should inherit the default acls from parent, plus bar group
+ AclEntry[] childDirExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", READ_EXECUTE),
+ aclEntry(DEFAULT, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, GROUP, "bar", READ_WRITE),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, READ_EXECUTE)
+ };
+ AclStatus childDirAcl = hdfs.getAclStatus(childDir);
+ assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());
+
+ Path parentFile = new Path(parent, "parentFile");
+ hdfs.create(parentFile).close();
+ hdfs.setPermission(parentFile, new FsPermission((short)0640));
+ // parent dir/parent file allows foo to access but not bar group
+ AclEntry[] parentFileExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, READ_EXECUTE)
+ };
+ AclStatus parentFileAcl = hdfs.getAclStatus(parentFile);
+ assertArrayEquals(parentFileExpectedAcl,
+ parentFileAcl.getEntries().toArray());
+
+ Path childFile = new Path(childDir, "childFile");
+ hdfs.create(childFile).close();
+ hdfs.setPermission(childFile, new FsPermission((short)0640));
+ // child dir/child file allows foo user and bar group to access
+ AclEntry[] childFileExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, "bar", READ_WRITE)
+ };
+ AclStatus childFileAcl = hdfs.getAclStatus(childFile);
+ assertArrayEquals(
+ childFileExpectedAcl, childFileAcl.getEntries().toArray());
+
+ // parent file should not be accessible for bar group
+ assertFalse(tryAccess(parentFile, "barUser", new String[]{"bar"}, READ));
+ // child file should be accessible for bar group
+ assertTrue(tryAccess(childFile, "barUser", new String[]{"bar"}, READ));
+ // parent file should be accessible for foo user
+ assertTrue(tryAccess(parentFile, "foo", new String[]{"fooGroup"}, READ));
+ // child file should be accessible for foo user
+ assertTrue(tryAccess(childFile, "foo", new String[]{"fooGroup"}, READ));
+
+ hdfs.delete(parent, true);
+ }
+
+ /**
+ * Verify that sub directory can restrict acl with acl inherited from parent.
+ * Create a parent dir and set default to allow foo and bar full access
+ * Create a sub dir and set default to restrict bar to empty access
+ *
+ * parent dir/file can be viewed by foo
+ * parent dir/child dir/file can be viewed by foo
+ * parent dir/child dir/file can not be viewed by bar
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testRestrictAtSubDir() throws Exception {
+ Path parent = new Path("/testRestrictAtSubDir");
+ hdfs.mkdirs(parent);
+ List<AclEntry> aclsParent = Lists.newArrayList(
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, "bar", ALL)
+ );
+ hdfs.setAcl(parent, aclsParent);
+ AclEntry[] parentDirExpectedAcl = new AclEntry[] {
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, GROUP, "bar", ALL),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, READ_EXECUTE)
+ };
+ AclStatus parentAcl = hdfs.getAclStatus(parent);
+ assertArrayEquals(parentDirExpectedAcl, parentAcl.getEntries().toArray());
+
+ Path parentFile = new Path(parent, "parentFile");
+ hdfs.create(parentFile).close();
+ hdfs.setPermission(parentFile, new FsPermission((short)0640));
+ AclEntry[] parentFileExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, "bar", ALL),
+ };
+ AclStatus parentFileAcl = hdfs.getAclStatus(parentFile);
+ assertArrayEquals(
+ parentFileExpectedAcl, parentFileAcl.getEntries().toArray());
+
+ Path childDir = new Path(parent, "childDir");
+ hdfs.mkdirs(childDir);
+ List<AclEntry> newAclsChild = Lists.newArrayList(
+ aclEntry(DEFAULT, GROUP, "bar", NONE)
+ );
+ hdfs.modifyAclEntries(childDir, newAclsChild);
+ AclEntry[] childDirExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, "bar", ALL),
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, GROUP, "bar", NONE),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, READ_EXECUTE)
+ };
+ AclStatus childDirAcl = hdfs.getAclStatus(childDir);
+ assertArrayEquals(childDirExpectedAcl, childDirAcl.getEntries().toArray());
+
+ Path childFile = new Path(childDir, "childFile");
+ hdfs.create(childFile).close();
+ hdfs.setPermission(childFile, new FsPermission((short)0640));
+ AclEntry[] childFileExpectedAcl = new AclEntry[] {
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, "bar", NONE)
+ };
+ AclStatus childFileAcl = hdfs.getAclStatus(childFile);
+ assertArrayEquals(
+ childFileExpectedAcl, childFileAcl.getEntries().toArray());
+
+ // child file should not be accessible for bar group
+ assertFalse(tryAccess(childFile, "barUser", new String[]{"bar"}, READ));
+ // child file should be accessible for foo user
+ assertTrue(tryAccess(childFile, "foo", new String[]{"fooGroup"}, READ));
+ // parent file should be accessible for bar group
+ assertTrue(tryAccess(parentFile, "barUser", new String[]{"bar"}, READ));
+ // parent file should be accessible for foo user
+ assertTrue(tryAccess(parentFile, "foo", new String[]{"fooGroup"}, READ));
+
+ hdfs.delete(parent, true);
+ }
+
+ private boolean tryAccess(Path path, String user,
+ String[] group, FsAction action) throws Exception {
+ UserGroupInformation testUser =
+ UserGroupInformation.createUserForTesting(
+ user, group);
+ FileSystem fs = testUser.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws Exception {
+ return FileSystem.get(conf);
+ }
+ });
+
+ boolean canAccess;
+ try {
+ fs.access(path, action);
+ canAccess = true;
+ } catch (AccessControlException e) {
+ canAccess = false;
+ }
+ return canAccess;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HADOOP-13631 S3Guard: implement
move() for LocalMetadataStore, add unit tests
Posted by fa...@apache.org.
HADOOP-13631 S3Guard: implement move() for LocalMetadataStore, add unit tests
Also, change DirListingMetadata#setAuthoritative() to take a boolean arg.
Setting this to false is a way to trigger clients to re-consult the backing
store if, for example, a user adds new files to a directory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/958d4cdb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/958d4cdb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/958d4cdb
Branch: refs/heads/HADOOP-13345
Commit: 958d4cdb45507e30b84b9582c42152f07a03b2ee
Parents: 0576745
Author: Aaron Fabbri <fa...@apache.org>
Authored: Tue Sep 27 18:19:48 2016 -0700
Committer: Aaron Fabbri <fa...@apache.org>
Committed: Fri Nov 4 20:30:59 2016 -0700
----------------------------------------------------------------------
.../fs/s3a/s3guard/DirListingMetadata.java | 5 +-
.../fs/s3a/s3guard/LocalMetadataStore.java | 44 +++++++++++--
.../hadoop/fs/s3a/s3guard/MetadataStore.java | 26 ++++++--
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 69 ++++++++++++++++++--
.../fs/s3a/s3guard/TestDirListingMetadata.java | 2 +-
5 files changed, 127 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/958d4cdb/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
index 1838f42..5bfb6c0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
@@ -97,9 +97,10 @@ public class DirListingMetadata {
/**
* Marks this directory listing as full and authoritative.
+ * @param authoritative see {@link #isAuthoritative()}.
*/
- public void setAuthoritative() {
- isAuthoritative = true;
+ public void setAuthoritative(boolean authoritative) {
+ this.isAuthoritative = authoritative;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/958d4cdb/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
index d47d85e..9f69189 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
@@ -28,6 +28,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
@@ -94,7 +95,7 @@ public class LocalMetadataStore implements MetadataStore {
// Remove target file/dir
fileHash.remove(path);
- // Update parent dir listing, if any
+ // Update this and parent dir listing, if any
dirHashDeleteFile(path);
if (recursive) {
@@ -116,9 +117,39 @@ public class LocalMetadataStore implements MetadataStore {
}
@Override
- public void move(Path src, Path dst) throws IOException {
- // TODO implement me
- throw new IOException("LocalMetadataStore#move() not implemented yet");
+ public void move(Collection<Path> pathsToDelete,
+ Collection<PathMetadata> pathsToCreate) throws IOException {
+
+ Preconditions.checkNotNull(pathsToDelete, "pathsToDelete is null");
+ Preconditions.checkNotNull(pathsToCreate, "pathsToCreate is null");
+ Preconditions.checkArgument(pathsToDelete.size() == pathsToCreate.size(),
+ "Must supply same number of paths to delete/create.");
+
+ // I feel dirty for using reentrant lock. :-|
+ synchronized (this) {
+
+ // 1. Delete pathsToDelete
+ for (Path p : pathsToDelete) {
+ delete(p);
+ }
+
+ // 2. Create new destination path metadata
+ for (PathMetadata meta : pathsToCreate) {
+ put(meta);
+ }
+
+ // 3. We now know full contents of all dirs in destination subtree
+ for (PathMetadata meta : pathsToCreate) {
+ FileStatus status = meta.getFileStatus();
+ if (status == null || status.isDirectory()) {
+ continue;
+ }
+ DirListingMetadata dir = listChildren(status.getPath());
+ if (dir != null) { // could be evicted already
+ dir.setAuthoritative(true);
+ }
+ }
+ }
}
@Override
@@ -205,6 +236,11 @@ public class LocalMetadataStore implements MetadataStore {
* Update dirHash to reflect deletion of file 'f'. Call with lock held.
*/
private void dirHashDeleteFile(Path path) {
+
+ /* If this path is a dir, remove its listing */
+ dirHash.remove(path);
+
+ /* Remove this path from parent's dir listing */
Path parent = path.getParent();
if (parent != null) {
DirListingMetadata dir = dirHash.get(parent);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/958d4cdb/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
index a6d29da..68009e3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.s3a.s3guard;
import java.io.Closeable;
import java.io.IOException;
+import java.util.Collection;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -84,14 +85,29 @@ public interface MetadataStore extends Closeable {
DirListingMetadata listChildren(Path path) throws IOException;
/**
- * Moves metadata from {@code src} to {@code dst}, including all descendants
- * recursively.
+ * Record the effects of a {@link FileSystem#rename(Path, Path)} in the
+ * MetadataStore. Clients provide explicit enumeration of the affected
+ * paths (recursively), before and after the rename.
*
- * @param src the source path
- * @param dst the new path of the file/dir after the move
+ * On the need to provide an enumeration of directory trees instead of just
+ * source and destination paths:
+ * Since a MetadataStore does not have to track all metadata for the
+ * underlying storage system, and a new MetadataStore may be created on an
+ * existing underlying filesystem, this move() may be the first time the
+ * MetadataStore sees the affected paths. Therefore, simply providing src
+ * and destination paths may not be enough to record the deletions (under
+ * src path) and creations (at destination) that are happening during the
+ * rename().
+ *
+ * @param pathsToDelete Collection of all paths that were removed from the
+ * source directory tree of the move.
+ * @param pathsToCreate Collection of all PathMetadata for the new paths
+ * that were created at the destination of the rename
+ * ().
* @throws IOException if there is an error
*/
- void move(Path src, Path dst) throws IOException;
+ void move(Collection<Path> pathsToDelete, Collection<PathMetadata>
+ pathsToCreate) throws IOException;
/**
* Saves metadata for exactly one path. For a deeply nested path, this method
http://git-wip-us.apache.org/repos/asf/hadoop/blob/958d4cdb/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index 701f355..462f8f9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,6 +31,7 @@ import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
@@ -285,10 +285,54 @@ public abstract class MetadataStoreTestBase extends Assert {
ms.listChildren(new Path("/a1/b1x")));
}
- @Ignore
+ @Test
public void testMove() throws Exception {
- // TODO
- throw new RuntimeException("TODO: implement move and tests.");
+ // Create test dir structure
+ createNewDirs("/a1", "/a2", "/a3");
+ createNewDirs("/a1/b1", "/a1/b2");
+ putListStatusFiles("/a1/b1", false, "/a1/b1/file1", "/a1/b1/file2");
+
+ // Assert root listing as expected
+ DirListingMetadata dirMeta = ms.listChildren(new Path("/"));
+ assertNotNull("Listing root", dirMeta);
+ Collection<PathMetadata> entries = dirMeta.getListing();
+ assertListingsEqual(entries, "/a1", "/a2", "/a3");
+
+ // Assert src listing as expected
+ dirMeta = ms.listChildren(new Path("/a1/b1"));
+ assertNotNull("Listing /a1/b1", dirMeta);
+ entries = dirMeta.getListing();
+ assertListingsEqual(entries, "/a1/b1/file1", "/a1/b1/file2");
+
+ // Do the move(): rename(/a1/b1, /b1)
+ Collection<Path> srcPaths = Arrays.asList(new Path("/a1/b1"),
+ new Path("/a1/b1/file1"), new Path("/a1/b1/file2"));
+
+ ArrayList<PathMetadata> destMetas = new ArrayList<>();
+ destMetas.add(new PathMetadata(makeDirStatus("/b1")));
+ destMetas.add(new PathMetadata(makeFileStatus("/b1/file1", 100)));
+ destMetas.add(new PathMetadata(makeFileStatus("/b1/file2", 100)));
+ ms.move(srcPaths, destMetas);
+
+ // Assert src is no longer there
+ dirMeta = ms.listChildren(new Path("/a1"));
+ assertNotNull("Listing /a1", dirMeta);
+ entries = dirMeta.getListing();
+ assertListingsEqual(entries, "/a1/b2");
+
+ PathMetadata meta = ms.get(new Path("/a1/b1/file1"));
+ // TODO allow return of PathMetadata with isDeleted == true
+ assertNull("Src path deleted", meta);
+
+ // Assert dest looks right
+ meta = ms.get(new Path("/b1/file1"));
+ assertNotNull("dest file not null", meta);
+ verifyBasicFileStatus(meta);
+
+ dirMeta = ms.listChildren(new Path("/b1"));
+ assertNotNull("dest listing not null", dirMeta);
+ entries = dirMeta.getListing();
+ assertListingsEqual(entries, "/b1/file1", "/b1/file2");
}
@@ -296,8 +340,18 @@ public abstract class MetadataStoreTestBase extends Assert {
* Helper functions.
*/
+ /** Builds array of Path objects based on parent dir and filenames. */
+ private Path[] buildPaths(String parent, String... paths) {
+
+ Path[] output = new Path[paths.length];
+ for (int i = 0; i < paths.length; i++) {
+ output[i] = new Path(parent, paths[i]);
+ }
+ return output;
+ }
+
/** Modifies paths input array and returns it. */
- private String[] buildPaths(String parent, String ...paths) {
+ private String[] buildPathStrings(String parent, String... paths) {
for (int i = 0; i < paths.length; i++) {
Path p = new Path(parent, paths[i]);
paths[i] = p.toString();
@@ -306,13 +360,14 @@ public abstract class MetadataStoreTestBase extends Assert {
}
private void commonTestPutListStatus(final String parent) throws IOException {
- putListStatusFiles(parent, true, buildPaths(parent, "file1", "file2",
+ putListStatusFiles(parent, true, buildPathStrings(parent, "file1", "file2",
"file3"));
DirListingMetadata dirMeta = ms.listChildren(new Path(parent));
assertNotNull("list after putListStatus", dirMeta);
Collection<PathMetadata> entries = dirMeta.getListing();
assertNotNull("listStatus has entries", entries);
- assertListingsEqual(entries, buildPaths(parent, "file1", "file2", "file3"));
+ assertListingsEqual(entries, buildPathStrings(parent, "file1", "file2",
+ "file3"));
}
private void setupListStatus() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/958d4cdb/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
index 99db001..05d3876 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
@@ -122,7 +122,7 @@ public class TestDirListingMetadata {
assertNotNull(meta.getListing());
assertTrue(meta.getListing().isEmpty());
assertFalse(meta.isAuthoritative());
- meta.setAuthoritative();
+ meta.setAuthoritative(true);
assertTrue(meta.isAuthoritative());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: MAPREDUCE-6795. Update the
document for JobConf#setNumReduceTasks. Contributed by Yiqun Lin.
Posted by fa...@apache.org.
MAPREDUCE-6795. Update the document for JobConf#setNumReduceTasks. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/235f39b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/235f39b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/235f39b7
Branch: refs/heads/HADOOP-13345
Commit: 235f39b7c44c5907bd4de063892905577c35cd80
Parents: 9d0c2ad
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 1 20:51:35 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 1 20:52:12 2016 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/mapred/JobConf.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/235f39b7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
index f2b0aae..85b4c93 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
@@ -1347,9 +1347,14 @@ public class JobConf extends Configuration {
* <b id="NoOfReduces">How many reduces?</b>
*
* <p>The right number of reduces seems to be <code>0.95</code> or
- * <code>1.75</code> multiplied by (<<i>no. of nodes</i>> *
- * <a href="{@docRoot}/../mapred-default.html#mapreduce.tasktracker.reduce.tasks.maximum">
- * mapreduce.tasktracker.reduce.tasks.maximum</a>).
+ * <code>1.75</code> multiplied by (
+ * <i>available memory for reduce tasks</i>
+ * (The value of this should be smaller than
+ * numNodes * yarn.nodemanager.resource.memory-mb
+ * since the resource of memory is shared by map tasks and other
+ * applications) /
+ * <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.reduce.memory.mb">
+ * mapreduce.reduce.memory.mb</a>).
* </p>
*
* <p>With <code>0.95</code> all of the reduces can launch immediately and
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-8500. Fix javadoc jars to
not contain entire target directory.
Posted by fa...@apache.org.
HADOOP-8500. Fix javadoc jars to not contain entire target directory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ba74bef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ba74bef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ba74bef
Branch: refs/heads/HADOOP-13345
Commit: 7ba74befcff2f1836c2d5123d64e92a3c7a8898c
Parents: 733aa99
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Oct 31 16:37:25 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Oct 31 16:37:25 2016 -0700
----------------------------------------------------------------------
hadoop-dist/pom.xml | 45 +++++++++++++++++++++++++++++++++++++++++++++
hadoop-project/pom.xml | 3 ---
2 files changed, 45 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ba74bef/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 4a7feaa..ad30db9 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -68,6 +68,51 @@
<configuration>
</configuration>
</plugin>
+
+ <!--
+ The "dist" profile automatically attaches many artifacts
+ to the module.
+
+ Disable these, since we are only concerned with assembling
+ the dist layout.
+ -->
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-jar</id>
+ <phase>never</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>hadoop-java-sources</id>
+ <phase>never</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>module-javadocs</id>
+ <phase>never</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Disable install plugin too, since we disabled all the artifacts -->
+ <plugin>
+ <artifactId>maven-install-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-install</id>
+ <phase>never</phase>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ba74bef/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7e3b1cc..ca567c5 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1568,9 +1568,6 @@
<goals>
<goal>jar</goal>
</goals>
- <configuration>
- <destDir>${project.build.directory}</destDir>
- </configuration>
</execution>
</executions>
</plugin>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HADOOP-13717. Normalize
daemonization behavior of the diskbalancer with balancer and mover.
Posted by fa...@apache.org.
HADOOP-13717. Normalize daemonization behavior of the diskbalancer with balancer and mover.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd7f5911
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd7f5911
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd7f5911
Branch: refs/heads/HADOOP-13345
Commit: bd7f5911c726c505a654d5d30b7dc5a92dca05bd
Parents: 0cd22d6
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Nov 1 16:48:45 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Nov 1 16:48:45 2016 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd7f5911/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 2299980..2c59004 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -116,6 +116,7 @@ function hdfscmd_case
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSAdmin
;;
diskbalancer)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DiskBalancerCLI
;;
envvars)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HADOOP-13345 Improved
consistency.. Fix compilation for latest trunk.
Posted by fa...@apache.org.
HADOOP-13345 Improved consistency.. Fix compilation for latest trunk.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56b715f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56b715f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56b715f8
Branch: refs/heads/HADOOP-13345
Commit: 56b715f8e249e1c926aa7c1038ff6776ec181912
Parents: 958d4cd
Author: Aaron Fabbri <fa...@apache.org>
Authored: Fri Nov 4 20:33:10 2016 -0700
Committer: Aaron Fabbri <fa...@apache.org>
Committed: Fri Nov 4 20:33:10 2016 -0700
----------------------------------------------------------------------
.../fs/s3a/s3guard/TestDirListingMetadata.java | 59 ++++++++++++--------
1 file changed, 35 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56b715f8/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
index 05d3876..3b0a059 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
@@ -37,6 +37,8 @@ import org.junit.rules.ExpectedException;
*/
public class TestDirListingMetadata {
+ private static final String TEST_OWNER = "hadoop";
+
@Rule
public ExpectedException exception = ExpectedException.none();
@@ -72,11 +74,11 @@ public class TestDirListingMetadata {
public void testListing() {
Path path = new Path("/path");
PathMetadata pathMeta1 = new PathMetadata(
- new S3AFileStatus(true, true, new Path(path, "dir1")));
+ new S3AFileStatus(true, new Path(path, "dir1"), TEST_OWNER));
PathMetadata pathMeta2 = new PathMetadata(
- new S3AFileStatus(true, false, new Path(path, "dir2")));
+ new S3AFileStatus(true, new Path(path, "dir2"), TEST_OWNER));
PathMetadata pathMeta3 = new PathMetadata(
- new S3AFileStatus(123, 456, new Path(path, "file1"), 789));
+ new S3AFileStatus(123, 456, new Path(path, "file1"), 8192, TEST_OWNER));
List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
DirListingMetadata meta = new DirListingMetadata(path, listing, false);
assertEquals(path, meta.getPath());
@@ -91,14 +93,7 @@ public class TestDirListingMetadata {
@Test
public void testListingUnmodifiable() {
Path path = new Path("/path");
- PathMetadata pathMeta1 = new PathMetadata(
- new S3AFileStatus(true, true, new Path(path, "dir1")));
- PathMetadata pathMeta2 = new PathMetadata(
- new S3AFileStatus(true, false, new Path(path, "dir2")));
- PathMetadata pathMeta3 = new PathMetadata(
- new S3AFileStatus(123, 456, new Path(path, "file1"), 789));
- List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
- DirListingMetadata meta = new DirListingMetadata(path, listing, false);
+ DirListingMetadata meta = makeTwoDirsOneFile(path);
assertNotNull(meta.getListing());
exception.expect(UnsupportedOperationException.class);
meta.getListing().clear();
@@ -130,11 +125,11 @@ public class TestDirListingMetadata {
public void testGet() {
Path path = new Path("/path");
PathMetadata pathMeta1 = new PathMetadata(
- new S3AFileStatus(true, true, new Path(path, "dir1")));
+ new S3AFileStatus(true, new Path(path, "dir1"), TEST_OWNER));
PathMetadata pathMeta2 = new PathMetadata(
- new S3AFileStatus(true, false, new Path(path, "dir2")));
+ new S3AFileStatus(true, new Path(path, "dir2"), TEST_OWNER));
PathMetadata pathMeta3 = new PathMetadata(
- new S3AFileStatus(123, 456, new Path(path, "file1"), 789));
+ new S3AFileStatus(123, 456, new Path(path, "file1"), 8192, TEST_OWNER));
List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
DirListingMetadata meta = new DirListingMetadata(path, listing, false);
assertEquals(path, meta.getPath());
@@ -181,11 +176,11 @@ public class TestDirListingMetadata {
public void testPut() {
Path path = new Path("/path");
PathMetadata pathMeta1 = new PathMetadata(
- new S3AFileStatus(true, true, new Path(path, "dir1")));
+ new S3AFileStatus(true, new Path(path, "dir1"), TEST_OWNER));
PathMetadata pathMeta2 = new PathMetadata(
- new S3AFileStatus(true, false, new Path(path, "dir2")));
+ new S3AFileStatus(true, new Path(path, "dir2"), TEST_OWNER));
PathMetadata pathMeta3 = new PathMetadata(
- new S3AFileStatus(123, 456, new Path(path, "file1"), 789));
+ new S3AFileStatus(123, 456, new Path(path, "file1"), 8192, TEST_OWNER));
List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
DirListingMetadata meta = new DirListingMetadata(path, listing, false);
assertEquals(path, meta.getPath());
@@ -196,7 +191,7 @@ public class TestDirListingMetadata {
assertTrue(meta.getListing().contains(pathMeta3));
assertFalse(meta.isAuthoritative());
PathMetadata pathMeta4 = new PathMetadata(
- new S3AFileStatus(true, true, new Path(path, "dir3")));
+ new S3AFileStatus(true, new Path(path, "dir3"), TEST_OWNER));
meta.put(pathMeta4.getFileStatus());
assertTrue(meta.getListing().contains(pathMeta4));
assertEquals(pathMeta4, meta.get(pathMeta4.getFileStatus().getPath()));
@@ -217,7 +212,7 @@ public class TestDirListingMetadata {
DirListingMetadata meta = new DirListingMetadata(path, null, false);
exception.expect(NullPointerException.class);
exception.expectMessage(notNullValue(String.class));
- meta.put(new S3AFileStatus(true, true, null));
+ meta.put(new S3AFileStatus(true, null, TEST_OWNER));
}
@Test
@@ -226,7 +221,7 @@ public class TestDirListingMetadata {
DirListingMetadata meta = new DirListingMetadata(path, null, false);
exception.expect(IllegalArgumentException.class);
exception.expectMessage(notNullValue(String.class));
- meta.put(new S3AFileStatus(true, true, new Path("/")));
+ meta.put(new S3AFileStatus(true, new Path("/"), TEST_OWNER));
}
@Test
@@ -235,18 +230,19 @@ public class TestDirListingMetadata {
DirListingMetadata meta = new DirListingMetadata(path, null, false);
exception.expect(IllegalArgumentException.class);
exception.expectMessage(notNullValue(String.class));
- meta.put(new S3AFileStatus(true, true, new Path("/different/ancestor")));
+ meta.put(new S3AFileStatus(true, new Path("/different/ancestor"),
+ TEST_OWNER));
}
@Test
public void testRemove() {
Path path = new Path("/path");
PathMetadata pathMeta1 = new PathMetadata(
- new S3AFileStatus(true, true, new Path(path, "dir1")));
+ new S3AFileStatus(true, new Path(path, "dir1"), TEST_OWNER));
PathMetadata pathMeta2 = new PathMetadata(
- new S3AFileStatus(true, false, new Path(path, "dir2")));
+ new S3AFileStatus(true, new Path(path, "dir2"), TEST_OWNER));
PathMetadata pathMeta3 = new PathMetadata(
- new S3AFileStatus(123, 456, new Path(path, "file1"), 789));
+ new S3AFileStatus(123, 456, new Path(path, "file1"), 8192, TEST_OWNER));
List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
DirListingMetadata meta = new DirListingMetadata(path, listing, false);
assertEquals(path, meta.getPath());
@@ -287,4 +283,19 @@ public class TestDirListingMetadata {
exception.expectMessage(notNullValue(String.class));
meta.remove(new Path("/different/ancestor"));
}
+
+ /*
+ * Create DirListingMetadata with two dirs and one file living in directory
+ * 'parent'
+ */
+ private static DirListingMetadata makeTwoDirsOneFile(Path parent) {
+ PathMetadata pathMeta1 = new PathMetadata(
+ new S3AFileStatus(true, new Path(parent, "dir1"), TEST_OWNER));
+ PathMetadata pathMeta2 = new PathMetadata(
+ new S3AFileStatus(true, new Path(parent, "dir2"), TEST_OWNER));
+ PathMetadata pathMeta3 = new PathMetadata(
+ new S3AFileStatus(123, 456, new Path(parent, "file1"), 8192, TEST_OWNER));
+ List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
+ return new DirListingMetadata(parent, listing, false);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-3732. Change
NodeHeartbeatResponse.java and RegisterNodeManagerResponse.java as abstract
classes. Contributed by Devaraj K.
Posted by fa...@apache.org.
YARN-3732. Change NodeHeartbeatResponse.java and RegisterNodeManagerResponse.java as abstract classes. Contributed by Devaraj K.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d0c2ad5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d0c2ad5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d0c2ad5
Branch: refs/heads/HADOOP-13345
Commit: 9d0c2ad5a5e3a93c8c48f66ab9b21756d42b645a
Parents: 7d2d8d2
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Fri Oct 28 12:11:43 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Nov 1 16:25:45 2016 +0530
----------------------------------------------------------------------
.../protocolrecords/NodeHeartbeatResponse.java | 95 ++++++++++++--------
.../RegisterNodeManagerResponse.java | 40 +++++----
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 15 ++--
.../pb/RegisterNodeManagerResponsePBImpl.java | 15 ++--
4 files changed, 99 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d0c2ad5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
index 09cafaf..7568bbb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
@@ -32,62 +32,85 @@ import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
-public interface NodeHeartbeatResponse {
- int getResponseId();
- NodeAction getNodeAction();
+/**
+ * Node Manager's heartbeat response.
+ */
+public abstract class NodeHeartbeatResponse {
+ public abstract int getResponseId();
+
+ public abstract NodeAction getNodeAction();
- List<ContainerId> getContainersToCleanup();
- List<ContainerId> getContainersToBeRemovedFromNM();
+ public abstract List<ContainerId> getContainersToCleanup();
- List<ApplicationId> getApplicationsToCleanup();
+ public abstract List<ContainerId> getContainersToBeRemovedFromNM();
+
+ public abstract List<ApplicationId> getApplicationsToCleanup();
// This tells NM the collectors' address info of related apps
- Map<ApplicationId, String> getAppCollectorsMap();
- void setAppCollectorsMap(Map<ApplicationId, String> appCollectorsMap);
+ public abstract Map<ApplicationId, String> getAppCollectorsMap();
- void setResponseId(int responseId);
- void setNodeAction(NodeAction action);
+ public abstract void setAppCollectorsMap(
+ Map<ApplicationId, String> appCollectorsMap);
- MasterKey getContainerTokenMasterKey();
- void setContainerTokenMasterKey(MasterKey secretKey);
-
- MasterKey getNMTokenMasterKey();
- void setNMTokenMasterKey(MasterKey secretKey);
+ public abstract void setResponseId(int responseId);
+
+ public abstract void setNodeAction(NodeAction action);
- void addAllContainersToCleanup(List<ContainerId> containers);
+ public abstract MasterKey getContainerTokenMasterKey();
+
+ public abstract void setContainerTokenMasterKey(MasterKey secretKey);
+
+ public abstract MasterKey getNMTokenMasterKey();
+
+ public abstract void setNMTokenMasterKey(MasterKey secretKey);
+
+ public abstract void addAllContainersToCleanup(List<ContainerId> containers);
// This tells NM to remove finished containers from its context. Currently, NM
// will remove finished containers from its context only after AM has actually
// received the finished containers in a previous allocate response
- void addContainersToBeRemovedFromNM(List<ContainerId> containers);
-
- void addAllApplicationsToCleanup(List<ApplicationId> applications);
+ public abstract void addContainersToBeRemovedFromNM(
+ List<ContainerId> containers);
- List<SignalContainerRequest> getContainersToSignalList();
- void addAllContainersToSignal(List<SignalContainerRequest> containers);
- long getNextHeartBeatInterval();
- void setNextHeartBeatInterval(long nextHeartBeatInterval);
-
- String getDiagnosticsMessage();
+ public abstract void addAllApplicationsToCleanup(
+ List<ApplicationId> applications);
+
+ public abstract List<SignalContainerRequest> getContainersToSignalList();
+
+ public abstract void addAllContainersToSignal(
+ List<SignalContainerRequest> containers);
+
+ public abstract long getNextHeartBeatInterval();
+
+ public abstract void setNextHeartBeatInterval(long nextHeartBeatInterval);
- void setDiagnosticsMessage(String diagnosticsMessage);
+ public abstract String getDiagnosticsMessage();
+
+ public abstract void setDiagnosticsMessage(String diagnosticsMessage);
// Credentials (i.e. hdfs tokens) needed by NodeManagers for application
// localizations and logAggreations.
- Map<ApplicationId, ByteBuffer> getSystemCredentialsForApps();
+ public abstract Map<ApplicationId, ByteBuffer> getSystemCredentialsForApps();
- void setSystemCredentialsForApps(
+ public abstract void setSystemCredentialsForApps(
Map<ApplicationId, ByteBuffer> systemCredentials);
- boolean getAreNodeLabelsAcceptedByRM();
- void setAreNodeLabelsAcceptedByRM(boolean areNodeLabelsAcceptedByRM);
+ public abstract boolean getAreNodeLabelsAcceptedByRM();
+
+ public abstract void setAreNodeLabelsAcceptedByRM(
+ boolean areNodeLabelsAcceptedByRM);
+
+ public abstract Resource getResource();
+
+ public abstract void setResource(Resource resource);
+
+ public abstract List<Container> getContainersToDecrease();
- Resource getResource();
- void setResource(Resource resource);
+ public abstract void addAllContainersToDecrease(
+ Collection<Container> containersToDecrease);
- List<Container> getContainersToDecrease();
- void addAllContainersToDecrease(Collection<Container> containersToDecrease);
+ public abstract ContainerQueuingLimit getContainerQueuingLimit();
- ContainerQueuingLimit getContainerQueuingLimit();
- void setContainerQueuingLimit(ContainerQueuingLimit containerQueuingLimit);
+ public abstract void setContainerQueuingLimit(
+ ContainerQueuingLimit containerQueuingLimit);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d0c2ad5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
index fb1da4c..675b375 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
@@ -22,34 +22,40 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
-public interface RegisterNodeManagerResponse {
- MasterKey getContainerTokenMasterKey();
+/**
+ * Node Manager's register response.
+ */
+public abstract class RegisterNodeManagerResponse {
+ public abstract MasterKey getContainerTokenMasterKey();
+
+ public abstract void setContainerTokenMasterKey(MasterKey secretKey);
+
+ public abstract MasterKey getNMTokenMasterKey();
- void setContainerTokenMasterKey(MasterKey secretKey);
+ public abstract void setNMTokenMasterKey(MasterKey secretKey);
- MasterKey getNMTokenMasterKey();
+ public abstract NodeAction getNodeAction();
- void setNMTokenMasterKey(MasterKey secretKey);
+ public abstract void setNodeAction(NodeAction nodeAction);
- NodeAction getNodeAction();
+ public abstract long getRMIdentifier();
- void setNodeAction(NodeAction nodeAction);
+ public abstract void setRMIdentifier(long rmIdentifier);
- long getRMIdentifier();
+ public abstract String getDiagnosticsMessage();
- void setRMIdentifier(long rmIdentifier);
+ public abstract void setDiagnosticsMessage(String diagnosticsMessage);
- String getDiagnosticsMessage();
+ public abstract void setRMVersion(String version);
- void setDiagnosticsMessage(String diagnosticsMessage);
+ public abstract String getRMVersion();
- void setRMVersion(String version);
+ public abstract Resource getResource();
- String getRMVersion();
+ public abstract void setResource(Resource resource);
- Resource getResource();
- void setResource(Resource resource);
+ public abstract boolean getAreNodeLabelsAcceptedByRM();
- boolean getAreNodeLabelsAcceptedByRM();
- void setAreNodeLabelsAcceptedByRM(boolean areNodeLabelsAcceptedByRM);
+ public abstract void setAreNodeLabelsAcceptedByRM(
+ boolean areNodeLabelsAcceptedByRM);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d0c2ad5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index cd85241..37bf570 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
@@ -57,12 +56,14 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.impl.pb.ContainerQueuingLimitPBImpl;
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
-
-public class NodeHeartbeatResponsePBImpl extends
- ProtoBase<NodeHeartbeatResponseProto> implements NodeHeartbeatResponse {
- NodeHeartbeatResponseProto proto = NodeHeartbeatResponseProto.getDefaultInstance();
- NodeHeartbeatResponseProto.Builder builder = null;
- boolean viaProto = false;
+/**
+ * PBImpl class for NodeHeartbeatResponse.
+ */
+public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse {
+ private NodeHeartbeatResponseProto proto = NodeHeartbeatResponseProto
+ .getDefaultInstance();
+ private NodeHeartbeatResponseProto.Builder builder = null;
+ private boolean viaProto = false;
private List<ContainerId> containersToCleanup = null;
private List<ContainerId> containersToBeRemovedFromNM = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d0c2ad5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
index 56b675b..2121fd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
@@ -32,11 +31,15 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
-
-public class RegisterNodeManagerResponsePBImpl extends ProtoBase<RegisterNodeManagerResponseProto> implements RegisterNodeManagerResponse {
- RegisterNodeManagerResponseProto proto = RegisterNodeManagerResponseProto.getDefaultInstance();
- RegisterNodeManagerResponseProto.Builder builder = null;
- boolean viaProto = false;
+/**
+ * PBImpl class for RegisterNodeManagerResponse.
+ */
+public class RegisterNodeManagerResponsePBImpl
+ extends RegisterNodeManagerResponse {
+ private RegisterNodeManagerResponseProto proto =
+ RegisterNodeManagerResponseProto.getDefaultInstance();
+ private RegisterNodeManagerResponseProto.Builder builder = null;
+ private boolean viaProto = false;
private Resource resource = null;
private MasterKey containerTokenMasterKey = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDFS-10756. Expose getTrashRoot to
HTTPFS and WebHDFS. Contributed by Yuanbo Liu.
Posted by fa...@apache.org.
HDFS-10756. Expose getTrashRoot to HTTPFS and WebHDFS. Contributed by Yuanbo Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8bab3dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8bab3dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8bab3dc
Branch: refs/heads/HADOOP-13345
Commit: d8bab3dcb693b2773ede9a6e4f71ae85ee056f79
Parents: 95665a6
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Nov 4 18:06:55 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Nov 4 18:06:55 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 3 +-
.../hadoop/hdfs/DFSOpsCountStatistics.java | 1 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 21 +++
.../hadoop/hdfs/web/resources/GetOpParam.java | 1 +
.../hadoop/fs/http/client/HttpFSFileSystem.java | 31 +++-
.../hadoop/fs/http/server/FSOperations.java | 23 +++
.../http/server/HttpFSParametersProvider.java | 1 +
.../hadoop/fs/http/server/HttpFSServer.java | 7 +
.../src/site/markdown/index.md | 2 +
.../fs/http/client/BaseTestHttpFSWith.java | 185 +++++++++++--------
.../hadoop/fs/http/server/TestHttpFSServer.java | 70 ++++++-
.../web/resources/NamenodeWebHdfsMethods.java | 12 ++
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 34 +++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 41 +++-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 22 +++
15 files changed, 366 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 375af90..529745c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -113,6 +113,7 @@ public abstract class FileSystem extends Configured implements Closeable {
public static final int SHUTDOWN_HOOK_PRIORITY = 10;
public static final String TRASH_PREFIX = ".Trash";
+ public static final String USER_HOME_PREFIX = "/user";
/** FileSystem cache */
static final Cache CACHE = new Cache();
@@ -1965,7 +1966,7 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
public Path getHomeDirectory() {
return this.makeQualified(
- new Path("/user/"+System.getProperty("user.name")));
+ new Path(USER_HOME_PREFIX + "/" + System.getProperty("user.name")));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
index d631dd4..4b2a761 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
@@ -64,6 +64,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
GET_STATUS(CommonStatisticNames.OP_GET_STATUS),
GET_STORAGE_POLICIES("op_get_storage_policies"),
GET_STORAGE_POLICY("op_get_storage_policy"),
+ GET_TRASH_ROOT("op_get_trash_root"),
GET_XATTR("op_get_xattr"),
LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS),
LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 782f113..92d8a13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -1622,6 +1622,27 @@ public class WebHdfsFileSystem extends FileSystem
}
@Override
+ public Path getTrashRoot(Path path) {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOT);
+
+ final HttpOpParam.Op op = GetOpParam.Op.GETTRASHROOT;
+ try {
+ String strTrashPath = new FsPathResponseRunner<String>(op, path) {
+ @Override
+ String decodeResponse(Map<?, ?> json) throws IOException {
+ return JsonUtilClient.getPath(json);
+ }
+ }.run();
+ return new Path(strTrashPath).makeQualified(getUri(), null);
+ } catch(IOException e) {
+ LOG.warn("Cannot find trash root of " + path, e);
+ // keep the same behavior with dfs
+ return super.getTrashRoot(path).makeQualified(getUri(), null);
+ }
+ }
+
+ @Override
public void access(final Path path, final FsAction mode) throws IOException {
final HttpOpParam.Op op = GetOpParam.Op.CHECKACCESS;
new FsPathRunner(op, path, new FsActionParam(mode)).run();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
index 8fdf581..635e6d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
@@ -37,6 +37,7 @@ public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
GET_BLOCK_LOCATIONS(false, HttpURLConnection.HTTP_OK),
GETACLSTATUS(false, HttpURLConnection.HTTP_OK),
GETXATTRS(false, HttpURLConnection.HTTP_OK),
+ GETTRASHROOT(false, HttpURLConnection.HTTP_OK),
LISTXATTRS(false, HttpURLConnection.HTTP_OK),
NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 77e3323..6cc3909 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -128,6 +128,8 @@ public class HttpFSFileSystem extends FileSystem
public static final String HOME_DIR_JSON = "Path";
+ public static final String TRASH_DIR_JSON = "Path";
+
public static final String SET_REPLICATION_JSON = "boolean";
public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream";
@@ -203,7 +205,7 @@ public class HttpFSFileSystem extends FileSystem
OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET),
GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET),
GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET),
- INSTRUMENTATION(HTTP_GET), GETACLSTATUS(HTTP_GET),
+ INSTRUMENTATION(HTTP_GET), GETACLSTATUS(HTTP_GET), GETTRASHROOT(HTTP_GET),
APPEND(HTTP_POST), CONCAT(HTTP_POST), TRUNCATE(HTTP_POST),
CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT),
SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT),
@@ -819,6 +821,33 @@ public class HttpFSFileSystem extends FileSystem
}
/**
+ * Get the root directory of Trash for a path in HDFS.
+ * 1. File in encryption zone returns /ez1/.Trash/username.
+ * 2. File not in encryption zone, or encountered exception when checking
+ * the encryption zone of the path, returns /users/username/.Trash.
+ * Caller appends either Current or checkpoint timestamp
+ * for trash destination.
+ * The default implementation returns "/user/username/.Trash".
+ * @param fullPath the trash root of the path to be determined.
+ * @return trash root
+ */
+ @Override
+ public Path getTrashRoot(Path fullPath) {
+ Map<String, String> params = new HashMap<>();
+ params.put(OP_PARAM, Operation.GETTRASHROOT.toString());
+ try {
+ HttpURLConnection conn = getConnection(
+ Operation.GETTRASHROOT.getMethod(), params, fullPath, true);
+ HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
+ return new Path((String) json.get(TRASH_DIR_JSON));
+ } catch (IOException ex) {
+ LOG.warn("Cannot find trash root of " + fullPath, ex);
+ return super.getTrashRoot(fullPath);
+ }
+ }
+
+ /**
* Set owner of a path (i.e. a file or a directory).
* The parameters username and groupname cannot both be null.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index 001bc92..ebdb73e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -1057,6 +1057,29 @@ public class FSOperations {
}
/**
+ * Executor that performs getting trash root FileSystemAccess
+ * files system operation.
+ */
+ @InterfaceAudience.Private
+ public static class FSTrashRoot
+ implements FileSystemAccess.FileSystemExecutor<JSONObject> {
+ private Path path;
+ public FSTrashRoot(String path) {
+ this.path = new Path(path);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public JSONObject execute(FileSystem fs) throws IOException {
+ Path trashRoot = fs.getTrashRoot(this.path);
+ JSONObject json = new JSONObject();
+ json.put(HttpFSFileSystem.TRASH_DIR_JSON, trashRoot.toUri().getPath());
+ return json;
+ }
+
+ }
+
+ /**
* Executor that gets the ACL information for a given file.
*/
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
index 25585c5..9e89405 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
@@ -60,6 +60,7 @@ public class HttpFSParametersProvider extends ParametersProvider {
PARAMS_DEF.put(Operation.GETFILECHECKSUM, new Class[]{});
PARAMS_DEF.put(Operation.GETFILEBLOCKLOCATIONS, new Class[]{});
PARAMS_DEF.put(Operation.GETACLSTATUS, new Class[]{});
+ PARAMS_DEF.put(Operation.GETTRASHROOT, new Class[]{});
PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{});
PARAMS_DEF.put(Operation.APPEND, new Class[]{DataParam.class});
PARAMS_DEF.put(Operation.CONCAT, new Class[]{SourcesParam.class});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
index 677bca7..3d81ac2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
@@ -339,6 +339,13 @@ public class HttpFSServer {
response = Response.ok(json).type(MediaType.APPLICATION_JSON).build();
break;
}
+ case GETTRASHROOT: {
+ FSOperations.FSTrashRoot command = new FSOperations.FSTrashRoot(path);
+ JSONObject json = fsExecute(user, command);
+ AUDIT_LOG.info("[{}]", path);
+ response = Response.ok(json).type(MediaType.APPLICATION_JSON).build();
+ break;
+ }
default: {
throw new IOException(
MessageFormat.format("Invalid HTTP GET operation [{0}]", op.value()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
index 3e8daa0..750b7f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
@@ -40,6 +40,8 @@ HttpFS HTTP web-service API calls are HTTP REST calls that map to a HDFS file sy
* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=list` returns the contents of the HDFS `/user/foo` directory in JSON format.
+* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=GETTRASHROOT` returns the path `/user/foo/.Trash`, if `/` is an encrypted zone, returns the path `/.Trash/foo`. See [more details](../hadoop-project-dist/hadoop-hdfs/TransparentEncryption.html#Rename_and_Trash_considerations) about trash path in an encrypted zone.
+
* `$ curl -X POST http://httpfs-host:14000/webhdfs/v1/user/foo/bar?op=mkdirs` creates the HDFS `/user/foo.bar` directory.
User and Developer Documentation
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index 4e25e56..e130c68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -430,6 +430,35 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
new Path("/tmp").toUri().getPath());
}
+ private void testTrashRoot() throws Exception {
+ if (!isLocalFS()) {
+ FileSystem fs = FileSystem.get(getProxiedFSConf());
+
+ final Path rootDir = new Path("/");
+ final Path fooPath = new Path(getProxiedFSTestDir(), "foo.txt");
+ OutputStream os = fs.create(fooPath);
+ os.write(1);
+ os.close();
+
+ Path trashPath = fs.getTrashRoot(rootDir);
+ Path fooTrashPath = fs.getTrashRoot(fooPath);
+ fs.close();
+
+ fs = getHttpFSFileSystem();
+ Path httpFSTrashPath = fs.getTrashRoot(rootDir);
+ Path httpFSFooTrashPath = fs.getTrashRoot(fooPath);
+ fs.close();
+
+ assertEquals(trashPath.toUri().getPath(),
+ httpFSTrashPath.toUri().getPath());
+ assertEquals(fooTrashPath.toUri().getPath(),
+ httpFSFooTrashPath.toUri().getPath());
+ // trash path is related to USER, not path
+ assertEquals(trashPath.toUri().getPath(),
+ fooTrashPath.toUri().getPath());
+ }
+ }
+
private void testMkdirs() throws Exception {
Path path = new Path(getProxiedFSTestDir(), "foo");
FileSystem fs = getHttpFSFileSystem();
@@ -916,86 +945,90 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
GET, OPEN, CREATE, APPEND, TRUNCATE, CONCAT, RENAME, DELETE, LIST_STATUS,
WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER,
SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, FILEACLS, DIRACLS, SET_XATTR,
- GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION, LIST_STATUS_BATCH
+ GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION, LIST_STATUS_BATCH,
+ GETTRASHROOT
}
private void operation(Operation op) throws Exception {
switch (op) {
- case GET:
- testGet();
- break;
- case OPEN:
- testOpen();
- break;
- case CREATE:
- testCreate();
- break;
- case APPEND:
- testAppend();
- break;
- case TRUNCATE:
- testTruncate();
- break;
- case CONCAT:
- testConcat();
- break;
- case RENAME:
- testRename();
- break;
- case DELETE:
- testDelete();
- break;
- case LIST_STATUS:
- testListStatus();
- break;
- case WORKING_DIRECTORY:
- testWorkingdirectory();
- break;
- case MKDIRS:
- testMkdirs();
- break;
- case SET_TIMES:
- testSetTimes();
- break;
- case SET_PERMISSION:
- testSetPermission();
- break;
- case SET_OWNER:
- testSetOwner();
- break;
- case SET_REPLICATION:
- testSetReplication();
- break;
- case CHECKSUM:
- testChecksum();
- break;
- case CONTENT_SUMMARY:
- testContentSummary();
- break;
- case FILEACLS:
- testFileAcls();
- break;
- case DIRACLS:
- testDirAcls();
- break;
- case SET_XATTR:
- testSetXAttr();
- break;
- case REMOVE_XATTR:
- testRemoveXAttr();
- break;
- case GET_XATTRS:
- testGetXAttrs();
- break;
- case LIST_XATTRS:
- testListXAttrs();
- break;
- case ENCRYPTION:
- testEncryption();
- break;
- case LIST_STATUS_BATCH:
- testListStatusBatch();
- break;
+ case GET:
+ testGet();
+ break;
+ case OPEN:
+ testOpen();
+ break;
+ case CREATE:
+ testCreate();
+ break;
+ case APPEND:
+ testAppend();
+ break;
+ case TRUNCATE:
+ testTruncate();
+ break;
+ case CONCAT:
+ testConcat();
+ break;
+ case RENAME:
+ testRename();
+ break;
+ case DELETE:
+ testDelete();
+ break;
+ case LIST_STATUS:
+ testListStatus();
+ break;
+ case WORKING_DIRECTORY:
+ testWorkingdirectory();
+ break;
+ case MKDIRS:
+ testMkdirs();
+ break;
+ case SET_TIMES:
+ testSetTimes();
+ break;
+ case SET_PERMISSION:
+ testSetPermission();
+ break;
+ case SET_OWNER:
+ testSetOwner();
+ break;
+ case SET_REPLICATION:
+ testSetReplication();
+ break;
+ case CHECKSUM:
+ testChecksum();
+ break;
+ case CONTENT_SUMMARY:
+ testContentSummary();
+ break;
+ case FILEACLS:
+ testFileAcls();
+ break;
+ case DIRACLS:
+ testDirAcls();
+ break;
+ case SET_XATTR:
+ testSetXAttr();
+ break;
+ case REMOVE_XATTR:
+ testRemoveXAttr();
+ break;
+ case GET_XATTRS:
+ testGetXAttrs();
+ break;
+ case LIST_XATTRS:
+ testListXAttrs();
+ break;
+ case ENCRYPTION:
+ testEncryption();
+ break;
+ case LIST_STATUS_BATCH:
+ testListStatusBatch();
+ break;
+ case GETTRASHROOT:
+ testTrashRoot();
+ break;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
index a9c36b0..fcbddc4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
@@ -43,7 +43,6 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttrCodec;
@@ -137,9 +136,9 @@ public class TestHttpFSServer extends HFSTestCase {
//HDFS configuration
File hadoopConfDir = new File(new File(homeDir, "conf"), "hadoop-conf");
hadoopConfDir.mkdirs();
- String fsDefaultName = TestHdfsHelper.getHdfsConf().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
- Configuration conf = new Configuration(false);
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName);
+ Configuration hdfsConf = TestHdfsHelper.getHdfsConf();
+ // Http Server's conf should be based on HDFS's conf
+ Configuration conf = new Configuration(hdfsConf);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
File hdfsSite = new File(hadoopConfDir, "hdfs-site.xml");
@@ -347,6 +346,20 @@ public class TestHttpFSServer extends HFSTestCase {
}
/**
+ * Given the JSON output from the GETTRASHPATH call, return the
+ * 'path' value.
+ *
+ * @param statusJson JSON from GETTRASHPATH
+ * @return The value of 'path' in statusJson
+ * @throws Exception
+ */
+ private String getPath(String statusJson) throws Exception {
+ JSONParser parser = new JSONParser();
+ JSONObject details = (JSONObject) parser.parse(statusJson);
+ return (String) details.get("Path");
+ }
+
+ /**
* Given the JSON output from the GETACLSTATUS call, return the
* 'entries' value as a List<String>.
* @param statusJson JSON from GETACLSTATUS
@@ -675,6 +688,40 @@ public class TestHttpFSServer extends HFSTestCase {
@TestDir
@TestJetty
@TestHdfs
+ public void testGetTrashRoot() throws Exception {
+ String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
+ createHttpFSServer(false);
+ String trashJson = getStatus("/", "GETTRASHROOT");
+ String trashPath = getPath(trashJson);
+
+ Path expectedPath = new Path(FileSystem.USER_HOME_PREFIX,
+ new Path(user, FileSystem.TRASH_PREFIX));
+ Assert.assertEquals(expectedPath.toUri().getPath(), trashPath);
+
+ byte[] array = new byte[]{0, 1, 2, 3};
+ FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
+ fs.mkdirs(new Path("/tmp"));
+ OutputStream os = fs.create(new Path("/tmp/foo"));
+ os.write(array);
+ os.close();
+
+ trashJson = getStatus("/tmp/foo", "GETTRASHROOT");
+ trashPath = getPath(trashJson);
+ Assert.assertEquals(expectedPath.toUri().getPath(), trashPath);
+
+ //TestHdfsHelp has already set up EZ environment
+ final Path ezFile = TestHdfsHelper.ENCRYPTED_FILE;
+ final Path ezPath = TestHdfsHelper.ENCRYPTION_ZONE;
+ trashJson = getStatus(ezFile.toUri().getPath(), "GETTRASHROOT");
+ trashPath = getPath(trashJson);
+ expectedPath = new Path(ezPath, new Path(FileSystem.TRASH_PREFIX, user));
+ Assert.assertEquals(expectedPath.toUri().getPath(), trashPath);
+ }
+
+ @Test
+ @TestDir
+ @TestJetty
+ @TestHdfs
public void testDelegationTokenOperations() throws Exception {
createHttpFSServer(true);
@@ -754,6 +801,21 @@ public class TestHttpFSServer extends HFSTestCase {
conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
conn.getResponseCode());
+
+ // getTrash test with delegation
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=GETTRASHROOT&delegation=" + tokenStr);
+ conn = (HttpURLConnection) url.openConnection();
+ Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
+ conn.getResponseCode());
+
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=GETTRASHROOT");
+ conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestProperty("Cookie",
+ AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
+ Assert.assertEquals(HttpURLConnection.HTTP_OK,
+ conn.getResponseCode());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index cea3339..15195e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -1057,6 +1057,11 @@ public class NamenodeWebHdfsMethods {
np.checkAccess(fullpath, FsAction.getFsAction(fsAction.getValue()));
return Response.ok().build();
}
+ case GETTRASHROOT: {
+ final String trashPath = getTrashRoot(fullpath, conf);
+ final String jsonStr = JsonUtil.toJsonString("Path", trashPath);
+ return Response.ok(jsonStr).type(MediaType.APPLICATION_JSON).build();
+ }
case LISTSTATUS_BATCH:
{
byte[] start = HdfsFileStatus.EMPTY_NAME;
@@ -1072,6 +1077,13 @@ public class NamenodeWebHdfsMethods {
}
}
+ private static String getTrashRoot(String fullPath,
+ Configuration conf) throws IOException {
+ FileSystem fs = FileSystem.get(conf != null ? conf : new Configuration());
+ return fs.getTrashRoot(
+ new org.apache.hadoop.fs.Path(fullPath)).toUri().getPath();
+ }
+
private static DirectoryListing getDirectoryListing(final NamenodeProtocols np,
final String p, byte[] startAfter) throws IOException {
final DirectoryListing listing = np.getListing(p, startAfter, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index f904bda..1d7d704 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -42,6 +42,7 @@ WebHDFS REST API
* [Get Content Summary of a Directory](#Get_Content_Summary_of_a_Directory)
* [Get File Checksum](#Get_File_Checksum)
* [Get Home Directory](#Get_Home_Directory)
+ * [Get Trash Root](#Get_Trash_Root)
* [Set Permission](#Set_Permission)
* [Set Owner](#Set_Owner)
* [Set Replication Factor](#Set_Replication_Factor)
@@ -149,6 +150,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
* [`GETFILECHECKSUM`](#Get_File_Checksum) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileChecksum)
* [`GETHOMEDIRECTORY`](#Get_Home_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getHomeDirectory)
* [`GETDELEGATIONTOKEN`](#Get_Delegation_Token) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getDelegationToken)
+ * [`GETTRASHROOT`](#Get_Trash_Root) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getTrashRoot)
* [`GETXATTRS`](#Get_an_XAttr) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getXAttr)
* [`GETXATTRS`](#Get_multiple_XAttrs) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getXAttrs)
* [`GETXATTRS`](#Get_all_XAttrs) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getXAttrs)
@@ -578,7 +580,7 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileSt
"group" : "supergroup",
"length" : 0,
"modificationTime": 1320895981256,
- "owner" : "szetszwo",
+ "owner" : "username",
"pathSuffix" : "bar",
"permission" : "711",
"replication" : 0,
@@ -815,10 +817,36 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileCh
Content-Type: application/json
Transfer-Encoding: chunked
- {"Path": "/user/szetszwo"}
+ {"Path": "/user/username"}
See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getHomeDirectory
+### Get Trash Root
+
+* Submit a HTTP GET request.
+
+ curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETTRASHROOT"
+
+ The client receives a response with a [`Path` JSON object](#Path_JSON_Schema):
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {"Path": "/user/username/.Trash"}
+
+ if the path is an encrypted zone path and user has permission of the path, the client receives a response like this:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {"Path": "/PATH/.Trash/username"}
+
+See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getTrashRoot
+
+For more details about trash root in an encrypted zone, please refer to [Transparent Encryption Guide](./TransparentEncryption.html#Rename_and_Trash_considerations).
+
### Set Permission
* Submit a HTTP PUT request.
@@ -1137,7 +1165,7 @@ Snapshot Operations
Content-Type: application/json
Transfer-Encoding: chunked
- {"Path": "/user/szetszwo/.snapshot/s1"}
+ {"Path": "/user/username/.snapshot/s1"}
See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).createSnapshot
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 8cce7ef..3a0586e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.tools.CryptoAdmin;
import org.apache.hadoop.hdfs.tools.DFSck;
import org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.security.AccessControlException;
@@ -1558,6 +1559,25 @@ public class TestEncryptionZones {
DFSTestUtil.createFile(fs, nestedEZFile, len, (short) 1, 0xFEED);
verifyShellDeleteWithTrash(shell, topEZFile);
verifyShellDeleteWithTrash(shell, nestedEZFile);
+
+ //Test nested EZ with webHDFS
+ final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(
+ conf, WebHdfsConstants.WEBHDFS_SCHEME);
+ final String currentUser =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ final Path expectedTopTrash = new Path(topEZ,
+ new Path(FileSystem.TRASH_PREFIX, currentUser));
+ final Path expectedNestedTrash = new Path(nestedEZ,
+ new Path(FileSystem.TRASH_PREFIX, currentUser));
+
+ final Path topTrash = webFS.getTrashRoot(topEZFile);
+ final Path nestedTrash = webFS.getTrashRoot(nestedEZFile);
+
+ assertEquals(expectedTopTrash.toUri().getPath(),
+ topTrash.toUri().getPath());
+ assertEquals(expectedNestedTrash.toUri().getPath(),
+ nestedTrash.toUri().getPath());
+
verifyShellDeleteWithTrash(shell, nestedEZ);
verifyShellDeleteWithTrash(shell, topEZ);
}
@@ -1566,6 +1586,8 @@ public class TestEncryptionZones {
public void testRootDirEZTrash() throws Exception {
final HdfsAdmin dfsAdmin =
new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ final String currentUser =
+ UserGroupInformation.getCurrentUser().getShortUserName();
final Path rootDir = new Path("/");
dfsAdmin.createEncryptionZone(rootDir, TEST_KEY, NO_TRASH);
final Path encFile = new Path("/encFile");
@@ -1578,10 +1600,23 @@ public class TestEncryptionZones {
// Trash path should be consistent
// if root path is an encryption zone
- Path encFileTrash = shell.getCurrentTrashDir(encFile);
- Path rootDirTrash = shell.getCurrentTrashDir(rootDir);
+ Path encFileCurrentTrash = shell.getCurrentTrashDir(encFile);
+ Path rootDirCurrentTrash = shell.getCurrentTrashDir(rootDir);
assertEquals("Root trash should be equal with ezFile trash",
- encFileTrash, rootDirTrash);
+ encFileCurrentTrash, rootDirCurrentTrash);
+
+ // Use webHDFS client to test trash root path
+ final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(
+ conf, WebHdfsConstants.WEBHDFS_SCHEME);
+ final Path expectedTrash = new Path(rootDir,
+ new Path(FileSystem.TRASH_PREFIX, currentUser));
+
+ Path webHDFSTrash = webFS.getTrashRoot(encFile);
+ assertEquals(expectedTrash.toUri().getPath(),
+ webHDFSTrash.toUri().getPath());
+ assertEquals(encFileCurrentTrash.getParent().toUri().getPath(),
+ webHDFSTrash.toUri().getPath());
+
}
@Test(timeout = 120000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 604a063..23d543d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -1072,4 +1072,26 @@ public class TestWebHDFS {
}
}
}
+
+ @Test
+ public void testGetTrashRoot() throws Exception {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ final String currentUser =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(
+ conf, WebHdfsConstants.WEBHDFS_SCHEME);
+
+ Path trashPath = webFS.getTrashRoot(new Path("/"));
+ Path expectedPath = new Path(FileSystem.USER_HOME_PREFIX,
+ new Path(currentUser, FileSystem.TRASH_PREFIX));
+ assertEquals(expectedPath.toUri().getPath(), trashPath.toUri().getPath());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: HADOOP-13573. S3Guard: create
basic contract tests for MetadataStore implementations. Contributed by Aaron
Fabbri.
Posted by fa...@apache.org.
HADOOP-13573. S3Guard: create basic contract tests for MetadataStore implementations. Contributed by Aaron Fabbri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a734dcf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a734dcf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a734dcf
Branch: refs/heads/HADOOP-13345
Commit: 2a734dcf4809640cf97768b0639e971d14f10dc7
Parents: a795393
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Sep 20 16:30:10 2016 -0700
Committer: Aaron Fabbri <fa...@apache.org>
Committed: Fri Nov 4 20:30:59 2016 -0700
----------------------------------------------------------------------
.../fs/s3a/s3guard/AbstractMSContract.java | 32 ++
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 424 +++++++++++++++++++
2 files changed, 456 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a734dcf/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java
new file mode 100644
index 0000000..7cb540a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import org.apache.hadoop.conf.Configured;
+
+import java.io.IOException;
+
+/**
+ * Test specification for MetadataStore contract tests. Supplies configuration
+ * and MetadataStore instance.
+ */
+public abstract class AbstractMSContract extends Configured {
+
+ public abstract MetadataStore getMetadataStore() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a734dcf/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
new file mode 100644
index 0000000..ac27696
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -0,0 +1,424 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Main test class for MetadataStore implementations.
+ * Implementations should each create a test by subclassing this and
+ * overriding {@link #createContract()}.
+ */
+public abstract class MetadataStoreTestBase extends Assert {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MetadataStoreTestBase.class);
+
+ /** Some dummy values for sanity-checking FileStatus contents. */
+ protected static final long BLOCK_SIZE = 32 * 1024 * 1024;
+ protected static final int REPLICATION = 1;
+ private static final FsPermission PERMISSION = new FsPermission((short)0755);
+ private static final String OWNER = "bob";
+ private static final String GROUP = "uncles";
+ private final long accessTime = System.currentTimeMillis();
+ private final long modTime = accessTime - 5000;
+
+ /**
+ * Each test should override this.
+ * @return Contract which specifies the MetadataStore under test plus config.
+ */
+ public abstract AbstractMSContract createContract();
+
+ /** The MetadataStore contract used to test against. */
+ private AbstractMSContract contract;
+
+ private MetadataStore ms;
+
+ @Before
+ public void setUp() throws Exception {
+ LOG.debug("== Setup. ==");
+ contract = createContract();
+ ms = contract.getMetadataStore();
+ assertNotNull("null MetadataStore", ms);
+ ms.initialize(contract.getConf());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ LOG.debug("== Tear down. ==");
+ if (ms != null) {
+ ms.close();
+ ms = null;
+ }
+ }
+
+ @Test
+ public void testPutNew() throws Exception {
+ /* create three dirs /da1, /da2, /da3 */
+ createNewDirs("/da1", "/da2", "/da3");
+
+ /* It is caller's responsibility to set up ancestor entries beyond the
+ * containing directory. We only track direct children of the directory.
+ * Thus this will not affect entry for /da1.
+ */
+ ms.put(new PathMetadata(makeFileStatus("/da1/db1/fc1", 100)));
+
+ assertEmptyDirs("/da1", "/da2", "/da3");
+ assertDirectorySize("/da1/db1", 1);
+
+ /* Check contents of dir status. */
+ PathMetadata dirMeta = ms.get(new Path("/da1"));
+ verifyDirStatus(dirMeta);
+
+ /* This already exists, and should silently replace it. */
+ ms.put(new PathMetadata(makeDirStatus("/da1/db1")));
+
+ /* If we had putNew(), and used it above, this would be empty again. */
+ assertDirectorySize("/da1", 1);
+
+ assertEmptyDirs("/da2", "/da3");
+
+ /* Ensure new files update correct parent dirs. */
+ ms.put(new PathMetadata(makeFileStatus("/da1/db1/fc1", 100)));
+ ms.put(new PathMetadata(makeFileStatus("/da1/db1/fc2", 200)));
+ assertDirectorySize("/da1", 1);
+ assertDirectorySize("/da1/db1", 2);
+ assertEmptyDirs("/da2", "/da3");
+ PathMetadata meta = ms.get(new Path("/da1/db1/fc2"));
+ assertNotNull("Get file after put new.", meta);
+ assertEquals("Cached file size correct.", 200,
+ meta.getFileStatus().getLen());
+ }
+
+ @Test
+ public void testPutOverwrite() throws Exception {
+ final String filePath = "/a1/b1/c1/some_file";
+ final String dirPath = "/a1/b1/c1/d1";
+ ms.put(new PathMetadata(makeFileStatus(filePath, 100)));
+ ms.put(new PathMetadata(makeDirStatus(dirPath)));
+ PathMetadata meta = ms.get(new Path(filePath));
+ verifyBasicFileStatus(meta);
+
+ ms.put(new PathMetadata(basicFileStatus(new Path(filePath), 9999, false)));
+ meta = ms.get(new Path(filePath));
+ assertEquals("Updated size", 9999, meta.getFileStatus().getLen());
+ }
+
+ @Test
+ public void testRootDirPutNew() throws Exception {
+ Path rootPath = new Path("/");
+
+ ms.put(new PathMetadata(makeFileStatus("/file1", 100)));
+ DirListingMetadata dir = ms.listChildren(rootPath);
+ assertNotNull("Root dir cached", dir);
+ assertFalse("Root not fully cached", dir.isAuthoritative());
+ assertNotNull("have root dir file listing", dir.getListing());
+ assertEquals("One file in root dir", 1, dir.getListing().size());
+ assertEquals("file1 in root dir", "/file1",
+ dir.getListing().iterator().next().getFileStatus().getPath()
+ .toString());
+ }
+
+ @Test
+ public void testDelete() throws Exception {
+ setUpDeleteTest();
+
+ ms.delete(new Path("/ADirectory1/db1/file2"));
+
+ /* Ensure delete happened. */
+ assertDirectorySize("/ADirectory1/db1", 1);
+ PathMetadata meta = ms.get(new Path("/ADirectory1/db1/file2"));
+ assertNull("File deleted", meta);
+ }
+
+ @Test
+ public void testDeleteSubtree() throws Exception {
+ setUpDeleteTest();
+ createNewDirs("/ADirectory1/db1/dc1", "/ADirectory1/db1/dc1/dd1");
+ ms.put(new PathMetadata(
+ makeFileStatus("/ADirectory1/db1/dc1/dd1/deepFile", 100)));
+ ms.deleteSubtree(new Path("/ADirectory1/db1"));
+
+ assertEmptyDirectory("/ADirectory1");
+ assertNotCached("/ADirectory1/file1");
+ assertNotCached("/ADirectory1/file2");
+ assertNotCached("/ADirectory1/db1/dc1/dd1/deepFile");
+ assertEmptyDirectory("/ADirectory2");
+ }
+
+
+ /*
+ * Some implementations might not support this. It was useful to test
+ * correctness of the LocalMetadataStore implementation, but feel free to
+ * override this to be a no-op.
+ */
+ @Test
+ public void testDeleteRecursiveRoot() throws Exception {
+ setUpDeleteTest();
+
+ ms.deleteSubtree(new Path("/"));
+ assertNotCached("/ADirectory1");
+ assertNotCached("/ADirectory2");
+ assertNotCached("/ADirectory2/db1");
+ assertNotCached("/ADirectory2/db1/file1");
+ assertNotCached("/ADirectory2/db1/file2");
+ }
+
+ @Test
+ public void testDeleteNonExisting() throws Exception {
+ // Path doesn't exist, but should silently succeed
+ ms.delete(new Path("/bobs/your/uncle"));
+
+ // Ditto.
+ ms.deleteSubtree(new Path("/internets"));
+ }
+
+
+ private void setUpDeleteTest() throws IOException {
+ createNewDirs("/ADirectory1", "/ADirectory2", "/ADirectory1/db1");
+ ms.put(new PathMetadata(makeFileStatus("/ADirectory1/db1/file1", 100)));
+ ms.put(new PathMetadata(makeFileStatus("/ADirectory1/db1/file2", 100)));
+
+ PathMetadata meta = ms.get(new Path("/ADirectory1/db1/file2"));
+ assertNotNull("Found test file", meta);
+
+ assertDirectorySize("/ADirectory1/db1", 2);
+ }
+
+ @Test
+ public void testGet() throws Exception {
+ final String filePath = "/a1/b1/c1/some_file";
+ final String dirPath = "/a1/b1/c1/d1";
+ ms.put(new PathMetadata(makeFileStatus(filePath, 100)));
+ ms.put(new PathMetadata(makeDirStatus(dirPath)));
+ PathMetadata meta = ms.get(new Path(filePath));
+ assertNotNull("Get found file", meta);
+ verifyBasicFileStatus(meta);
+
+ meta = ms.get(new Path(dirPath));
+ assertNotNull("Get found file (dir)", meta);
+ assertTrue("Found dir", meta.getFileStatus().isDirectory());
+
+ meta = ms.get(new Path("bollocks"));
+ assertNull("Don't get non-existent file", meta);
+ }
+
+
+ @Test
+ public void testListChildren() throws Exception {
+ setupListStatus();
+
+ DirListingMetadata dirMeta;
+ try {
+ dirMeta = ms.listChildren(new Path("/"));
+ if (dirMeta != null) {
+ /* Cache has no way of knowing it has all entries for root unless we
+ * specifically tell it via put() with
+ * DirListingMetadata.isAuthoritative = true */
+ assertFalse("Root dir is not cached, or partially cached",
+ dirMeta.isAuthoritative());
+ assertListingsEqual(dirMeta.getListing(), "/a1", "/a2");
+ }
+ } catch (FileNotFoundException f) { }
+
+ dirMeta = ms.listChildren(new Path("/a1"));
+ assertListingsEqual(dirMeta.getListing(), "/a1/b1", "/a1/b2");
+
+ // TODO
+ // 1. Add properties query to MetadataStore interface
+ // supportsAuthoritativeDirectories() or something.
+ // 2. Add "isNew" flag to MetadataStore.put(DirListingMetadata)
+ // 3. If #1 is true, assert that directory is still fully cached here.
+ // assertTrue("Created dir is fully cached", dirMeta.isAuthoritative());
+
+ dirMeta = ms.listChildren(new Path("/a1/b1"));
+ assertListingsEqual(dirMeta.getListing(), "/a1/b1/file1", "/a1/b1/file2",
+ "/a1/b1/c1");
+ }
+
+ @Test
+ public void testDirListingRoot() throws Exception {
+ commonTestPutListStatus("/");
+ }
+
+ @Test
+ public void testPutDirListing() throws Exception {
+ commonTestPutListStatus("/a");
+ }
+
+ @Test
+ public void testInvalidListChildren() throws Exception {
+ setupListStatus();
+ assertNull("missing path returns null",
+ ms.listChildren(new Path("/a1/b1x")));
+ }
+
+ @Ignore
+ public void testMove() throws Exception {
+ // TODO
+ throw new RuntimeException("TODO: implement move and tests.");
+ }
+
+
+ /*
+ * Helper functions.
+ */
+
+ /** Modifies paths input array and returns it. */
+ private String[] buildPaths(String parent, String ...paths) {
+ for (int i = 0; i < paths.length; i++) {
+ Path p = new Path(parent, paths[i]);
+ paths[i] = p.toString();
+ }
+ return paths;
+ }
+
+ private void commonTestPutListStatus(final String parent) throws IOException {
+ putListStatusFiles(parent, true, buildPaths(parent, "file1", "file2",
+ "file3"));
+ DirListingMetadata dirMeta = ms.listChildren(new Path(parent));
+ assertNotNull("list after putListStatus", dirMeta);
+ Collection<PathMetadata> entries = dirMeta.getListing();
+ assertNotNull("listStatus has entries", entries);
+ assertListingsEqual(entries, buildPaths(parent, "file1", "file2", "file3"));
+ }
+
+ private void setupListStatus() throws IOException {
+ createNewDirs("/a1", "/a2", "/a1/b1", "/a1/b2", "/a1/b1/c1",
+ "/a1/b1/c1/d1");
+ ms.put(new PathMetadata(makeFileStatus("/a1/b1/file1", 100)));
+ ms.put(new PathMetadata(makeFileStatus("/a1/b1/file2", 100)));
+ }
+
+ private void assertListingsEqual(Collection<PathMetadata> listing,
+ String ...pathStrs) {
+ Set<Path> a = new HashSet<>();
+ for (PathMetadata meta : listing) {
+ a.add(meta.getFileStatus().getPath());
+ }
+
+ Set<Path> b = new HashSet<>();
+ for (String ps : pathStrs) {
+ b.add(new Path(ps));
+ }
+ assertTrue("Same set of files", a.equals(b));
+ }
+
+ private void putListStatusFiles(String dirPath, boolean authoritative,
+ String... filenames) throws IOException {
+ ArrayList<PathMetadata> metas = new ArrayList<>(filenames .length);
+ for (int i = 0; i < filenames.length; i++) {
+ metas.add(new PathMetadata(makeFileStatus(filenames[i], 100)));
+ }
+ DirListingMetadata dirMeta =
+ new DirListingMetadata(new Path(dirPath), metas, authoritative);
+ ms.put(dirMeta);
+ }
+
+ private void createNewDirs(String... dirs)
+ throws IOException {
+ for (String pathStr : dirs) {
+ ms.put(new PathMetadata(makeDirStatus(pathStr)));
+ }
+ }
+
+ private void assertDirectorySize(String pathStr, int size)
+ throws IOException {
+ DirListingMetadata dirMeta = ms.listChildren(new Path(pathStr));
+ assertNotNull("Directory " + pathStr + " in cache", dirMeta);
+ assertEquals("Number of entries in dir " + pathStr, size,
+ nonDeleted(dirMeta.getListing()).size());
+ }
+
+ /** @return only file statuses which are *not* marked deleted. */
+ private Collection<PathMetadata> nonDeleted(
+ Collection<PathMetadata> statuses) {
+ /* TODO: filter out paths marked for deletion. */
+ return statuses;
+ }
+
+ private void assertNotCached(String pathStr) throws IOException {
+ // TODO this should return an entry with deleted flag set
+ PathMetadata meta = ms.get(new Path(pathStr));
+ assertNull(pathStr + " not cached.", meta);
+ }
+
+ private void assertEmptyDirectory(String pathStr) throws IOException {
+ assertDirectorySize(pathStr, 0);
+ }
+
+ private void assertEmptyDirs(String ...dirs) throws IOException {
+ for (String pathStr : dirs) {
+ assertEmptyDirectory(pathStr);
+ }
+ }
+
+ private FileStatus basicFileStatus(Path path, int size, boolean isDir) {
+ return new FileStatus(size, isDir, REPLICATION, BLOCK_SIZE, modTime,
+ accessTime, PERMISSION, OWNER, GROUP, path);
+ }
+
+ private FileStatus makeFileStatus(String pathStr, int size) {
+ return basicFileStatus(new Path(pathStr), size, false);
+ }
+
+ private void verifyBasicFileStatus(PathMetadata meta) {
+ FileStatus status = meta.getFileStatus();
+ assertFalse("Not a dir", status.isDirectory());
+ assertEquals("Replication value", REPLICATION, status.getReplication());
+ assertEquals("Access time", accessTime, status.getAccessTime());
+ assertEquals("Mod time", modTime, status.getModificationTime());
+ assertEquals("Block size", BLOCK_SIZE, status.getBlockSize());
+ assertEquals("Owner", OWNER, status.getOwner());
+ assertEquals("Group", GROUP, status.getGroup());
+ assertEquals("Permission", PERMISSION, status.getPermission());
+ }
+
+ private FileStatus makeDirStatus(String pathStr) {
+ return basicFileStatus(new Path(pathStr), 0, true);
+ }
+
+ private void verifyDirStatus(PathMetadata meta) {
+ FileStatus status = meta.getFileStatus();
+ assertTrue("Is a dir", status.isDirectory());
+ assertEquals("zero length", 0, status.getLen());
+ assertEquals("Replication value", REPLICATION, status.getReplication());
+ assertEquals("Access time", accessTime, status.getAccessTime());
+ assertEquals("Mod time", modTime, status.getModificationTime());
+ assertEquals("Owner", OWNER, status.getOwner());
+ assertEquals("Group", GROUP, status.getGroup());
+ assertEquals("Permission", PERMISSION, status.getPermission());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HADOOP-12453. Support decoding KMS
Delegation Token with its own Identifier. Contributed by Xiaoyu Yao.
Posted by fa...@apache.org.
HADOOP-12453. Support decoding KMS Delegation Token with its own Identifier. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7154a20b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7154a20b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7154a20b
Branch: refs/heads/HADOOP-13345
Commit: 7154a20bcb1559c23aeb3b78b920bed03d834cb5
Parents: 20c4d8e
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Nov 3 13:09:03 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Nov 3 13:09:03 2016 -0700
----------------------------------------------------------------------
.../crypto/key/kms/KMSClientProvider.java | 4 +-
.../crypto/key/kms/KMSDelegationToken.java | 52 ++++++++++++++++++++
...apache.hadoop.security.token.TokenIdentifier | 14 ++++++
.../key/kms/server/KMSAuthenticationFilter.java | 4 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 17 ++++---
5 files changed, 79 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154a20b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index db0ee85..2b6ae9e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -98,8 +98,8 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
private static final String ANONYMOUS_REQUESTS_DISALLOWED = "Anonymous requests are disallowed";
- public static final String TOKEN_KIND_STR = "kms-dt";
- public static final Text TOKEN_KIND = new Text(TOKEN_KIND_STR);
+ public static final String TOKEN_KIND_STR = KMSDelegationToken.TOKEN_KIND_STR;
+ public static final Text TOKEN_KIND = KMSDelegationToken.TOKEN_KIND;
public static final String SCHEME_NAME = "kms";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154a20b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
new file mode 100644
index 0000000..adeebf2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
+
+/**
+ * Holder class for KMS delegation tokens.
+ */
+@InterfaceAudience.Private
+public final class KMSDelegationToken {
+
+ public static final String TOKEN_KIND_STR = "kms-dt";
+ public static final Text TOKEN_KIND = new Text(TOKEN_KIND_STR);
+
+ // Utility class is not supposed to be instantiated.
+ private KMSDelegationToken() {
+ }
+
+ /**
+ * DelegationTokenIdentifier used for the KMS.
+ */
+ public static class KMSDelegationTokenIdentifier
+ extends DelegationTokenIdentifier {
+
+ public KMSDelegationTokenIdentifier() {
+ super(TOKEN_KIND);
+ }
+
+ @Override
+ public Text getKind() {
+ return TOKEN_KIND;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154a20b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
new file mode 100644
index 0000000..b65f151
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -0,0 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.hadoop.crypto.key.kms.KMSDelegationToken$KMSDelegationTokenIdentifier
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154a20b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
index 8efef73..45e48e9 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.crypto.key.kms.server;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.KMSDelegationToken;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter;
@@ -72,7 +72,7 @@ public class KMSAuthenticationFilter
KerberosDelegationTokenAuthenticationHandler.class.getName());
}
props.setProperty(DelegationTokenAuthenticationHandler.TOKEN_KIND,
- KMSClientProvider.TOKEN_KIND_STR);
+ KMSDelegationToken.TOKEN_KIND_STR);
return props;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154a20b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index de600f8..384d11a 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.KMSDelegationToken;
import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
@@ -1804,13 +1805,13 @@ public class TestKMS {
InetSocketAddress kmsAddr =
new InetSocketAddress(getKMSUrl().getHost(),
getKMSUrl().getPort());
- Assert.assertEquals(KMSClientProvider.TOKEN_KIND,
+ Assert.assertEquals(KMSDelegationToken.TOKEN_KIND,
credentials.getToken(SecurityUtil.buildTokenService(kmsAddr)).
getKind());
// Test non-renewer user cannot renew.
for (Token<?> token : tokens) {
- if (!(token.getKind().equals(KMSClientProvider.TOKEN_KIND))) {
+ if (!(token.getKind().equals(KMSDelegationToken.TOKEN_KIND))) {
LOG.info("Skipping token {}", token);
continue;
}
@@ -1843,7 +1844,7 @@ public class TestKMS {
boolean renewed = false;
for (Token<?> token : tokens) {
if (!(token.getKind()
- .equals(KMSClientProvider.TOKEN_KIND))) {
+ .equals(KMSDelegationToken.TOKEN_KIND))) {
LOG.info("Skipping token {}", token);
continue;
}
@@ -1863,7 +1864,7 @@ public class TestKMS {
// test delegation token cancellation
for (Token<?> token : tokens) {
if (!(token.getKind()
- .equals(KMSClientProvider.TOKEN_KIND))) {
+ .equals(KMSDelegationToken.TOKEN_KIND))) {
LOG.info("Skipping token {}", token);
continue;
}
@@ -1927,7 +1928,7 @@ public class TestKMS {
final Credentials credentials = new Credentials();
kpdte.addDelegationTokens("client", credentials);
Assert.assertEquals(1, credentials.getAllTokens().size());
- Assert.assertEquals(KMSClientProvider.TOKEN_KIND, credentials.
+ Assert.assertEquals(KMSDelegationToken.TOKEN_KIND, credentials.
getToken(SecurityUtil.buildTokenService(kmsAddr)).getKind());
UserGroupInformation.getCurrentUser().addCredentials(credentials);
LOG.info("Added kms dt to credentials: {}", UserGroupInformation.
@@ -1971,14 +1972,14 @@ public class TestKMS {
final Credentials newCreds = new Credentials();
kpdte.addDelegationTokens("client", newCreds);
Assert.assertEquals(1, newCreds.getAllTokens().size());
- Assert.assertEquals(KMSClientProvider.TOKEN_KIND,
+ Assert.assertEquals(KMSDelegationToken.TOKEN_KIND,
newCreds.getToken(SecurityUtil.buildTokenService(kmsAddr)).
getKind());
// Using job 1's DT should fail.
final Credentials oldCreds = new Credentials();
for (Token<?> token : job1Token) {
- if (token.getKind().equals(KMSClientProvider.TOKEN_KIND)) {
+ if (token.getKind().equals(KMSDelegationToken.TOKEN_KIND)) {
oldCreds
.addToken(SecurityUtil.buildTokenService(kmsAddr), token);
}
@@ -1995,7 +1996,7 @@ public class TestKMS {
// Using the new DT should succeed.
Assert.assertEquals(1, newCreds.getAllTokens().size());
- Assert.assertEquals(KMSClientProvider.TOKEN_KIND,
+ Assert.assertEquals(KMSDelegationToken.TOKEN_KIND,
newCreds.getToken(SecurityUtil.buildTokenService(kmsAddr)).
getKind());
UserGroupInformation.getCurrentUser().addCredentials(newCreds);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-5720. Update document for
"rmadmin -replaceLabelOnNode". Contributred by Tao Jie
Posted by fa...@apache.org.
YARN-5720. Update document for "rmadmin -replaceLabelOnNode". Contributred by Tao Jie
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e754960
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e754960
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e754960
Branch: refs/heads/HADOOP-13345
Commit: 0e75496049001b00c00901fcbfe3d4229b2fd64a
Parents: 7e521c5
Author: Naganarasimha <na...@apache.org>
Authored: Thu Nov 3 12:32:22 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Thu Nov 3 12:32:22 2016 +0530
----------------------------------------------------------------------
.../hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md | 2 +-
.../hadoop-yarn-site/src/site/markdown/YarnCommands.md | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e754960/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
index af75bfe..a87658d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
@@ -89,7 +89,7 @@ Notes:
###Add/modify node-to-labels mapping to YARN
* Configuring nodes to labels mapping in **Centralized** NodeLabel setup
- * Executing ```yarn rmadmin -replaceLabelsOnNode \u201cnode1[:port]=label1 node2=label2\u201d```. Added label1 to node1, label2 to node2. If user don\u2019t specify port, it added the label to all ```NodeManagers``` running on the node.
+ * Executing ```yarn rmadmin -replaceLabelsOnNode \u201cnode1[:port]=label1 node2=label2\u201d [-failOnUnknownNodes]```. Added label1 to node1, label2 to node2. If user don\u2019t specify port, it adds the label to all ```NodeManagers``` running on the node. If option ```-failOnUnknownNodes``` is set, this command will fail if specified nodes are unknown.
* Configuring nodes to labels mapping in **Distributed** NodeLabel setup
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e754960/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 8f954ff..1d51b1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -215,7 +215,7 @@ Usage:
-getGroups [username]
-addToClusterNodeLabels <"label1(exclusive=true),label2(exclusive=false),label3">
-removeFromClusterNodeLabels <label1,label2,label3> (label splitted by ",")
- -replaceLabelsOnNode <"node1[:port]=label1,label2 node2[:port]=label1,label2">
+ -replaceLabelsOnNode <"node1[:port]=label1,label2 node2[:port]=label1,label2"> [-failOnUnknownNodes]
-directlyAccessNodeLabelStore
-refreshClusterMaxPriority
-updateNodeResource [NodeID] [MemSize] [vCores] ([OvercommitTimeout])
@@ -239,7 +239,7 @@ Usage:
| -getGroups [username] | Get groups the specified user belongs to. |
| -addToClusterNodeLabels <"label1(exclusive=true),label2(exclusive=false),label3"> | Add to cluster node labels. Default exclusivity is true. |
| -removeFromClusterNodeLabels <label1,label2,label3> (label splitted by ",") | Remove from cluster node labels. |
-| -replaceLabelsOnNode <"node1[:port]=label1,label2 node2[:port]=label1,label2"> | Replace labels on nodes (please note that we do not support specifying multiple labels on a single host for now.) |
+| -replaceLabelsOnNode <"node1[:port]=label1,label2 node2[:port]=label1,label2"> [-failOnUnknownNodes]| Replace labels on nodes (please note that we do not support specifying multiple labels on a single host for now.) -failOnUnknownNodes is optional, when we set this option, it will fail if specified nodes are unknown.|
| -directlyAccessNodeLabelStore | This is DEPRECATED, will be removed in future releases. Directly access node label store, with this option, all node label related operations will not connect RM. Instead, they will access/modify stored node labels directly. By default, it is false (access via RM). AND PLEASE NOTE: if you configured yarn.node-labels.fs-store.root-dir to a local directory (instead of NFS or HDFS), this option will only work when the command run on the machine where RM is running. |
| -refreshClusterMaxPriority | Refresh cluster max priority |
| -updateNodeResource [NodeID] [MemSize] [vCores] \([OvercommitTimeout]\) | Update resource on specific node. |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-5822. Log ContainerRuntime
initialization error in LinuxContainerExecutor. Contributed by Sidharta
Seethana.
Posted by fa...@apache.org.
YARN-5822. Log ContainerRuntime initialization error in LinuxContainerExecutor. Contributed by Sidharta Seethana.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ee0e317
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ee0e317
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ee0e317
Branch: refs/heads/HADOOP-13345
Commit: 9ee0e3172ed20b951b458cbbdc4799675f2a2a51
Parents: 352cbaa
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Nov 3 22:33:43 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu Nov 3 22:33:43 2016 +0530
----------------------------------------------------------------------
.../hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ee0e317/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index cc12b20..323c443 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -270,6 +270,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
this.linuxContainerRuntime = runtime;
}
} catch (ContainerExecutionException e) {
+ LOG.error("Failed to initialize linux container runtime(s)!", e);
throw new IOException("Failed to initialize linux container runtime(s)!");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDFS-10633. DiskBalancer : Add the
description for the new setting dfs.disk.balancer.plan.threshold.percent in
HDFSDiskbalancer.md. Contributed by Yiqun Lin.
Posted by fa...@apache.org.
HDFS-10633. DiskBalancer : Add the description for the new setting dfs.disk.balancer.plan.threshold.percent in HDFSDiskbalancer.md. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34173a42
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34173a42
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34173a42
Branch: refs/heads/HADOOP-13345
Commit: 34173a42d450350982a99e673a36f15215c564bf
Parents: 808f5d8
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Nov 1 11:14:03 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Nov 1 11:14:03 2016 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 6 ++++--
.../hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md | 1 +
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34173a42/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index e28dc54..02c36cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4294,8 +4294,10 @@
<name>dfs.disk.balancer.plan.threshold.percent</name>
<value>10</value>
<description>
- The percentage that disk tolerance that we are ok with in
- a plan.
+ The percentage threshold value for volume Data Density in a plan.
+ If the absolute value of volume Data Density which is out of
+ threshold value in a node, it means that the volumes corresponding to
+ the disks should do the balancing in the plan. The default value is 10.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34173a42/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
index dfb43b6..b7e55cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
@@ -125,6 +125,7 @@ There is a set of diskbalancer settings that can be controlled via hdfs-site.xml
|`dfs.disk.balancer.max.disk.throughputInMBperSec` | This controls the maximum disk bandwidth consumed by diskbalancer while copying data. If a value like 10MB is specified then diskbalancer on the average will only copy 10MB/S. The default value is 10MB/S.|
|`dfs.disk.balancer.max.disk.errors`| sets the value of maximum number of errors we can ignore for a specific move between two disks before it is abandoned. For example, if a plan has 3 pair of disks to copy between , and the first disk set encounters more than 5 errors, then we abandon the first copy and start the second copy in the plan. The default value of max errors is set to 5.|
|`dfs.disk.balancer.block.tolerance.percent`| The tolerance percent specifies when we have reached a good enough value for any copy step. For example, if you specify 10% then getting close to 10% of the target value is good enough.|
+|`dfs.disk.balancer.plan.threshold.percent`| The percentage threshold value for volume Data Density in a plan. If the absolute value of volume Data Density which is out of threshold value in a node, it means that the volumes corresponding to the disks should do the balancing in the plan. The default value is 10.|
Debugging
---------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDFS-11095. BlockManagerSafeMode
should respect extension period default config value (30s). Contributed by
Mingliang Liu
Posted by fa...@apache.org.
HDFS-11095. BlockManagerSafeMode should respect extension period default config value (30s). Contributed by Mingliang Liu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b0c17f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b0c17f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b0c17f8
Branch: refs/heads/HADOOP-13345
Commit: 9b0c17f85232950ecd927427e5ad9d28dd9d277f
Parents: f2800f5
Author: Mingliang Liu <li...@apache.org>
Authored: Wed Nov 2 13:01:54 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed Nov 2 14:58:24 2016 -0700
----------------------------------------------------------------------
.../blockmanagement/BlockManagerSafeMode.java | 11 +++++++++--
.../src/main/resources/hdfs-default.xml | 5 +++--
.../TestBlockManagerSafeMode.java | 20 ++++++++++++++++++++
3 files changed, 32 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0c17f8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
index ea70c86..a0017a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
@@ -35,12 +35,16 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.util.Daemon;
+import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.atomic.AtomicLong;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY;
@@ -96,7 +100,8 @@ class BlockManagerSafeMode {
private long blockReplQueueThreshold;
/** How long (in ms) is the extension period. */
- private final int extension;
+ @VisibleForTesting
+ final long extension;
/** Timestamp of the first time when thresholds are met. */
private final AtomicLong reachedTime = new AtomicLong();
/** Timestamp of the safe mode initialized. */
@@ -143,7 +148,9 @@ class BlockManagerSafeMode {
this.replQueueThreshold =
conf.getFloat(DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY,
(float) threshold);
- this.extension = conf.getInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0);
+ this.extension = conf.getTimeDuration(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY,
+ DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT,
+ MILLISECONDS);
this.inRollBack = isInRollBackMode(NameNode.getStartupOption(conf));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0c17f8/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 02c36cf..b111193 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -847,8 +847,9 @@
<name>dfs.namenode.safemode.extension</name>
<value>30000</value>
<description>
- Determines extension of safe mode in milliseconds
- after the threshold level is reached.
+ Determines extension of safe mode in milliseconds after the threshold level
+ is reached. Support multiple time unit suffix (case insensitive), as
+ described in dfs.heartbeat.interval.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0c17f8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
index dc18a22..6f97401 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
@@ -38,6 +38,7 @@ import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.concurrent.TimeoutException;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -392,6 +393,25 @@ public class TestBlockManagerSafeMode {
"are cleared.", bmSafeMode.leaveSafeMode(false));
}
+ @Test(timeout = 10000)
+ public void testExtensionConfig() {
+ final Configuration conf = new HdfsConfiguration();
+ bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf);
+ assertEquals(DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT, bmSafeMode.extension);
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, "30000");
+ bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf);
+ assertEquals(30000, bmSafeMode.extension);
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, "20s");
+ bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf);
+ assertEquals(20 * 1000, bmSafeMode.extension);
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, "7m");
+ bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf);
+ assertEquals(7 * 60 * 1000, bmSafeMode.extension);
+ }
+
/**
* Test get safe mode tip.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HADOOP-13784. Output javadoc
inside the target directory.
Posted by fa...@apache.org.
HADOOP-13784. Output javadoc inside the target directory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcc07ad8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcc07ad8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcc07ad8
Branch: refs/heads/HADOOP-13345
Commit: dcc07ad8c8abf9feea569283b4a4a096d893168a
Parents: 730cb0c
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Nov 2 18:34:33 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Nov 2 18:34:33 2016 -0700
----------------------------------------------------------------------
hadoop-project-dist/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcc07ad8/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 4423d94..7908d1a 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -106,7 +106,7 @@
<source>${maven.compile.source}</source>
<charset>${maven.compile.encoding}</charset>
<reportOutputDirectory>${project.build.directory}/site</reportOutputDirectory>
- <destDir>api</destDir>
+ <destDir>${project.build.directory}/api</destDir>
<groups>
<group>
<title>${project.name} API</title>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-5001. Aggregated Logs root
directory is created with wrong group if nonexistent. Contributed by Haibo
Chen
Posted by fa...@apache.org.
YARN-5001. Aggregated Logs root directory is created with wrong group if nonexistent. Contributed by Haibo Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76893a41
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76893a41
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76893a41
Branch: refs/heads/HADOOP-13345
Commit: 76893a41003d57d94eb1a5f486010815266046af
Parents: 34173a4
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Nov 1 20:23:24 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Nov 1 20:23:24 2016 +0000
----------------------------------------------------------------------
.../logaggregation/LogAggregationService.java | 15 ++++++++++++
.../TestLogAggregationService.java | 24 +++++++++++++++++++-
2 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76893a41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
index a4ae643..38315fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
@@ -258,6 +258,21 @@ public class LogAggregationService extends AbstractService implements
remoteFS.getWorkingDirectory());
remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS));
remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+
+ UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
+ String primaryGroupName = null;
+ try {
+ primaryGroupName = loginUser.getPrimaryGroupName();
+ } catch (IOException e) {
+ LOG.warn("No primary group found. The remote root log directory" +
+ " will be created with the HDFS superuser being its group " +
+ "owner. JobHistoryServer may be unable to read the directory.");
+ }
+ // set owner on the remote directory only if the primary group exists
+ if (primaryGroupName != null) {
+ remoteFS.setOwner(qualified,
+ loginUser.getShortUserName(), primaryGroupName);
+ }
} catch (IOException e) {
throw new YarnRuntimeException("Failed to create remoteLogDir ["
+ this.remoteRootLogDir + "]", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76893a41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index e7330aa..4e9829a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -608,7 +608,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
throws Exception {
this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
File aNewFile = new File(String.valueOf("tmp"+System.currentTimeMillis()));
- this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
aNewFile.getAbsolutePath());
LogAggregationService logAggregationService = spy(
@@ -626,6 +626,28 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
}
@Test
+ public void testRemoteRootLogDirIsCreatedWithCorrectGroupOwner()
+ throws IOException {
+ this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+ Path aNewFile = new Path(String.valueOf("tmp"+System.currentTimeMillis()));
+ this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, aNewFile.getName());
+
+ LogAggregationService logAggregationService = new LogAggregationService(
+ dispatcher, this.context, this.delSrvc, super.dirsHandler);
+ logAggregationService.init(this.conf);
+ logAggregationService.verifyAndCreateRemoteLogDir(this.conf);
+
+ String targetGroup =
+ UserGroupInformation.getLoginUser().getPrimaryGroupName();
+ FileSystem fs = FileSystem.get(this.conf);
+ FileStatus fileStatus = fs.getFileStatus(aNewFile);
+ Assert.assertEquals("The new aggregate file is not successfully created",
+ fileStatus.getGroup(), targetGroup);
+
+ fs.delete(aNewFile, true);
+ }
+
+ @Test
public void testAppLogDirCreation() throws Exception {
final String logSuffix = "logs";
this.conf.set(YarnConfiguration.NM_LOG_DIRS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-5815. Random failure of
TestApplicationPriority.testOrderOfActivatingThePriorityApplicationOnRMRestart
(Bibin A Chundatt via Varun Saxena)
Posted by fa...@apache.org.
YARN-5815. Random failure of TestApplicationPriority.testOrderOfActivatingThePriorityApplicationOnRMRestart (Bibin A Chundatt via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37791901
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37791901
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37791901
Branch: refs/heads/HADOOP-13345
Commit: 377919010b687dbf95f62082201cf91f5a7a2318
Parents: 0dc2a6a
Author: Varun Saxena <va...@apache.org>
Authored: Thu Nov 3 00:37:09 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Thu Nov 3 00:37:09 2016 +0530
----------------------------------------------------------------------
.../capacity/TestApplicationPriority.java | 23 ++++++++++----------
1 file changed, 11 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37791901/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index 8bd5314..523d49d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -684,36 +684,35 @@ public class TestApplicationPriority {
(LeafQueue) ((CapacityScheduler) scheduler).getQueue("default");
// wait for all applications to get added to scheduler
- int count = 5;
+ int count = 50;
while (count-- > 0) {
- if ((defaultQueue.getNumActiveApplications() + defaultQueue
- .getNumPendingApplications()) == 3) {
+ if (defaultQueue.getNumPendingApplications() == 3) {
break;
}
- Thread.sleep(500);
+ Thread.sleep(50);
}
// Before NM registration, AMResourceLimit threshold is 0. So no
// applications get activated.
Assert.assertEquals(0, defaultQueue.getNumActiveApplications());
+ Assert.assertEquals(3, defaultQueue.getNumPendingApplications());
// NM resync to new RM
nm1.registerNode();
dispatcher1.await();
- Assert.assertEquals(2, defaultQueue.getNumActiveApplications());
- Assert.assertEquals(1, defaultQueue.getNumPendingApplications());
-
-
- // wait for activating one applications
- count = 5;
+ // wait for activating applications
+ count = 50;
while (count-- > 0) {
- if (defaultQueue.getOrderingPolicy().getSchedulableEntities().size() == 2) {
+ if (defaultQueue.getNumActiveApplications() == 2) {
break;
}
- Thread.sleep(500);
+ Thread.sleep(50);
}
+ Assert.assertEquals(2, defaultQueue.getNumActiveApplications());
+ Assert.assertEquals(1, defaultQueue.getNumPendingApplications());
+
// verify for order of activated applications iterator
iterator =
defaultQueue.getOrderingPolicy().getSchedulableEntities().iterator();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-5697. Use CliParser to parse
options in RMAdminCLI. Contributed by Tao jie
Posted by fa...@apache.org.
YARN-5697. Use CliParser to parse options in RMAdminCLI. Contributed by Tao jie
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aacf214a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aacf214a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aacf214a
Branch: refs/heads/HADOOP-13345
Commit: aacf214a274fb5149437f287c542339966c03ea5
Parents: fa1512a
Author: Naganarasimha <na...@apache.org>
Authored: Wed Nov 2 04:44:33 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Wed Nov 2 04:44:33 2016 +0530
----------------------------------------------------------------------
.../hadoop/yarn/client/cli/RMAdminCLI.java | 191 +++++++++++--------
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 19 +-
2 files changed, 126 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aacf214a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 640f8e3..82a910b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -29,6 +29,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -80,7 +86,6 @@ public class RMAdminCLI extends HAAdmin {
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
- private boolean directlyAccessNodeLabelStore = false;
static CommonNodeLabelsManager localNodeLabelsManager = null;
private static final String NO_LABEL_ERR_MSG =
"No cluster node-labels are specified";
@@ -130,8 +135,9 @@ public class RMAdminCLI extends HAAdmin {
new UsageInfo("<label1,label2,label3> (label splitted by \",\")",
"remove from cluster node labels"))
.put("-replaceLabelsOnNode",
- new UsageInfo("[-failOnUnknownNodes] " +
- "<\"node1[:port]=label1,label2 node2[:port]=label1,label2\">",
+ new UsageInfo(
+ "<\"node1[:port]=label1,label2 node2[:port]=label1,label2\"> "
+ + "[-failOnUnknownNodes] ",
"replace labels on nodes"
+ " (please note that we do not support specifying multiple"
+ " labels on a single host for now.)\n\t\t"
@@ -248,8 +254,9 @@ public class RMAdminCLI extends HAAdmin {
" [-addToClusterNodeLabels <\"label1(exclusive=true),"
+ "label2(exclusive=false),label3\">]" +
" [-removeFromClusterNodeLabels <label1,label2,label3>]" +
- " [-replaceLabelsOnNode [-failOnUnknownNodes] "
- + "<\"node1[:port]=label1,label2 node2[:port]=label1\">]" +
+ " [-replaceLabelsOnNode " +
+ "<\"node1[:port]=label1,label2 node2[:port]=label1\"> " +
+ "[-failOnUnknownNodes]]" +
" [-directlyAccessNodeLabelStore]" +
" [-refreshClusterMaxPriority]" +
" [-updateNodeResource [NodeID] [MemSize] [vCores]" +
@@ -576,11 +583,26 @@ public class RMAdminCLI extends HAAdmin {
return labels;
}
- private int addToClusterNodeLabels(String args) throws IOException,
- YarnException {
- List<NodeLabel> labels = buildNodeLabelsFromStr(args);
+ private int handleAddToClusterNodeLabels(String[] args, String cmd,
+ boolean isHAEnabled) throws IOException, YarnException, ParseException {
+ Options opts = new Options();
+ opts.addOption("addToClusterNodeLabels", true,
+ "Add to cluster node labels.");
+ opts.addOption("directlyAccessNodeLabelStore", false,
+ "Directly access node label store.");
+ int exitCode = -1;
+ CommandLine cliParser = null;
+ try {
+ cliParser = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ System.err.println(NO_LABEL_ERR_MSG);
+ printUsage(args[0], isHAEnabled);
+ return exitCode;
+ }
- if (directlyAccessNodeLabelStore) {
+ List<NodeLabel> labels = buildNodeLabelsFromStr(
+ cliParser.getOptionValue("addToClusterNodeLabels"));
+ if (cliParser.hasOption("directlyAccessNodeLabelStore")) {
getNodeLabelManagerInstance(getConf()).addToCluserNodeLabels(labels);
} else {
ResourceManagerAdministrationProtocol adminProtocol =
@@ -592,11 +614,26 @@ public class RMAdminCLI extends HAAdmin {
return 0;
}
- private int removeFromClusterNodeLabels(String args) throws IOException,
- YarnException {
- Set<String> labels = buildNodeLabelNamesFromStr(args);
+ private int handleRemoveFromClusterNodeLabels(String[] args, String cmd,
+ boolean isHAEnabled) throws IOException, YarnException, ParseException {
+ Options opts = new Options();
+ opts.addOption("removeFromClusterNodeLabels", true,
+ "Remove From cluster node labels.");
+ opts.addOption("directlyAccessNodeLabelStore", false,
+ "Directly access node label store.");
+ int exitCode = -1;
+ CommandLine cliParser = null;
+ try {
+ cliParser = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ System.err.println(NO_LABEL_ERR_MSG);
+ printUsage(args[0], isHAEnabled);
+ return exitCode;
+ }
- if (directlyAccessNodeLabelStore) {
+ Set<String> labels = buildNodeLabelNamesFromStr(
+ cliParser.getOptionValue("removeFromClusterNodeLabels"));
+ if (cliParser.hasOption("directlyAccessNodeLabelStore")) {
getNodeLabelManagerInstance(getConf()).removeFromClusterNodeLabels(
labels);
} else {
@@ -659,14 +696,35 @@ public class RMAdminCLI extends HAAdmin {
return map;
}
- private int replaceLabelsOnNodes(String args, boolean failOnUnknownNodes)
- throws IOException, YarnException {
- Map<NodeId, Set<String>> map = buildNodeLabelsMapFromStr(args);
- return replaceLabelsOnNodes(map, failOnUnknownNodes);
+ private int handleReplaceLabelsOnNodes(String[] args, String cmd,
+ boolean isHAEnabled) throws IOException, YarnException, ParseException {
+ Options opts = new Options();
+ opts.addOption("replaceLabelsOnNode", true,
+ "Replace label on node.");
+ opts.addOption("failOnUnknownNodes", false,
+ "Fail on unknown nodes.");
+ opts.addOption("directlyAccessNodeLabelStore", false,
+ "Directly access node label store.");
+ int exitCode = -1;
+ CommandLine cliParser = null;
+ try {
+ cliParser = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ System.err.println(NO_MAPPING_ERR_MSG);
+ printUsage(args[0], isHAEnabled);
+ return exitCode;
+ }
+
+ Map<NodeId, Set<String>> map = buildNodeLabelsMapFromStr(
+ cliParser.getOptionValue("replaceLabelsOnNode"));
+ return replaceLabelsOnNodes(map,
+ cliParser.hasOption("failOnUnknownNodes"),
+ cliParser.hasOption("directlyAccessNodeLabelStore"));
}
private int replaceLabelsOnNodes(Map<NodeId, Set<String>> map,
- boolean failOnUnknownNodes) throws IOException, YarnException {
+ boolean failOnUnknownNodes, boolean directlyAccessNodeLabelStore)
+ throws IOException, YarnException {
if (directlyAccessNodeLabelStore) {
getNodeLabelManagerInstance(getConf()).replaceLabelsOnNode(map);
} else {
@@ -682,18 +740,6 @@ public class RMAdminCLI extends HAAdmin {
@Override
public int run(String[] args) throws Exception {
- // -directlyAccessNodeLabelStore is a additional option for node label
- // access, so just search if we have specified this option, and remove it
- List<String> argsList = new ArrayList<String>();
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("-directlyAccessNodeLabelStore")) {
- directlyAccessNodeLabelStore = true;
- } else {
- argsList.add(args[i]);
- }
- }
- args = argsList.toArray(new String[0]);
-
YarnConfiguration yarnConf =
getConf() == null ? new YarnConfiguration() : new YarnConfiguration(
getConf());
@@ -766,37 +812,11 @@ public class RMAdminCLI extends HAAdmin {
} else if ("-updateNodeResource".equals(cmd)) {
exitCode = handleUpdateNodeResource(args, cmd, isHAEnabled);
} else if ("-addToClusterNodeLabels".equals(cmd)) {
- if (i >= args.length) {
- System.err.println(NO_LABEL_ERR_MSG);
- printUsage("", isHAEnabled);
- exitCode = -1;
- } else {
- exitCode = addToClusterNodeLabels(args[i]);
- }
+ exitCode = handleAddToClusterNodeLabels(args, cmd, isHAEnabled);
} else if ("-removeFromClusterNodeLabels".equals(cmd)) {
- if (i >= args.length) {
- System.err.println(NO_LABEL_ERR_MSG);
- printUsage("", isHAEnabled);
- exitCode = -1;
- } else {
- exitCode = removeFromClusterNodeLabels(args[i]);
- }
+ exitCode = handleRemoveFromClusterNodeLabels(args, cmd, isHAEnabled);
} else if ("-replaceLabelsOnNode".equals(cmd)) {
- if (i >= args.length) {
- System.err.println(NO_MAPPING_ERR_MSG);
- printUsage("", isHAEnabled);
- exitCode = -1;
- } else if ("-failOnUnknownNodes".equals(args[i])) {
- if (i + 1 >= args.length) {
- System.err.println(NO_MAPPING_ERR_MSG);
- printUsage("", isHAEnabled);
- exitCode = -1;
- } else {
- exitCode = replaceLabelsOnNodes(args[i + 1], true);
- }
- } else {
- exitCode = replaceLabelsOnNodes(args[i], false);
- }
+ exitCode = handleReplaceLabelsOnNodes(args, cmd, isHAEnabled);
} else {
exitCode = -1;
System.err.println(cmd.substring(1) + ": Unknown command");
@@ -834,28 +854,47 @@ public class RMAdminCLI extends HAAdmin {
// A helper method to reduce the number of lines of run()
private int handleRefreshNodes(String[] args, String cmd, boolean isHAEnabled)
- throws IOException, YarnException {
- if (args.length == 1) {
- return refreshNodes();
- } else if (args.length == 3 || args.length == 4) {
- // if the graceful timeout specified
- if ("-g".equals(args[1]) || "-graceful".equals(args[1])) {
- int timeout = -1;
- String trackingMode;
- if (args.length == 4) {
- timeout = validateTimeout(args[2]);
- trackingMode = validateTrackingMode(args[3]);
- } else {
- trackingMode = validateTrackingMode(args[2]);
- }
- return refreshNodes(timeout, trackingMode);
+ throws IOException, YarnException, ParseException {
+ Options opts = new Options();
+ opts.addOption("refreshNodes", false,
+ "Refresh the hosts information at the ResourceManager.");
+ Option gracefulOpt = new Option("g", "graceful", true,
+ "Wait for timeout before marking the NodeManager as decommissioned.");
+ gracefulOpt.setOptionalArg(true);
+ opts.addOption(gracefulOpt);
+ opts.addOption("client", false,
+ "Indicates the timeout tracking should be handled by the client.");
+ opts.addOption("server", false,
+ "Indicates the timeout tracking should be handled by the RM.");
+
+ int exitCode = -1;
+ CommandLine cliParser = null;
+ try {
+ cliParser = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ System.out.println("Missing argument for options");
+ printUsage(args[0], isHAEnabled);
+ return exitCode;
+ }
+
+ int timeout = -1;
+ if (cliParser.hasOption("g")) {
+ String strTimeout = cliParser.getOptionValue("g");
+ if (strTimeout != null) {
+ timeout = validateTimeout(strTimeout);
+ }
+ String trackingMode = null;
+ if (cliParser.hasOption("client")) {
+ trackingMode = "client";
+ } else if (cliParser.hasOption("server")) {
+ trackingMode = "server";
} else {
printUsage(cmd, isHAEnabled);
return -1;
}
+ return refreshNodes(timeout, trackingMode);
} else {
- printUsage(cmd, isHAEnabled);
- return -1;
+ return refreshNodes();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aacf214a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
index 9e20a43..dd331e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
@@ -469,8 +469,9 @@ public class TestRMAdminCLI {
"[username]] [-addToClusterNodeLabels " +
"<\"label1(exclusive=true),label2(exclusive=false),label3\">] " +
"[-removeFromClusterNodeLabels <label1,label2,label3>] " +
- "[-replaceLabelsOnNode [-failOnUnknownNodes] " +
- "<\"node1[:port]=label1,label2 node2[:port]=label1\">] " +
+ "[-replaceLabelsOnNode " +
+ "<\"node1[:port]=label1,label2 node2[:port]=label1\"> " +
+ "[-failOnUnknownNodes]] " +
"[-directlyAccessNodeLabelStore] [-refreshClusterMaxPriority] " +
"[-updateNodeResource [NodeID] [MemSize] [vCores] " +
"([OvercommitTimeout]) [-help [cmd]]"));
@@ -564,8 +565,8 @@ public class TestRMAdminCLI {
+ " [username]] [-addToClusterNodeLabels <\"label1(exclusive=true),"
+ "label2(exclusive=false),label3\">]"
+ " [-removeFromClusterNodeLabels <label1,label2,label3>] [-replaceLabelsOnNode "
- + "[-failOnUnknownNodes] "
- + "<\"node1[:port]=label1,label2 node2[:port]=label1\">] [-directlyAccessNodeLabelStore] "
+ + "<\"node1[:port]=label1,label2 node2[:port]=label1\"> "
+ + "[-failOnUnknownNodes]] [-directlyAccessNodeLabelStore] "
+ "[-refreshClusterMaxPriority] "
+ "[-updateNodeResource [NodeID] [MemSize] [vCores] "
+ "([OvercommitTimeout]) "
@@ -614,13 +615,11 @@ public class TestRMAdminCLI {
dummyNodeLabelsManager.removeFromClusterNodeLabels(ImmutableSet.of("x", "y"));
// change the sequence of "-directlyAccessNodeLabelStore" and labels,
- // should not matter
+ // should fail
args =
new String[] { "-addToClusterNodeLabels",
"-directlyAccessNodeLabelStore", "x,y" };
- assertEquals(0, rmAdminCLI.run(args));
- assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().containsAll(
- ImmutableSet.of("x", "y")));
+ assertEquals(-1, rmAdminCLI.run(args));
// local node labels manager will be close after running
assertTrue(dummyNodeLabelsManager.getServiceState() == STATE.STOPPED);
@@ -770,6 +769,10 @@ public class TestRMAdminCLI {
assertTrue(0 != rmAdminCLI.run(args));
// no labels, should fail
+ args = new String[] { "-replaceLabelsOnNode", "-failOnUnknownNodes" };
+ assertTrue(0 != rmAdminCLI.run(args));
+
+ // no labels, should fail
args =
new String[] { "-replaceLabelsOnNode", "-directlyAccessNodeLabelStore" };
assertTrue(0 != rmAdminCLI.run(args));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: Revert "HADOOP-13565.
KerberosAuthenticationHandler#authenticate should not rebuild SPN based on
client request. Contributed by Xiaoyu Yao."
Posted by fa...@apache.org.
Revert "HADOOP-13565. KerberosAuthenticationHandler#authenticate should not rebuild SPN based on client request. Contributed by Xiaoyu Yao."
This reverts commit 9097e2efe4c92d83c8fab88dc11be84505a6cab5.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95665a6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95665a6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95665a6e
Branch: refs/heads/HADOOP-13345
Commit: 95665a6eea32ff7134ea556db4dd4ae068364fc0
Parents: 6bb741f
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Nov 4 16:02:47 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Fri Nov 4 16:02:47 2016 -0700
----------------------------------------------------------------------
.../authentication/server/KerberosAuthenticationHandler.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95665a6e/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
index 07c2a31..c6d1881 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
@@ -343,6 +343,8 @@ public class KerberosAuthenticationHandler implements AuthenticationHandler {
authorization = authorization.substring(KerberosAuthenticator.NEGOTIATE.length()).trim();
final Base64 base64 = new Base64(0);
final byte[] clientToken = base64.decode(authorization);
+ final String serverName = InetAddress.getByName(request.getServerName())
+ .getCanonicalHostName();
try {
token = Subject.doAs(serverSubject, new PrivilegedExceptionAction<AuthenticationToken>() {
@@ -352,7 +354,10 @@ public class KerberosAuthenticationHandler implements AuthenticationHandler {
GSSContext gssContext = null;
GSSCredential gssCreds = null;
try {
- gssCreds = gssManager.createCredential(null,
+ gssCreds = gssManager.createCredential(
+ gssManager.createName(
+ KerberosUtil.getServicePrincipal("HTTP", serverName),
+ KerberosUtil.getOidInstance("NT_GSS_KRB5_PRINCIPAL")),
GSSCredential.INDEFINITE_LIFETIME,
new Oid[]{
KerberosUtil.getOidInstance("GSS_SPNEGO_MECH_OID"),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HADOOP-13792. Stackoverflow for
schemeless defaultFS with trailing slash. Contributed by John Zhuge
Posted by fa...@apache.org.
HADOOP-13792. Stackoverflow for schemeless defaultFS with trailing slash. Contributed by John Zhuge
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abfc15d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abfc15d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abfc15d5
Branch: refs/heads/HADOOP-13345
Commit: abfc15d5ef966e99e8fe05d155ad2557e8cd67e8
Parents: 0aafc12
Author: Mingliang Liu <li...@apache.org>
Authored: Fri Nov 4 10:46:08 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Fri Nov 4 10:46:27 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 6 +-
.../org/apache/hadoop/fs/TestDefaultUri.java | 115 +++++++++++++++++++
2 files changed, 120 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/abfc15d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 39b5b95..375af90 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -188,7 +188,11 @@ public abstract class FileSystem extends Configured implements Closeable {
* @return the uri of the default filesystem
*/
public static URI getDefaultUri(Configuration conf) {
- return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS)));
+ URI uri = URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS)));
+ if (uri.getScheme() == null) {
+ throw new IllegalArgumentException("No scheme in default FS: " + uri);
+ }
+ return uri;
}
/** Set the default filesystem URI in a configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/abfc15d5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
new file mode 100644
index 0000000..f232735
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Test;
+
+/**
+ * Test default URI related APIs in {@link FileSystem}.
+ */
+public class TestDefaultUri {
+ private Configuration conf = new Configuration();
+
+ @Test
+ public void tetGetDefaultUri() {
+ conf.set(FS_DEFAULT_NAME_KEY, "hdfs://nn_host");
+ URI uri = FileSystem.getDefaultUri(conf);
+ assertThat(uri.getScheme(), is("hdfs"));
+ assertThat(uri.getAuthority(), is("nn_host"));
+ }
+
+ @Test
+ public void tetGetDefaultUriWithPort() {
+ conf.set(FS_DEFAULT_NAME_KEY, "hdfs://nn_host:5432");
+ URI uri = FileSystem.getDefaultUri(conf);
+ assertThat(uri.getScheme(), is("hdfs"));
+ assertThat(uri.getAuthority(), is("nn_host:5432"));
+ }
+
+ @Test
+ public void tetGetDefaultUriTrailingSlash() {
+ conf.set(FS_DEFAULT_NAME_KEY, "hdfs://nn_host/");
+ URI uri = FileSystem.getDefaultUri(conf);
+ assertThat(uri.getScheme(), is("hdfs"));
+ assertThat(uri.getAuthority(), is("nn_host"));
+ }
+
+ @Test
+ public void tetGetDefaultUriNoScheme() {
+ conf.set(FS_DEFAULT_NAME_KEY, "nn_host");
+ URI uri = FileSystem.getDefaultUri(conf);
+ assertThat(uri.getScheme(), is("hdfs"));
+ assertThat(uri.getAuthority(), is("nn_host"));
+ }
+
+ @Test
+ public void tetGetDefaultUriNoSchemeTrailingSlash() {
+ conf.set(FS_DEFAULT_NAME_KEY, "nn_host/");
+ try {
+ FileSystem.getDefaultUri(conf);
+ fail("Expect IAE: No scheme in default FS");
+ } catch (IllegalArgumentException e) {
+ GenericTestUtils.assertExceptionContains(
+ "No scheme in default FS", e);
+ }
+ }
+
+ @Test
+ public void tetFsGet() throws IOException {
+ conf.set(FS_DEFAULT_NAME_KEY, "file:///");
+ FileSystem fs = FileSystem.get(conf);
+ assertThat(fs, instanceOf(LocalFileSystem.class));
+ }
+
+ @Test
+ public void tetFsGetNoScheme() throws IOException {
+ // Bare host name or address indicates hdfs scheme
+ conf.set(FS_DEFAULT_NAME_KEY, "nn_host");
+ try {
+ FileSystem.get(conf);
+ fail("Expect IOE: No FileSystem for scheme: hdfs");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains(
+ "No FileSystem for scheme: hdfs", e);
+ }
+ }
+
+ @Test
+ public void tetFsGetNoSchemeTrailingSlash() throws IOException {
+ // Bare host name or address with trailing slash is invalid
+ conf.set(FS_DEFAULT_NAME_KEY, "nn_host/");
+ try {
+ FileSystem.get(conf);
+ fail("Expect IAE: No scheme in default FS");
+ } catch (IllegalArgumentException e) {
+ GenericTestUtils.assertExceptionContains(
+ "No scheme in default FS", e);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-2995. Enhance UI to show
cluster resource utilization of various container Execution types.
(Konstantinos Karanasos via asuresh)
Posted by fa...@apache.org.
YARN-2995. Enhance UI to show cluster resource utilization of various container Execution types. (Konstantinos Karanasos via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0aafc122
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0aafc122
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0aafc122
Branch: refs/heads/HADOOP-13345
Commit: 0aafc122d401a2f49d0f9ca556343f9ba871e979
Parents: 19b3779
Author: Arun Suresh <as...@apache.org>
Authored: Fri Nov 4 07:31:54 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Nov 4 07:31:54 2016 -0700
----------------------------------------------------------------------
.../v2/app/rm/RMContainerAllocator.java | 6 +
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 4 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 4 +-
.../yarn/server/api/records/NodeStatus.java | 7 +-
.../records/OpportunisticContainersStatus.java | 152 +++++++++++++++++++
.../api/records/QueuedContainersStatus.java | 45 ------
.../api/records/impl/pb/NodeStatusPBImpl.java | 38 ++---
.../pb/OpportunisticContainersStatusPBImpl.java | 139 +++++++++++++++++
.../impl/pb/QueuedContainersStatusPBImpl.java | 84 ----------
.../main/proto/yarn_server_common_protos.proto | 12 +-
.../protocolrecords/TestProtocolRecords.java | 17 ++-
.../nodemanager/NodeStatusUpdaterImpl.java | 17 ++-
.../containermanager/ContainerManager.java | 3 +
.../containermanager/ContainerManagerImpl.java | 8 +-
.../monitor/ContainersMonitorImpl.java | 6 +-
.../queuing/QueuingContainerManagerImpl.java | 32 ++++
.../nodemanager/webapp/ContainerPage.java | 1 +
.../nodemanager/webapp/dao/ContainerInfo.java | 7 +
.../webapp/TestNMWebServicesContainers.java | 2 +-
...pportunisticContainerAllocatorAMService.java | 5 +-
.../server/resourcemanager/rmnode/RMNode.java | 4 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 17 ++-
.../rmnode/RMNodeStatusEvent.java | 6 +-
.../scheduler/AbstractYarnScheduler.java | 3 +-
.../distributed/NodeQueueLoadMonitor.java | 10 +-
.../resourcemanager/webapp/NodesPage.java | 75 ++++++---
.../resourcemanager/webapp/dao/NodeInfo.java | 41 ++++-
.../yarn/server/resourcemanager/MockNodes.java | 4 +-
.../distributed/TestNodeQueueLoadMonitor.java | 8 +-
.../resourcemanager/webapp/TestNodesPage.java | 33 ++++
.../resourcemanager/webapp/TestRMWebApp.java | 1 +
.../webapp/TestRMWebServicesNodes.java | 41 ++++-
32 files changed, 599 insertions(+), 233 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index db8f337..200fa1f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -240,6 +240,8 @@ public class RMContainerAllocator extends RMContainerRequestor
this.scheduledRequests.setNumOpportunisticMapsPer100(
conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PER_100,
MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PER_100));
+ LOG.info(this.scheduledRequests.getNumOpportunisticMapsPer100() +
+ "% of the mappers will be scheduled using OPPORTUNISTIC containers");
}
@Override
@@ -1060,6 +1062,10 @@ public class RMContainerAllocator extends RMContainerRequestor
this.numOpportunisticMapsPer100 = numMaps;
}
+ int getNumOpportunisticMapsPer100() {
+ return this.numOpportunisticMapsPer100;
+ }
+
@VisibleForTesting
final LinkedHashMap<TaskAttemptId, ContainerRequest> reduces =
new LinkedHashMap<TaskAttemptId, ContainerRequest>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index c598aa0..b18fb4a 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
@@ -191,7 +191,7 @@ public class NodeInfo {
return null;
}
- public QueuedContainersStatus getQueuedContainersStatus() {
+ public OpportunisticContainersStatus getOpportunisticContainersStatus() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 6d0ffbd..4edc216 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
@@ -180,7 +180,7 @@ public class RMNodeWrapper implements RMNode {
return Collections.EMPTY_LIST;
}
- public QueuedContainersStatus getQueuedContainersStatus() {
+ public OpportunisticContainersStatus getOpportunisticContainersStatus() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
index 89e054b..440cd0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
@@ -125,10 +125,11 @@ public abstract class NodeStatus {
@Private
@Unstable
- public abstract QueuedContainersStatus getQueuedContainersStatus();
+ public abstract OpportunisticContainersStatus
+ getOpportunisticContainersStatus();
@Private
@Unstable
- public abstract void setQueuedContainersStatus(
- QueuedContainersStatus queuedContainersStatus);
+ public abstract void setOpportunisticContainersStatus(
+ OpportunisticContainersStatus opportunisticContainersStatus);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OpportunisticContainersStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OpportunisticContainersStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OpportunisticContainersStatus.java
new file mode 100644
index 0000000..732db2a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OpportunisticContainersStatus.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p> <code>OpportunisticContainersStatus</code> captures information
+ * pertaining to the state of execution of the opportunistic containers within a
+ * node. </p>
+ */
+@Private
+@Unstable
+public abstract class OpportunisticContainersStatus {
+ public static OpportunisticContainersStatus newInstance() {
+ return Records.newRecord(OpportunisticContainersStatus.class);
+ }
+
+ /**
+ * Returns the number of currently running opportunistic containers on the
+ * node.
+ *
+ * @return number of running opportunistic containers.
+ */
+ @Private
+ @Unstable
+ public abstract int getRunningOpportContainers();
+
+ /**
+ * Sets the number of running opportunistic containers.
+ *
+ * @param runningOpportContainers number of running opportunistic containers.
+ */
+ @Private
+ @Unstable
+ public abstract void setRunningOpportContainers(int runningOpportContainers);
+
+ /**
+ * Returns memory currently used on the node for running opportunistic
+ * containers.
+ *
+ * @return memory (in bytes) used for running opportunistic containers.
+ */
+ @Private
+ @Unstable
+ public abstract long getOpportMemoryUsed();
+
+ /**
+ * Sets the memory used on the node for running opportunistic containers.
+ *
+ * @param opportMemoryUsed memory (in bytes) used for running opportunistic
+ * containers.
+ */
+ @Private
+ @Unstable
+ public abstract void setOpportMemoryUsed(long opportMemoryUsed);
+
+ /**
+ * Returns CPU cores currently used on the node for running opportunistic
+ * containers.
+ *
+ * @return CPU cores used for running opportunistic containers.
+ */
+ @Private
+ @Unstable
+ public abstract int getOpportCoresUsed();
+
+ /**
+ * Sets the CPU cores used on the node for running opportunistic containers.
+ *
+ * @param opportCoresUsed memory (in bytes) used for running opportunistic
+ * containers.
+ */
+ @Private
+ @Unstable
+ public abstract void setOpportCoresUsed(int opportCoresUsed);
+
+ /**
+ * Returns the number of queued opportunistic containers on the node.
+ *
+ * @return number of queued opportunistic containers.
+ */
+ @Private
+ @Unstable
+ public abstract int getQueuedOpportContainers();
+
+ /**
+ * Sets the number of queued opportunistic containers on the node.
+ *
+ * @param queuedOpportContainers number of queued opportunistic containers.
+ */
+ @Private
+ @Unstable
+ public abstract void setQueuedOpportContainers(int queuedOpportContainers);
+
+ /**
+ * Returns the length of the containers queue on the node.
+ *
+ * @return length of the containers queue.
+ */
+ @Private
+ @Unstable
+ public abstract int getWaitQueueLength();
+
+ /**
+ * Sets the length of the containers queue on the node.
+ *
+ * @param waitQueueLength length of the containers queue.
+ */
+ @Private
+ @Unstable
+ public abstract void setWaitQueueLength(int waitQueueLength);
+
+ /**
+ * Returns the estimated time that a container will have to wait if added to
+ * the queue of the node.
+ *
+ * @return estimated queuing time.
+ */
+ @Private
+ @Unstable
+ public abstract int getEstimatedQueueWaitTime();
+
+ /**
+ * Sets the estimated time that a container will have to wait if added to the
+ * queue of the node.
+ *
+ * @param queueWaitTime estimated queuing time.
+ */
+ @Private
+ @Unstable
+ public abstract void setEstimatedQueueWaitTime(int queueWaitTime);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java
deleted file mode 100644
index fb567d5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * <p>
- * <code>QueuedContainersStatus</code> captures information pertaining to the
- * state of execution of the Queueable containers within a node.
- * </p>
- */
-@Private
-@Evolving
-public abstract class QueuedContainersStatus {
- public static QueuedContainersStatus newInstance() {
- return Records.newRecord(QueuedContainersStatus.class);
- }
-
- public abstract int getEstimatedQueueWaitTime();
-
- public abstract void setEstimatedQueueWaitTime(int queueWaitTime);
-
- public abstract int getWaitQueueLength();
-
- public abstract void setWaitQueueLength(int waitQueueLength);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
index d6a1737..8aebc6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
@@ -41,9 +41,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.QueuedContainersStatusProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.OpportunisticContainersStatusProto;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -404,25 +404,25 @@ public class NodeStatusPBImpl extends NodeStatus {
}
@Override
- public synchronized QueuedContainersStatus getQueuedContainersStatus() {
- NodeStatusProtoOrBuilder p =
- this.viaProto ? this.proto : this.builder;
- if (!p.hasQueuedContainerStatus()) {
+ public synchronized OpportunisticContainersStatus
+ getOpportunisticContainersStatus() {
+ NodeStatusProtoOrBuilder p = this.viaProto ? this.proto : this.builder;
+ if (!p.hasOpportunisticContainersStatus()) {
return null;
}
- return convertFromProtoFormat(p.getQueuedContainerStatus());
+ return convertFromProtoFormat(p.getOpportunisticContainersStatus());
}
@Override
- public synchronized void setQueuedContainersStatus(
- QueuedContainersStatus queuedContainersStatus) {
+ public synchronized void setOpportunisticContainersStatus(
+ OpportunisticContainersStatus opportunisticContainersStatus) {
maybeInitBuilder();
- if (queuedContainersStatus == null) {
- this.builder.clearQueuedContainerStatus();
+ if (opportunisticContainersStatus == null) {
+ this.builder.clearOpportunisticContainersStatus();
return;
}
- this.builder.setQueuedContainerStatus(
- convertToProtoFormat(queuedContainersStatus));
+ this.builder.setOpportunisticContainersStatus(
+ convertToProtoFormat(opportunisticContainersStatus));
}
private NodeIdProto convertToProtoFormat(NodeId nodeId) {
@@ -468,14 +468,14 @@ public class NodeStatusPBImpl extends NodeStatus {
return new ResourceUtilizationPBImpl(p);
}
- private QueuedContainersStatusProto convertToProtoFormat(
- QueuedContainersStatus r) {
- return ((QueuedContainersStatusPBImpl) r).getProto();
+ private OpportunisticContainersStatusProto convertToProtoFormat(
+ OpportunisticContainersStatus r) {
+ return ((OpportunisticContainersStatusPBImpl) r).getProto();
}
- private QueuedContainersStatus convertFromProtoFormat(
- QueuedContainersStatusProto p) {
- return new QueuedContainersStatusPBImpl(p);
+ private OpportunisticContainersStatus convertFromProtoFormat(
+ OpportunisticContainersStatusProto p) {
+ return new OpportunisticContainersStatusPBImpl(p);
}
private ContainerPBImpl convertFromProtoFormat(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OpportunisticContainersStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OpportunisticContainersStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OpportunisticContainersStatusPBImpl.java
new file mode 100644
index 0000000..8399713
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OpportunisticContainersStatusPBImpl.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
+
+/**
+ * Protocol Buffer implementation of OpportunisticContainersStatus.
+ */
+public class OpportunisticContainersStatusPBImpl
+ extends OpportunisticContainersStatus {
+
+ private YarnServerCommonProtos.OpportunisticContainersStatusProto proto =
+ YarnServerCommonProtos.OpportunisticContainersStatusProto
+ .getDefaultInstance();
+ private YarnServerCommonProtos.OpportunisticContainersStatusProto.Builder
+ builder = null;
+ private boolean viaProto = false;
+
+ public OpportunisticContainersStatusPBImpl() {
+ builder =
+ YarnServerCommonProtos.OpportunisticContainersStatusProto.newBuilder();
+ }
+
+ public OpportunisticContainersStatusPBImpl(YarnServerCommonProtos
+ .OpportunisticContainersStatusProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public YarnServerCommonProtos.OpportunisticContainersStatusProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = YarnServerCommonProtos.OpportunisticContainersStatusProto
+ .newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int getRunningOpportContainers() {
+ YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getRunningOpportContainers();
+ }
+
+ @Override
+ public void setRunningOpportContainers(int runningOpportContainers) {
+ maybeInitBuilder();
+ builder.setRunningOpportContainers(runningOpportContainers);
+ }
+
+ @Override
+ public long getOpportMemoryUsed() {
+ YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getOpportMemoryUsed();
+ }
+
+ @Override
+ public void setOpportMemoryUsed(long opportMemoryUsed) {
+ maybeInitBuilder();
+ builder.setOpportMemoryUsed(opportMemoryUsed);
+ }
+
+ @Override
+ public int getOpportCoresUsed() {
+ YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getOpportCoresUsed();
+ }
+
+ @Override
+ public void setOpportCoresUsed(int opportCoresUsed) {
+ maybeInitBuilder();
+ builder.setOpportCoresUsed(opportCoresUsed);
+ }
+
+ @Override
+ public int getQueuedOpportContainers() {
+ YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getQueuedOpportContainers();
+ }
+
+ @Override
+ public void setQueuedOpportContainers(int queuedOpportContainers) {
+ maybeInitBuilder();
+ builder.setQueuedOpportContainers(queuedOpportContainers);
+ }
+
+ @Override
+ public int getWaitQueueLength() {
+ YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getWaitQueueLength();
+ }
+
+ @Override
+ public void setWaitQueueLength(int waitQueueLength) {
+ maybeInitBuilder();
+ builder.setWaitQueueLength(waitQueueLength);
+ }
+
+ @Override
+ public int getEstimatedQueueWaitTime() {
+ YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getEstimatedQueueWaitTime();
+ }
+
+ @Override
+ public void setEstimatedQueueWaitTime(int queueWaitTime) {
+ maybeInitBuilder();
+ builder.setEstimatedQueueWaitTime(queueWaitTime);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java
deleted file mode 100644
index 16b80dd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.api.records.impl.pb;
-
-import org.apache.hadoop.yarn.proto.YarnServerCommonProtos;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
-
-/**
- * Protocol Buffer implementation of QueuedContainersStatus.
- */
-public class QueuedContainersStatusPBImpl extends QueuedContainersStatus {
-
- private YarnServerCommonProtos.QueuedContainersStatusProto proto =
- YarnServerCommonProtos.QueuedContainersStatusProto.getDefaultInstance();
- private YarnServerCommonProtos.QueuedContainersStatusProto.Builder builder =
- null;
- private boolean viaProto = false;
-
- public QueuedContainersStatusPBImpl() {
- builder = YarnServerCommonProtos.QueuedContainersStatusProto.newBuilder();
- }
-
- public QueuedContainersStatusPBImpl(YarnServerCommonProtos
- .QueuedContainersStatusProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- public YarnServerCommonProtos.QueuedContainersStatusProto getProto() {
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder =
- YarnServerCommonProtos.QueuedContainersStatusProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- @Override
- public int getEstimatedQueueWaitTime() {
- YarnServerCommonProtos.QueuedContainersStatusProtoOrBuilder p =
- viaProto ? proto : builder;
- return p.getEstimatedQueueWaitTime();
- }
-
- @Override
- public void setEstimatedQueueWaitTime(int queueWaitTime) {
- maybeInitBuilder();
- builder.setEstimatedQueueWaitTime(queueWaitTime);
- }
-
- @Override
- public int getWaitQueueLength() {
- YarnServerCommonProtos.QueuedContainersStatusProtoOrBuilder p =
- viaProto ? proto : builder;
- return p.getWaitQueueLength();
- }
-
- @Override
- public void setWaitQueueLength(int waitQueueLength) {
- maybeInitBuilder();
- builder.setWaitQueueLength(waitQueueLength);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
index c23d557..98b172d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
@@ -39,12 +39,16 @@ message NodeStatusProto {
optional ResourceUtilizationProto containers_utilization = 6;
optional ResourceUtilizationProto node_utilization = 7;
repeated ContainerProto increased_containers = 8;
- optional QueuedContainersStatusProto queued_container_status = 9;
+ optional OpportunisticContainersStatusProto opportunistic_containers_status = 9;
}
-message QueuedContainersStatusProto {
- optional int32 estimated_queue_wait_time = 1;
- optional int32 wait_queue_length = 2;
+message OpportunisticContainersStatusProto {
+ optional int32 running_opport_containers = 1;
+ optional int64 opport_memory_used = 2;
+ optional int32 opport_cores_used = 3;
+ optional int32 queued_opport_containers = 4;
+ optional int32 wait_queue_length = 5;
+ optional int32 estimated_queue_wait_time = 6;
}
message MasterKeyProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
index 9f4b436..b9cde24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodeHeartbeatRe
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterNodeManagerRequestPBImpl;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.Test;
@@ -146,11 +146,11 @@ public class TestProtocolRecords {
Records.newRecord(NodeHeartbeatRequest.class);
NodeStatus nodeStatus =
Records.newRecord(NodeStatus.class);
- QueuedContainersStatus queuedContainersStatus = Records.newRecord
- (QueuedContainersStatus.class);
- queuedContainersStatus.setEstimatedQueueWaitTime(123);
- queuedContainersStatus.setWaitQueueLength(321);
- nodeStatus.setQueuedContainersStatus(queuedContainersStatus);
+ OpportunisticContainersStatus opportunisticContainersStatus =
+ Records.newRecord(OpportunisticContainersStatus.class);
+ opportunisticContainersStatus.setEstimatedQueueWaitTime(123);
+ opportunisticContainersStatus.setWaitQueueLength(321);
+ nodeStatus.setOpportunisticContainersStatus(opportunisticContainersStatus);
record.setNodeStatus(nodeStatus);
NodeHeartbeatRequestPBImpl pb = new
@@ -159,9 +159,10 @@ public class TestProtocolRecords {
Assert.assertEquals(123,
pb.getNodeStatus()
- .getQueuedContainersStatus().getEstimatedQueueWaitTime());
+ .getOpportunisticContainersStatus().getEstimatedQueueWaitTime());
Assert.assertEquals(321,
- pb.getNodeStatus().getQueuedContainersStatus().getWaitQueueLength());
+ pb.getNodeStatus().getOpportunisticContainersStatus()
+ .getWaitQueueLength());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index f692bf1..ec7e1d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -75,7 +75,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
@@ -465,16 +465,21 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
createKeepAliveApplicationList(), nodeHealthStatus,
containersUtilization, nodeUtilization, increasedContainers);
- nodeStatus.setQueuedContainersStatus(getQueuedContainerStatus());
+ nodeStatus.setOpportunisticContainersStatus(
+ getOpportunisticContainersStatus());
return nodeStatus;
}
- private QueuedContainersStatus getQueuedContainerStatus() {
- QueuedContainersStatus status = QueuedContainersStatus.newInstance();
- status.setWaitQueueLength(
- this.context.getQueuingContext().getQueuedContainers().size());
+ /**
+ * Get the status of the OPPORTUNISTIC containers.
+ * @return the status of the OPPORTUNISTIC containers.
+ */
+ private OpportunisticContainersStatus getOpportunisticContainersStatus() {
+ OpportunisticContainersStatus status =
+ this.context.getContainerManager().getOpportunisticContainersStatus();
return status;
}
+
/**
* Get the aggregated utilization of the containers in this node.
* @return Resource utilization of all the containers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java
index 0da02b3..1cbb8c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.service.ServiceStateChangeListener;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.nodemanager.ContainerManagerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor
.ContainersMonitor;
@@ -35,6 +36,8 @@ public interface ContainerManager extends ServiceStateChangeListener,
ContainersMonitor getContainersMonitor();
+ OpportunisticContainersStatus getOpportunisticContainersStatus();
+
void updateQueuingLimit(ContainerQueuingLimit queuingLimit);
void setBlockNewContainerRequests(boolean blockNewContainerRequests);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index e8c2b75..76933ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -88,6 +88,7 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedContainersEvent;
import org.apache.hadoop.yarn.server.nodemanager.CMgrDecreaseContainersResourceEvent;
@@ -1521,8 +1522,13 @@ public class ContainerManagerImpl extends CompositeService implements
}
@Override
+ public OpportunisticContainersStatus getOpportunisticContainersStatus() {
+ return null;
+ }
+
+ @Override
public void updateQueuingLimit(ContainerQueuingLimit queuingLimit) {
- LOG.trace("Implementation does not support queuing of Containers !!");
+ LOG.trace("Implementation does not support queuing of Containers!!");
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 432cd67..72e99d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -304,21 +304,21 @@ public class ContainersMonitorImpl extends AbstractService implements
/**
* @return Virtual memory limit for the process tree in bytes
*/
- synchronized long getVmemLimit() {
+ public synchronized long getVmemLimit() {
return this.vmemLimit;
}
/**
* @return Physical memory limit for the process tree in bytes
*/
- synchronized long getPmemLimit() {
+ public synchronized long getPmemLimit() {
return this.pmemLimit;
}
/**
* @return Number of cpu vcores assigned
*/
- synchronized int getCpuVcores() {
+ public synchronized int getCpuVcores() {
return this.cpuVcores;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
index 5d2f4d4..495f57a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
@@ -80,10 +81,14 @@ public class QueuingContainerManagerImpl extends ContainerManagerImpl {
private ConcurrentMap<ContainerId, AllocatedContainerInfo>
allocatedOpportunisticContainers;
+ private long allocatedMemoryOpportunistic;
+ private int allocatedVCoresOpportunistic;
+
private Queue<AllocatedContainerInfo> queuedGuaranteedContainers;
private Queue<AllocatedContainerInfo> queuedOpportunisticContainers;
private Set<ContainerId> opportunisticContainersToKill;
+ private final OpportunisticContainersStatus opportunisticContainersStatus;
private final ContainerQueuingLimit queuingLimit;
public QueuingContainerManagerImpl(Context context, ContainerExecutor exec,
@@ -93,10 +98,14 @@ public class QueuingContainerManagerImpl extends ContainerManagerImpl {
dirsHandler);
this.allocatedGuaranteedContainers = new ConcurrentHashMap<>();
this.allocatedOpportunisticContainers = new ConcurrentHashMap<>();
+ this.allocatedMemoryOpportunistic = 0;
+ this.allocatedVCoresOpportunistic = 0;
this.queuedGuaranteedContainers = new ConcurrentLinkedQueue<>();
this.queuedOpportunisticContainers = new ConcurrentLinkedQueue<>();
this.opportunisticContainersToKill = Collections.synchronizedSet(
new HashSet<ContainerId>());
+ this.opportunisticContainersStatus =
+ OpportunisticContainersStatus.newInstance();
this.queuingLimit = ContainerQueuingLimit.newInstance();
}
@@ -196,6 +205,8 @@ public class QueuingContainerManagerImpl extends ContainerManagerImpl {
} else {
allocatedOpportunisticContainers.put(pti.getContainerId(),
allocatedContainerInfo);
+ allocatedMemoryOpportunistic += pti.getPmemLimit();
+ allocatedVCoresOpportunistic += pti.getCpuVcores();
}
getContainersMonitor().increaseContainersAllocation(pti);
@@ -267,6 +278,11 @@ public class QueuingContainerManagerImpl extends ContainerManagerImpl {
if (contToRemove != null) {
getContainersMonitor().decreaseContainersAllocation(contToRemove
.getPti());
+
+ if (contToRemove.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
+ allocatedMemoryOpportunistic -= contToRemove.getPti().getPmemLimit();
+ allocatedVCoresOpportunistic -= contToRemove.getPti().getCpuVcores();
+ }
}
}
@@ -557,6 +573,22 @@ public class QueuingContainerManagerImpl extends ContainerManagerImpl {
}
@Override
+ public OpportunisticContainersStatus getOpportunisticContainersStatus() {
+ opportunisticContainersStatus
+ .setRunningOpportContainers(allocatedOpportunisticContainers.size());
+ opportunisticContainersStatus
+ .setOpportMemoryUsed(allocatedMemoryOpportunistic);
+ opportunisticContainersStatus
+ .setOpportCoresUsed(allocatedVCoresOpportunistic);
+ opportunisticContainersStatus
+ .setQueuedOpportContainers(queuedOpportunisticContainers.size());
+ opportunisticContainersStatus.setWaitQueueLength(
+ queuedGuaranteedContainers.size() +
+ queuedOpportunisticContainers.size());
+ return opportunisticContainersStatus;
+ }
+
+ @Override
public void updateQueuingLimit(ContainerQueuingLimit limit) {
this.queuingLimit.setMaxQueueLength(limit.getMaxQueueLength());
// TODO: Include wait time as well once it is implemented
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
index a1e0bc7..4beccc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
@@ -86,6 +86,7 @@ public class ContainerPage extends NMView implements YarnWebParams {
._("User", info.getUser())
._("TotalMemoryNeeded", info.getMemoryNeeded())
._("TotalVCoresNeeded", info.getVCoresNeeded())
+ ._("ExecutionType", info.getExecutionType())
._("logs", info.getShortLogLink(), "Link to logs");
html._(InfoBlock.class);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
index 10a8156..26d3f02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
@@ -46,6 +46,7 @@ public class ContainerInfo {
protected String user;
protected long totalMemoryNeededMB;
protected long totalVCoresNeeded;
+ private String executionType;
protected String containerLogsLink;
protected String nodeId;
@XmlTransient
@@ -84,6 +85,8 @@ public class ContainerInfo {
this.totalMemoryNeededMB = res.getMemorySize();
this.totalVCoresNeeded = res.getVirtualCores();
}
+ this.executionType =
+ container.getContainerTokenIdentifier().getExecutionType().name();
this.containerLogsShortLink = ujoin("containerlogs", this.id,
container.getUser());
@@ -143,6 +146,10 @@ public class ContainerInfo {
return this.totalVCoresNeeded;
}
+ public String getExecutionType() {
+ return this.executionType;
+ }
+
public List<String> getContainerLogFiles() {
return this.containerLogFiles;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
index 36acfe0..a99ce28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
@@ -515,7 +515,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
public void verifyNodeContainerInfo(JSONObject info, Container cont)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 10, info.length());
+ assertEquals("incorrect number of elements", 11, info.length());
verifyNodeContainerInfoGeneric(cont, info.getString("id"),
info.getString("state"), info.getString("user"),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index 815d29d..bdd5718 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -81,8 +81,9 @@ import java.util.List;
/**
* The OpportunisticContainerAllocatorAMService is started instead of the
- * ApplicationMasterService if distributed scheduling is enabled for the YARN
- * cluster.
+ * ApplicationMasterService if opportunistic scheduling is enabled for the YARN
+ * cluster (either centralized or distributed opportunistic scheduling).
+ *
* It extends the functionality of the ApplicationMasterService by servicing
* clients (AMs and AMRMProxy request interceptors) that understand the
* DistributedSchedulingProtocol.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 10e2afa..6d95b56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
/**
* Node managers information on available resources
@@ -170,7 +170,7 @@ public interface RMNode {
public List<Container> pullNewlyIncreasedContainers();
- QueuedContainersStatus getQueuedContainersStatus();
+ OpportunisticContainersStatus getOpportunisticContainersStatus();
long getUntrackedTimeStamp();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index d3d115f..71ebddb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -61,7 +61,7 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent;
@@ -134,7 +134,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private ResourceUtilization nodeUtilization;
/* Container Queue Information for the node.. Used by Distributed Scheduler */
- private QueuedContainersStatus queuedContainersStatus;
+ private OpportunisticContainersStatus opportunisticContainersStatus;
private final ContainerAllocationExpirer containerAllocationExpirer;
/* set of containers that have just launched */
@@ -1169,7 +1169,8 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
- rmNode.setQueuedContainersStatus(statusEvent.getContainerQueueInfo());
+ rmNode.setOpportunisticContainersStatus(
+ statusEvent.getOpportunisticContainersStatus());
NodeHealthStatus remoteNodeHealthStatus = updateRMNodeFromStatusEvents(
rmNode, statusEvent);
NodeState initialState = rmNode.getState();
@@ -1480,22 +1481,22 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
return this.originalTotalCapability;
}
- public QueuedContainersStatus getQueuedContainersStatus() {
+ public OpportunisticContainersStatus getOpportunisticContainersStatus() {
this.readLock.lock();
try {
- return this.queuedContainersStatus;
+ return this.opportunisticContainersStatus;
} finally {
this.readLock.unlock();
}
}
- public void setQueuedContainersStatus(QueuedContainersStatus
- queuedContainersStatus) {
+ public void setOpportunisticContainersStatus(
+ OpportunisticContainersStatus opportunisticContainersStatus) {
this.writeLock.lock();
try {
- this.queuedContainersStatus = queuedContainersStatus;
+ this.opportunisticContainersStatus = opportunisticContainersStatus;
} finally {
this.writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
index 5eeaabe..f9fe159 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -80,8 +80,8 @@ public class RMNodeStatusEvent extends RMNodeEvent {
return this.logAggregationReportsForApps;
}
- public QueuedContainersStatus getContainerQueueInfo() {
- return this.nodeStatus.getQueuedContainersStatus();
+ public OpportunisticContainersStatus getOpportunisticContainersStatus() {
+ return this.nodeStatus.getOpportunisticContainersStatus();
}
public void setLogAggregationReportsForApps(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 5e6a726..93df1e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -570,7 +571,7 @@ public abstract class AbstractYarnScheduler
return;
}
- if (!rmContainer.isRemotelyAllocated()) {
+ if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
completedContainerInternal(rmContainer, containerStatus, event);
} else {
ContainerId containerId = rmContainer.getContainerId();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
index b80a17c..232b4ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
@@ -24,7 +24,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.ClusterMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -195,11 +195,11 @@ public class NodeQueueLoadMonitor implements ClusterMonitor {
@Override
public void updateNode(RMNode rmNode) {
LOG.debug("Node update event from: " + rmNode.getNodeID());
- QueuedContainersStatus queuedContainersStatus =
- rmNode.getQueuedContainersStatus();
+ OpportunisticContainersStatus opportunisticContainersStatus =
+ rmNode.getOpportunisticContainersStatus();
int estimatedQueueWaitTime =
- queuedContainersStatus.getEstimatedQueueWaitTime();
- int waitQueueLength = queuedContainersStatus.getWaitQueueLength();
+ opportunisticContainersStatus.getEstimatedQueueWaitTime();
+ int waitQueueLength = opportunisticContainersStatus.getWaitQueueLength();
// Add nodes to clusterNodes. If estimatedQueueTime is -1, ignore node
// UNLESS comparator is based on queue length.
ReentrantReadWriteLock.WriteLock writeLock = clusterNodesLock.writeLock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index 7063421..7b68b84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -18,17 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL;
-import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
-
-import java.util.Collection;
-
+import com.google.inject.Inject;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
@@ -42,18 +35,29 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import com.google.inject.Inject;
+import java.util.Collection;
+
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
class NodesPage extends RmView {
static class NodesBlock extends HtmlBlock {
final ResourceManager rm;
private static final long BYTES_IN_MB = 1024 * 1024;
+ private static boolean opportunisticContainersEnabled;
@Inject
NodesBlock(ResourceManager rm, ViewContext ctx) {
super(ctx);
this.rm = rm;
+ this.opportunisticContainersEnabled = YarnConfiguration
+ .isOpportunisticContainerAllocationEnabled(
+ this.rm.getRMContext().getYarnConfiguration());
}
@Override
@@ -61,9 +65,10 @@ class NodesPage extends RmView {
html._(MetricsOverviewTable.class);
ResourceScheduler sched = rm.getResourceScheduler();
+
String type = $(NODE_STATE);
String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim();
- TBODY<TABLE<Hamlet>> tbody =
+ Hamlet.TR<Hamlet.THEAD<TABLE<Hamlet>>> trbody =
html.table("#nodes").thead().tr()
.th(".nodelabels", "Node Labels")
.th(".rack", "Rack")
@@ -71,13 +76,29 @@ class NodesPage extends RmView {
.th(".nodeaddress", "Node Address")
.th(".nodehttpaddress", "Node HTTP Address")
.th(".lastHealthUpdate", "Last health-update")
- .th(".healthReport", "Health-report")
- .th(".containers", "Containers")
- .th(".mem", "Mem Used")
- .th(".mem", "Mem Avail")
- .th(".vcores", "VCores Used")
- .th(".vcores", "VCores Avail")
- .th(".nodeManagerVersion", "Version")._()._().tbody();
+ .th(".healthReport", "Health-report");
+
+ if (!this.opportunisticContainersEnabled) {
+ trbody.th(".containers", "Containers")
+ .th(".mem", "Mem Used")
+ .th(".mem", "Mem Avail")
+ .th(".vcores", "VCores Used")
+ .th(".vcores", "VCores Avail");
+ } else {
+ trbody.th(".containers", "Running Containers (G)")
+ .th(".mem", "Mem Used (G)")
+ .th(".mem", "Mem Avail (G)")
+ .th(".vcores", "VCores Used (G)")
+ .th(".vcores", "VCores Avail (G)")
+ .th(".containers", "Running Containers (O)")
+ .th(".mem", "Mem Used (O)")
+ .th(".vcores", "VCores Used (O)")
+ .th(".containers", "Queued Containers");
+ }
+
+ TBODY<TABLE<Hamlet>> tbody =
+ trbody.th(".nodeManagerVersion", "Version")._()._().tbody();
+
NodeState stateFilter = null;
if (type != null && !type.isEmpty()) {
stateFilter = NodeState.valueOf(StringUtils.toUpperCase(type));
@@ -153,7 +174,23 @@ class NodesPage extends RmView {
.append("\",\"").append(String.valueOf(info.getUsedVirtualCores()))
.append("\",\"")
.append(String.valueOf(info.getAvailableVirtualCores()))
- .append("\",\"").append(ni.getNodeManagerVersion())
+ .append("\",\"");
+
+ // If opportunistic containers are enabled, add extra fields.
+ if (this.opportunisticContainersEnabled) {
+ nodeTableData
+ .append(String.valueOf(info.getNumRunningOpportContainers()))
+ .append("\",\"").append("<br title='")
+ .append(String.valueOf(info.getUsedMemoryOpport())).append("'>")
+ .append(StringUtils.byteDesc(info.getUsedMemoryOpport()))
+ .append("\",\"")
+ .append(String.valueOf(info.getUsedVirtualCoresOpport()))
+ .append("\",\"")
+ .append(String.valueOf(info.getNumQueuedContainers()))
+ .append("\",\"");
+ }
+
+ nodeTableData.append(ni.getNodeManagerVersion())
.append("\"],\n");
}
if (nodeTableData.charAt(nodeTableData.length() - 2) == ',') {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 4a6aa4b..aee7717 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
@@ -49,6 +50,10 @@ public class NodeInfo {
protected long availMemoryMB;
protected long usedVirtualCores;
protected long availableVirtualCores;
+ private int numRunningOpportContainers;
+ private long usedMemoryOpport; // Memory in bytes.
+ private long usedVirtualCoresOpport;
+ private int numQueuedContainers;
protected ArrayList<String> nodeLabels = new ArrayList<String>();
protected ResourceUtilizationInfo resourceUtilization;
@@ -66,7 +71,8 @@ public class NodeInfo {
this.usedMemoryMB = report.getUsedResource().getMemorySize();
this.availMemoryMB = report.getAvailableResource().getMemorySize();
this.usedVirtualCores = report.getUsedResource().getVirtualCores();
- this.availableVirtualCores = report.getAvailableResource().getVirtualCores();
+ this.availableVirtualCores =
+ report.getAvailableResource().getVirtualCores();
}
this.id = id.toString();
this.rack = ni.getRackName();
@@ -76,7 +82,22 @@ public class NodeInfo {
this.lastHealthUpdate = ni.getLastHealthReportTime();
this.healthReport = String.valueOf(ni.getHealthReport());
this.version = ni.getNodeManagerVersion();
-
+
+ // Status of opportunistic containers.
+ this.numRunningOpportContainers = 0;
+ this.usedMemoryOpport = 0;
+ this.usedVirtualCoresOpport = 0;
+ this.numQueuedContainers = 0;
+ OpportunisticContainersStatus opportStatus =
+ ni.getOpportunisticContainersStatus();
+ if (opportStatus != null) {
+ this.numRunningOpportContainers =
+ opportStatus.getRunningOpportContainers();
+ this.usedMemoryOpport = opportStatus.getOpportMemoryUsed();
+ this.usedVirtualCoresOpport = opportStatus.getOpportCoresUsed();
+ this.numQueuedContainers = opportStatus.getQueuedOpportContainers();
+ }
+
// add labels
Set<String> labelSet = ni.getNodeLabels();
if (labelSet != null) {
@@ -140,6 +161,22 @@ public class NodeInfo {
return this.availableVirtualCores;
}
+ public int getNumRunningOpportContainers() {
+ return numRunningOpportContainers;
+ }
+
+ public long getUsedMemoryOpport() {
+ return usedMemoryOpport;
+ }
+
+ public long getUsedVirtualCoresOpport() {
+ return usedVirtualCoresOpport;
+ }
+
+ public int getNumQueuedContainers() {
+ return numQueuedContainers;
+ }
+
public ArrayList<String> getNodeLabels() {
return this.nodeLabels;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 5a89e54..7959e74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
@@ -260,7 +260,7 @@ public class MockNodes {
return this.nodeUtilization;
}
- public QueuedContainersStatus getQueuedContainersStatus() {
+ public OpportunisticContainersStatus getOpportunisticContainersStatus() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java
index 5f63923..dfd21ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
-import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.junit.Assert;
import org.junit.Test;
@@ -183,13 +183,13 @@ public class TestNodeQueueLoadMonitor {
RMNode node1 = Mockito.mock(RMNode.class);
NodeId nID1 = new FakeNodeId(host, port);
Mockito.when(node1.getNodeID()).thenReturn(nID1);
- QueuedContainersStatus status1 =
- Mockito.mock(QueuedContainersStatus.class);
+ OpportunisticContainersStatus status1 =
+ Mockito.mock(OpportunisticContainersStatus.class);
Mockito.when(status1.getEstimatedQueueWaitTime())
.thenReturn(waitTime);
Mockito.when(status1.getWaitQueueLength())
.thenReturn(queueLength);
- Mockito.when(node1.getQueuedContainersStatus()).thenReturn(status1);
+ Mockito.when(node1.getOpportunisticContainersStatus()).thenReturn(status1);
return node1;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
index 71d077b..cc97674 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodesPage.NodesBlock;
@@ -49,6 +50,7 @@ public class TestNodesPage {
// future. In that case this value should be adjusted to the new value.
final int numberOfThInMetricsTable = 23;
final int numberOfActualTableHeaders = 13;
+ private final int numberOfThForOpportunisticContainers = 4;
private Injector injector;
@@ -135,4 +137,35 @@ public class TestNodesPage {
Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
.print("<td");
}
+
+ @Test
+ public void testNodesBlockRenderForOpportunisticContainers() {
+ final RMContext mockRMContext =
+ TestRMWebApp.mockRMContext(3, numberOfRacks, numberOfNodesPerRack,
+ 8 * TestRMWebApp.GiB);
+ mockRMContext.getYarnConfiguration().setBoolean(
+ YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
+ injector =
+ WebAppTests.createMockInjector(RMContext.class, mockRMContext,
+ new Module() {
+ @Override
+ public void configure(Binder binder) {
+ try {
+ binder.bind(ResourceManager.class).toInstance(
+ TestRMWebApp.mockRm(mockRMContext));
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ });
+ injector.getInstance(NodesBlock.class).render();
+ PrintWriter writer = injector.getInstance(PrintWriter.class);
+ WebAppTests.flushOutput(injector);
+
+ Mockito.verify(writer, Mockito.times(
+ numberOfActualTableHeaders + numberOfThInMetricsTable +
+ numberOfThForOpportunisticContainers)).print("<th");
+ Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
+ .print("<td");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index edf3b3f..df8d67c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -200,6 +200,7 @@ public class TestRMWebApp {
}
};
rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
+ rmContext.setYarnConfiguration(new YarnConfiguration());
return rmContext;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-2995. Enhance UI to show
cluster resource utilization of various container Execution types.
(Konstantinos Karanasos via asuresh)
Posted by fa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aafc122/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index 71f8950..3ff8f6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -722,13 +723,17 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
"aggregatedContainersPhysicalMemoryMB"),
WebServicesTestUtils.getXmlInt(element,
"aggregatedContainersVirtualMemoryMB"),
- WebServicesTestUtils.getXmlFloat(element, "containersCPUUsage"));
+ WebServicesTestUtils.getXmlFloat(element, "containersCPUUsage"),
+ WebServicesTestUtils.getXmlInt(element, "numRunningOpportContainers"),
+ WebServicesTestUtils.getXmlLong(element, "usedMemoryOpport"),
+ WebServicesTestUtils.getXmlInt(element, "usedVirtualCoresOpport"),
+ WebServicesTestUtils.getXmlInt(element, "numQueuedContainers"));
}
}
public void verifyNodeInfo(JSONObject nodeInfo, RMNode nm)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 14, nodeInfo.length());
+ assertEquals("incorrect number of elements", 18, nodeInfo.length());
JSONObject resourceInfo = nodeInfo.getJSONObject("resourceUtilization");
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
@@ -745,21 +750,29 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
resourceInfo.getDouble("nodeCPUUsage"),
resourceInfo.getInt("aggregatedContainersPhysicalMemoryMB"),
resourceInfo.getInt("aggregatedContainersVirtualMemoryMB"),
- resourceInfo.getDouble("containersCPUUsage"));
+ resourceInfo.getDouble("containersCPUUsage"),
+ nodeInfo.getInt("numRunningOpportContainers"),
+ nodeInfo.getLong("usedMemoryOpport"),
+ nodeInfo.getInt("usedVirtualCoresOpport"),
+ nodeInfo.getInt("numQueuedContainers"));
}
public void verifyNodeInfoGeneric(RMNode node, String state, String rack,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
- int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores,
- long availVirtualCores, String version, int nodePhysicalMemoryMB,
- int nodeVirtualMemoryMB, double nodeCPUUsage,
+ int numContainers, long usedMemoryMB, long availMemoryMB,
+ long usedVirtualCores, long availVirtualCores, String version,
+ int nodePhysicalMemoryMB, int nodeVirtualMemoryMB, double nodeCPUUsage,
int containersPhysicalMemoryMB, int containersVirtualMemoryMB,
- double containersCPUUsage)
+ double containersCPUUsage, int numRunningOpportContainers,
+ long usedMemoryOpport, int usedVirtualCoresOpport,
+ int numQueuedContainers)
throws JSONException, Exception {
ResourceScheduler sched = rm.getResourceScheduler();
SchedulerNodeReport report = sched.getNodeReport(node.getNodeID());
+ OpportunisticContainersStatus opportunisticStatus =
+ node.getOpportunisticContainersStatus();
WebServicesTestUtils.checkStringMatch("state", node.getState().toString(),
state);
@@ -807,6 +820,20 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
.getAvailableResource().getVirtualCores(), availVirtualCores);
}
+
+ if (opportunisticStatus != null) {
+ assertEquals("numRunningOpportContainers doesn't match: " +
+ numRunningOpportContainers,
+ opportunisticStatus.getRunningOpportContainers(),
+ numRunningOpportContainers);
+ assertEquals("usedMemoryOpport doesn't match: " + usedMemoryOpport,
+ opportunisticStatus.getOpportMemoryUsed(), usedMemoryOpport);
+ assertEquals(
+ "usedVirtualCoresOpport doesn't match: " + usedVirtualCoresOpport,
+ opportunisticStatus.getOpportCoresUsed(), usedVirtualCoresOpport);
+ assertEquals("numQueuedContainers doesn't match: " + numQueuedContainers,
+ opportunisticStatus.getQueuedOpportContainers(), numQueuedContainers);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HADOOP-13603. Ignore package line
length checkstyle rule. Contributed by Shane Kumpf.
Posted by fa...@apache.org.
HADOOP-13603. Ignore package line length checkstyle rule. Contributed by Shane Kumpf.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b577f43
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b577f43
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b577f43
Branch: refs/heads/HADOOP-13345
Commit: 5b577f43a269caeee776a59695427985d0cd1697
Parents: 235f39b
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 1 20:58:13 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 1 20:58:13 2016 +0900
----------------------------------------------------------------------
hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b577f43/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
----------------------------------------------------------------------
diff --git a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
index 033529f..851b57d 100644
--- a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
+++ b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
@@ -124,7 +124,7 @@
<!-- Checks for Size Violations. -->
<!-- See http://checkstyle.sf.net/config_sizes.html -->
<module name="LineLength">
- <property name="ignorePattern" value="^import"/>
+ <property name="ignorePattern" value="^(package|import) .*"/>
</module>
<module name="MethodLength"/>
<module name="ParameterNumber"/>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDFS-11065. Add space quota tests
for heterogenous storages. Contributed by Xiaobing Zhou
Posted by fa...@apache.org.
HDFS-11065. Add space quota tests for heterogenous storages. Contributed by Xiaobing Zhou
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa1512a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa1512a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa1512a7
Branch: refs/heads/HADOOP-13345
Commit: fa1512a740b2ed2661743d6b5483ef3eb49e5634
Parents: 76893a4
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Nov 1 14:42:03 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Nov 1 14:42:03 2016 -0700
----------------------------------------------------------------------
.../server/namenode/TestQuotaByStorageType.java | 166 ++++++++++++++++++-
1 file changed, 163 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa1512a7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java
index ee02fd0..be24e01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java
@@ -31,14 +31,21 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
- import org.apache.hadoop.hdfs.protocol.HdfsConstants;
- import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
+import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
import org.apache.hadoop.test.GenericTestUtils;
- import org.junit.After;
+import org.apache.hadoop.test.PathUtils;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.allOf;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
public class TestQuotaByStorageType {
@@ -789,4 +796,157 @@ public class TestQuotaByStorageType {
assertEquals(cs.getTypeQuota(t), -1);
}
}
+
+
+
+ /**
+ * Tests space quota for storage policy = WARM.
+ */
+ @Test
+ public void testStorageSpaceQuotaWithWarmPolicy() throws IOException {
+ final Path testDir = new Path(
+ PathUtils.getTestPath(getClass()),
+ GenericTestUtils.getMethodName());
+ assertTrue(dfs.mkdirs(testDir));
+
+ /* set policy to HOT */
+ dfs.setStoragePolicy(testDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);
+
+ /* init space quota */
+ final long storageSpaceQuota = BLOCKSIZE * 6;
+ final long storageTypeSpaceQuota = BLOCKSIZE * 1;
+
+ /* set space quota */
+ dfs.setQuota(testDir, HdfsConstants.QUOTA_DONT_SET, storageSpaceQuota);
+
+ /* init vars */
+ Path createdFile;
+ final long fileLen = BLOCKSIZE;
+
+ /**
+ * create one file with 3 replicas, REPLICATION * BLOCKSIZE go to DISK due
+ * to HOT policy
+ */
+ createdFile = new Path(testDir, "file1.data");
+ DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16, fileLen, BLOCKSIZE,
+ REPLICATION, seed);
+ assertTrue(dfs.exists(createdFile));
+ assertTrue(dfs.isFile(createdFile));
+
+ /* set space quota for DISK */
+ dfs.setQuotaByStorageType(testDir, StorageType.DISK, storageTypeSpaceQuota);
+
+ /* set policy to WARM */
+ dfs.setStoragePolicy(testDir, HdfsConstants.WARM_STORAGE_POLICY_NAME);
+
+ /* create another file with 3 replicas */
+ try {
+ createdFile = new Path(testDir, "file2.data");
+ /**
+ * This will fail since quota on DISK is 1 block but space consumed on
+ * DISK is already 3 blocks due to the first file creation.
+ */
+ DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16, fileLen,
+ BLOCKSIZE, REPLICATION, seed);
+ fail("should fail on QuotaByStorageTypeExceededException");
+ } catch (QuotaByStorageTypeExceededException e) {
+ LOG.info("Got expected exception ", e);
+ assertThat(e.toString(),
+ is(allOf(containsString("Quota by storage type"),
+ containsString("DISK on path"),
+ containsString(testDir.toString()))));
+ }
+ }
+
+ /**
+ * Tests if changing replication factor results in copying file as quota
+ * doesn't exceed.
+ */
+ @Test(timeout = 30000)
+ public void testStorageSpaceQuotaWithRepFactor() throws IOException {
+
+ final Path testDir = new Path(
+ PathUtils.getTestPath(getClass()),
+ GenericTestUtils.getMethodName());
+ assertTrue(dfs.mkdirs(testDir));
+
+ final long storageSpaceQuota = BLOCKSIZE * 2;
+
+ /* set policy to HOT */
+ dfs.setStoragePolicy(testDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);
+
+ /* set space quota */
+ dfs.setQuota(testDir, HdfsConstants.QUOTA_DONT_SET, storageSpaceQuota);
+
+ /* init vars */
+ Path createdFile = null;
+ final long fileLen = BLOCKSIZE;
+
+ try {
+ /* create one file with 3 replicas */
+ createdFile = new Path(testDir, "file1.data");
+ DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16, fileLen,
+ BLOCKSIZE, REPLICATION, seed);
+ fail("should fail on DSQuotaExceededException");
+ } catch (DSQuotaExceededException e) {
+ LOG.info("Got expected exception ", e);
+ assertThat(e.toString(),
+ is(allOf(containsString("DiskSpace quota"),
+ containsString(testDir.toString()))));
+ }
+
+ /* try creating file again with 2 replicas */
+ createdFile = new Path(testDir, "file2.data");
+ DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16, fileLen, BLOCKSIZE,
+ (short) 2, seed);
+ assertTrue(dfs.exists(createdFile));
+ assertTrue(dfs.isFile(createdFile));
+ }
+
+ /**
+ * Tests if clearing quota per heterogeneous storage doesn't result in
+ * clearing quota for another storage.
+ *
+ * @throws IOException
+ */
+ @Test(timeout = 30000)
+ public void testStorageSpaceQuotaPerQuotaClear() throws IOException {
+ final Path testDir = new Path(
+ PathUtils.getTestPath(getClass()),
+ GenericTestUtils.getMethodName());
+ assertTrue(dfs.mkdirs(testDir));
+
+ final long diskSpaceQuota = BLOCKSIZE * 1;
+ final long ssdSpaceQuota = BLOCKSIZE * 2;
+
+ /* set space quota */
+ dfs.setQuotaByStorageType(testDir, StorageType.DISK, diskSpaceQuota);
+ dfs.setQuotaByStorageType(testDir, StorageType.SSD, ssdSpaceQuota);
+
+ final INode testDirNode = fsdir.getINode4Write(testDir.toString());
+ assertTrue(testDirNode.isDirectory());
+ assertTrue(testDirNode.isQuotaSet());
+
+ /* verify space quota by storage type */
+ assertEquals(diskSpaceQuota,
+ testDirNode.asDirectory().getDirectoryWithQuotaFeature().getQuota()
+ .getTypeSpace(StorageType.DISK));
+ assertEquals(ssdSpaceQuota,
+ testDirNode.asDirectory().getDirectoryWithQuotaFeature().getQuota()
+ .getTypeSpace(StorageType.SSD));
+
+ /* clear DISK space quota */
+ dfs.setQuotaByStorageType(
+ testDir,
+ StorageType.DISK,
+ HdfsConstants.QUOTA_RESET);
+
+ /* verify space quota by storage type after clearing DISK's */
+ assertEquals(-1,
+ testDirNode.asDirectory().getDirectoryWithQuotaFeature().getQuota()
+ .getTypeSpace(StorageType.DISK));
+ assertEquals(ssdSpaceQuota,
+ testDirNode.asDirectory().getDirectoryWithQuotaFeature().getQuota()
+ .getTypeSpace(StorageType.SSD));
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HDFS-10909. De-duplicate code in
ErasureCodingWorker#initializeStripedReadThreadPool and
DFSClient#initThreadsNumForStripedReads. (Manoj Govindassamy via lei)
Posted by fa...@apache.org.
HDFS-10909. De-duplicate code in ErasureCodingWorker#initializeStripedReadThreadPool and DFSClient#initThreadsNumForStripedReads. (Manoj Govindassamy via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5920619
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5920619
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5920619
Branch: refs/heads/HADOOP-13345
Commit: b59206190e6f773fc223bcb81774a09715551367
Parents: 9b0c17f
Author: Lei Xu <le...@apache.org>
Authored: Wed Nov 2 16:45:25 2016 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Wed Nov 2 16:45:25 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 30 ++----------
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 50 ++++++++++++++++++++
.../erasurecode/ErasureCodingWorker.java | 23 +++------
3 files changed, 61 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5920619/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 93c0ff0..0128b07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2800,37 +2800,17 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
/**
* Create thread pool for parallel reading in striped layout,
* STRIPED_READ_THREAD_POOL, if it does not already exist.
- * @param num Number of threads for striped reads thread pool.
+ * @param numThreads Number of threads for striped reads thread pool.
*/
- private void initThreadsNumForStripedReads(int num) {
- assert num > 0;
+ private void initThreadsNumForStripedReads(int numThreads) {
+ assert numThreads > 0;
if (STRIPED_READ_THREAD_POOL != null) {
return;
}
synchronized (DFSClient.class) {
if (STRIPED_READ_THREAD_POOL == null) {
- STRIPED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num, 60,
- TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
- new Daemon.DaemonFactory() {
- private final AtomicInteger threadIndex = new AtomicInteger(0);
-
- @Override
- public Thread newThread(Runnable r) {
- Thread t = super.newThread(r);
- t.setName("stripedRead-" + threadIndex.getAndIncrement());
- return t;
- }
- },
- new ThreadPoolExecutor.CallerRunsPolicy() {
- @Override
- public void rejectedExecution(Runnable runnable,
- ThreadPoolExecutor e) {
- LOG.info("Execution for striped reading rejected, "
- + "Executing in current thread");
- // will run in the current thread
- super.rejectedExecution(runnable, e);
- }
- });
+ STRIPED_READ_THREAD_POOL = DFSUtilClient.getThreadPoolExecutor(1,
+ numThreads, 60, "StripedRead-", true);
STRIPED_READ_THREAD_POOL.allowCoreThreadTimeOut(true);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5920619/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index b93632a..d267530 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import com.google.common.primitives.SignedBytes;
import org.apache.hadoop.conf.Configuration;
@@ -51,6 +52,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.KMSUtil;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
@@ -81,6 +83,10 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY;
@@ -776,4 +782,48 @@ public class DFSUtilClient {
DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY,
DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT);
}
+
+ /**
+ * Utility to create a {@link ThreadPoolExecutor}.
+ *
+ * @param corePoolSize - min threads in the pool, even if idle
+ * @param maxPoolSize - max threads in the pool
+ * @param keepAliveTimeSecs - max seconds beyond which excess idle threads
+ * will be terminated
+ * @param threadNamePrefix - name prefix for the pool threads
+ * @param runRejectedExec - when true, rejected tasks from
+ * ThreadPoolExecutor are run in the context of calling thread
+ * @return ThreadPoolExecutor
+ */
+ public static ThreadPoolExecutor getThreadPoolExecutor(int corePoolSize,
+ int maxPoolSize, long keepAliveTimeSecs, String threadNamePrefix,
+ boolean runRejectedExec) {
+ Preconditions.checkArgument(corePoolSize > 0);
+ ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(corePoolSize,
+ maxPoolSize, keepAliveTimeSecs, TimeUnit.SECONDS,
+ new SynchronousQueue<Runnable>(), new Daemon.DaemonFactory() {
+ private final AtomicInteger threadIndex = new AtomicInteger(0);
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = super.newThread(r);
+ t.setName(threadNamePrefix + threadIndex.getAndIncrement());
+ return t;
+ }
+ });
+ if (runRejectedExec) {
+ threadPoolExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor
+ .CallerRunsPolicy() {
+ @Override
+ public void rejectedExecution(Runnable runnable,
+ ThreadPoolExecutor e) {
+ LOG.info(threadNamePrefix + " task is rejected by " +
+ "ThreadPoolExecutor. Executing it in current thread.");
+ // will run in the current thread
+ super.rejectedExecution(runnable, e);
+ }
+ });
+ }
+ return threadPoolExecutor;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5920619/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index aacbb2d..1492e5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -20,13 +20,13 @@ package org.apache.hadoop.hdfs.server.datanode.erasurecode;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo;
import org.apache.hadoop.util.Daemon;
import org.slf4j.Logger;
import java.util.Collection;
-import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -89,22 +89,11 @@ public final class ErasureCodingWorker {
stripedReadPool.allowCoreThreadTimeOut(true);
}
- private void initializeStripedBlkReconstructionThreadPool(int num) {
- LOG.debug("Using striped block reconstruction; pool threads={}", num);
- stripedReconstructionPool = new ThreadPoolExecutor(2, num, 60,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(),
- new Daemon.DaemonFactory() {
- private final AtomicInteger threadIdx = new AtomicInteger(0);
-
- @Override
- public Thread newThread(Runnable r) {
- Thread t = super.newThread(r);
- t.setName("stripedBlockReconstruction-"
- + threadIdx.getAndIncrement());
- return t;
- }
- });
+ private void initializeStripedBlkReconstructionThreadPool(int numThreads) {
+ LOG.debug("Using striped block reconstruction; pool threads={}",
+ numThreads);
+ stripedReconstructionPool = DFSUtilClient.getThreadPoolExecutor(2,
+ numThreads, 60, "StripedBlockReconstruction-", false);
stripedReconstructionPool.allowCoreThreadTimeOut(true);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDFS-11098. Datanode in tests
cannot start in Windows after HDFS-10638 (Contributed by Vinayakumar B)
Posted by fa...@apache.org.
HDFS-11098. Datanode in tests cannot start in Windows after HDFS-10638 (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69dd5fa2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69dd5fa2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69dd5fa2
Branch: refs/heads/HADOOP-13345
Commit: 69dd5fa2d43eefeec112f36b91a13513ac21a763
Parents: 5cad93d
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri Nov 4 08:31:43 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Fri Nov 4 08:31:43 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/datanode/StorageLocation.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69dd5fa2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java
index a040395..1bd3782 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java
@@ -62,8 +62,11 @@ public class StorageLocation implements Comparable<StorageLocation>{
// make sure all URIs that point to a file have the same scheme
try {
File uriFile = new File(uri.getPath());
- String absPath = uriFile.getAbsolutePath();
- uri = new URI("file", null, absPath, uri.getQuery(), uri.getFragment());
+ String uriStr = uriFile.toURI().normalize().toString();
+ if (uriStr.endsWith("/")) {
+ uriStr = uriStr.substring(0, uriStr.length() - 1);
+ }
+ uri = new URI(uriStr);
} catch (URISyntaxException e) {
throw new IllegalArgumentException(
"URI: " + uri + " is not in the expected format");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HADOOP-13448 S3Guard: Define
MetadataStore interface.
Posted by fa...@apache.org.
HADOOP-13448 S3Guard: Define MetadataStore interface.
Contributed by Chris Nauroth and Aaron Fabbri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a795393a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a795393a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a795393a
Branch: refs/heads/HADOOP-13345
Commit: a795393a998b2dcc3f75f181ffc8919700c74f4c
Parents: d8bab3d
Author: Aaron Fabbri <fa...@apache.org>
Authored: Fri Sep 9 12:14:55 2016 -0700
Committer: Aaron Fabbri <fa...@apache.org>
Committed: Fri Nov 4 20:30:59 2016 -0700
----------------------------------------------------------------------
.../fs/s3a/s3guard/DirListingMetadata.java | 159 ++++++++++
.../hadoop/fs/s3a/s3guard/MetadataStore.java | 127 ++++++++
.../hadoop/fs/s3a/s3guard/PathMetadata.java | 72 +++++
.../hadoop/fs/s3a/s3guard/package-info.java | 30 ++
.../fs/s3a/s3guard/TestDirListingMetadata.java | 290 +++++++++++++++++++
5 files changed, 678 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a795393a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
new file mode 100644
index 0000000..acb3e51
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * {@code DirListingMetadata} models a directory listing stored in a
+ * {@link MetadataStore}. Instances of this class are mutable and thread-safe.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DirListingMetadata {
+
+ private final Path path;
+
+ /** Using a map for fast find / remove with large directories. */
+ private Map<Path, PathMetadata> listMap = new ConcurrentHashMap<>();
+
+ private boolean isAuthoritative;
+
+ /**
+ * Create a directory listing metadata container.
+ *
+ * @param path Path of the directory.
+ * @param listing Entries in the directory.
+ * @param isAuthoritative true iff listing is the full contents of the
+ * directory, and the calling client reports that this may be cached as
+ * the full and authoritative listing of all files in the directory.
+ */
+ public DirListingMetadata(Path path, Collection<PathMetadata> listing,
+ boolean isAuthoritative) {
+ Preconditions.checkNotNull(path, "path must be non-null");
+ this.path = path;
+ if (listing != null) {
+ for (PathMetadata entry : listing) {
+ listMap.put(entry.getFileStatus().getPath(), entry);
+ }
+ }
+ this.isAuthoritative = isAuthoritative;
+ }
+
+ /**
+ * @return {@code Path} of the directory that contains this listing.
+ */
+ public Path getPath() {
+ return path;
+ }
+
+ /**
+ * @return entries in the directory
+ */
+ public Collection<PathMetadata> getListing() {
+ return Collections.unmodifiableCollection(listMap.values());
+ }
+
+ /**
+ * @return true iff this directory listing is full and authoritative within
+ * the scope of the {@code MetadataStore} that returned it.
+ */
+ public boolean isAuthoritative() {
+ return isAuthoritative;
+ }
+
+ /**
+ * Marks this directory listing as full and authoritative.
+ */
+ public void setAuthoritative() {
+ isAuthoritative = true;
+ }
+
+ /**
+ * Lookup entry within this directory listing. This may return null if the
+ * {@code MetadataStore} only tracks a partial set of the directory entries.
+ * In the case where {@link #isAuthoritative()} is true, however, this
+ * function returns null iff the directory is known not to contain the listing
+ * at given path (within the scope of the {@code MetadataStore} that returned
+ * it).
+ *
+ * @param childPath path of entry to look for.
+ * @return entry, or null if it is not present or not being tracked.
+ */
+ public PathMetadata get(Path childPath) {
+ checkChildPath(childPath);
+ return listMap.get(childPath);
+ }
+
+ /**
+ * Remove entry from this directory.
+ *
+ * @param childPath path of entry to remove.
+ */
+ public void remove(Path childPath) {
+ checkChildPath(childPath);
+ listMap.remove(childPath);
+ }
+
+ /**
+ * Add an entry to the directory listing. If this listing already contains a
+ * {@code FileStatus} with the same path, it will be replaced.
+ *
+ * @param childFileStatus entry to add to this directory listing.
+ */
+ public void put(FileStatus childFileStatus) {
+ Preconditions.checkNotNull(childFileStatus,
+ "childFileStatus must be non-null");
+ Path childPath = childFileStatus.getPath();
+ checkChildPath(childPath);
+ listMap.put(childPath, new PathMetadata(childFileStatus));
+ }
+
+ @Override
+ public String toString() {
+ return "DirListingMetadata{" +
+ "path=" + path +
+ ", listMap=" + listMap +
+ ", isAuthoritative=" + isAuthoritative +
+ '}';
+ }
+
+ /**
+ * Performs pre-condition checks on child path arguments.
+ *
+ * @param childPath path to check.
+ */
+ private void checkChildPath(Path childPath) {
+ Preconditions.checkNotNull(childPath, "childPath must be non-null");
+ Preconditions.checkArgument(!childPath.isRoot(),
+ "childPath cannot be the root path");
+ Preconditions.checkArgument(childPath.getParent().equals(path),
+ "childPath must be a child of path");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a795393a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
new file mode 100644
index 0000000..c034b9a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * {@code MetadataStore} defines the set of operations that any metadata store
+ * implementation must provide.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface MetadataStore extends Closeable {
+
+ /**
+ * Performs one-time initialization of the metadata store.
+ *
+ * @param conf {@code Configuration}
+ * @throws IOException if there is an error
+ */
+ void initialize(Configuration conf) throws IOException;
+
+ /**
+ * Deletes exactly one path.
+ *
+ * @param path the path to delete
+ * @throws IOException if there is an error
+ */
+ void delete(Path path) throws IOException;
+
+ /**
+ * Deletes the entire sub-tree rooted at the given path.
+ *
+ * In addition to affecting future calls to {@link #get(Path)},
+ * implementations must also update any stored {@code DirListingMetadata}
+ * objects which track the parent of this file.
+ *
+ * @param path the root of the sub-tree to delete
+ * @throws IOException if there is an error
+ */
+ void deleteSubtree(Path path) throws IOException;
+
+ /**
+ * Gets metadata for a path.
+ *
+ * @param path the path to get
+ * @return metadata for {@code path}, {@code null} if not found
+ * @throws IOException if there is an error
+ */
+ PathMetadata get(Path path) throws IOException;
+
+ /**
+ * Lists metadata for all direct children of a path.
+ *
+ * @param path the path to list
+ * @return metadata for all direct children of {@code path} which are being
+ * tracked by the MetadataStore, or {@code null} if the path was not found
+ * in the MetadataStore.
+ * @throws IOException if there is an error
+ */
+ DirListingMetadata listChildren(Path path) throws IOException;
+
+ /**
+ * Moves metadata from {@code src} to {@code dst}, including all descendants
+ * recursively.
+ *
+ * @param src the source path
+ * @param dst the destination path
+ * @throws IOException if there is an error
+ */
+ void move(Path src, Path dst) throws IOException;
+
+ /**
+ * Saves metadata for exactly one path. For a deeply nested path, this method
+ * will not automatically create the full ancestry. Callers need to ensure
+ * saving the full path ancestry.
+ *
+ * Implementations must update any {@code DirListingMetadata} objects which
+ * track the parent of this file.
+ *
+ * @param meta the metadata to save
+ * @throws IOException if there is an error
+ */
+ void put(PathMetadata meta) throws IOException;
+
+ /**
+ * Save directory listing metadata. Callers may save a partial directory
+ * listing for a given path, or may store a complete and authoritative copy
+ * of the directory listing. {@code MetadataStore} implementations may
+ * subsequently keep track of all modifications to the directory contents at
+ * this path, and return authoritative results from subsequent calls to
+ * {@link #listChildren(Path)}. See {@link DirListingMetadata}.
+ *
+ * Any authoritative results returned are only authoritative for the scope
+ * of the {@code MetadataStore}: A per-process {@code MetadataStore}, for
+ * example, would only show results visible to that process, potentially
+ * missing metadata updates (create, delete) made to the same path by
+ * another process.
+ *
+ * @param meta Directory listing metadata.
+ * @throws IOException if there is an error
+ */
+ void put(DirListingMetadata meta) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a795393a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
new file mode 100644
index 0000000..37fbfec
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+
+/**
+ * {@code PathMetadata} models path metadata stored in the
+ * {@link MetadataStore}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+class PathMetadata {
+
+ private final FileStatus fileStatus;
+
+ /**
+ * Creates a new {@code PathMetadata} containing given {@code FileStatus}.
+ *
+ * @param fileStatus file status
+ */
+ public PathMetadata(FileStatus fileStatus) {
+ Preconditions.checkNotNull(fileStatus, "fileStatus must be non-null");
+ this.fileStatus = fileStatus;
+ }
+
+ /**
+ * @return {@code FileStatus} contained in this {@code PathMetadata}.
+ */
+ public final FileStatus getFileStatus() {
+ return fileStatus;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof PathMetadata)) {
+ return false;
+ }
+ return this.fileStatus.equals(((PathMetadata)o).fileStatus);
+ }
+
+ @Override
+ public int hashCode() {
+ return fileStatus.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "PathMetadata{" +
+ "fileStatus=" + fileStatus +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a795393a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/package-info.java
new file mode 100644
index 0000000..34f36b8
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/package-info.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains classes related to S3Guard: a feature of S3A to mask
+ * the eventual consistency behavior of S3 and optimize access patterns by
+ * coordinating with a strongly consistent external store for file system
+ * metadata.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a795393a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
new file mode 100644
index 0000000..99db001
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import static org.hamcrest.CoreMatchers.*;
+import static org.junit.Assert.*;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileStatus;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Unit tests of {@link DirListingMetadata}.
+ */
+public class TestDirListingMetadata {
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testNullPath() {
+ exception.expect(NullPointerException.class);
+ exception.expectMessage(notNullValue(String.class));
+ new DirListingMetadata(null, null, false);
+ }
+
+ @Test
+ public void testNullListing() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ assertEquals(path, meta.getPath());
+ assertNotNull(meta.getListing());
+ assertTrue(meta.getListing().isEmpty());
+ assertFalse(meta.isAuthoritative());
+ }
+
+ @Test
+ public void testEmptyListing() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path,
+ Collections.emptyList(), false);
+ assertEquals(path, meta.getPath());
+ assertNotNull(meta.getListing());
+ assertTrue(meta.getListing().isEmpty());
+ assertFalse(meta.isAuthoritative());
+ }
+
+ @Test
+ public void testListing() {
+ Path path = new Path("/path");
+ PathMetadata pathMeta1 = new PathMetadata(
+ new S3AFileStatus(true, true, new Path(path, "dir1")));
+ PathMetadata pathMeta2 = new PathMetadata(
+ new S3AFileStatus(true, false, new Path(path, "dir2")));
+ PathMetadata pathMeta3 = new PathMetadata(
+ new S3AFileStatus(123, 456, new Path(path, "file1"), 789));
+ List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
+ DirListingMetadata meta = new DirListingMetadata(path, listing, false);
+ assertEquals(path, meta.getPath());
+ assertNotNull(meta.getListing());
+ assertFalse(meta.getListing().isEmpty());
+ assertTrue(meta.getListing().contains(pathMeta1));
+ assertTrue(meta.getListing().contains(pathMeta2));
+ assertTrue(meta.getListing().contains(pathMeta3));
+ assertFalse(meta.isAuthoritative());
+ }
+
+ @Test
+ public void testListingUnmodifiable() {
+ Path path = new Path("/path");
+ PathMetadata pathMeta1 = new PathMetadata(
+ new S3AFileStatus(true, true, new Path(path, "dir1")));
+ PathMetadata pathMeta2 = new PathMetadata(
+ new S3AFileStatus(true, false, new Path(path, "dir2")));
+ PathMetadata pathMeta3 = new PathMetadata(
+ new S3AFileStatus(123, 456, new Path(path, "file1"), 789));
+ List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
+ DirListingMetadata meta = new DirListingMetadata(path, listing, false);
+ assertNotNull(meta.getListing());
+ exception.expect(UnsupportedOperationException.class);
+ meta.getListing().clear();
+ }
+
+ @Test
+ public void testAuthoritative() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, true);
+ assertEquals(path, meta.getPath());
+ assertNotNull(meta.getListing());
+ assertTrue(meta.getListing().isEmpty());
+ assertTrue(meta.isAuthoritative());
+ }
+
+ @Test
+ public void testSetAuthoritative() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ assertEquals(path, meta.getPath());
+ assertNotNull(meta.getListing());
+ assertTrue(meta.getListing().isEmpty());
+ assertFalse(meta.isAuthoritative());
+ meta.setAuthoritative();
+ assertTrue(meta.isAuthoritative());
+ }
+
+ @Test
+ public void testGet() {
+ Path path = new Path("/path");
+ PathMetadata pathMeta1 = new PathMetadata(
+ new S3AFileStatus(true, true, new Path(path, "dir1")));
+ PathMetadata pathMeta2 = new PathMetadata(
+ new S3AFileStatus(true, false, new Path(path, "dir2")));
+ PathMetadata pathMeta3 = new PathMetadata(
+ new S3AFileStatus(123, 456, new Path(path, "file1"), 789));
+ List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
+ DirListingMetadata meta = new DirListingMetadata(path, listing, false);
+ assertEquals(path, meta.getPath());
+ assertNotNull(meta.getListing());
+ assertFalse(meta.getListing().isEmpty());
+ assertTrue(meta.getListing().contains(pathMeta1));
+ assertTrue(meta.getListing().contains(pathMeta2));
+ assertTrue(meta.getListing().contains(pathMeta3));
+ assertFalse(meta.isAuthoritative());
+ assertEquals(pathMeta1, meta.get(pathMeta1.getFileStatus().getPath()));
+ assertEquals(pathMeta2, meta.get(pathMeta2.getFileStatus().getPath()));
+ assertEquals(pathMeta3, meta.get(pathMeta3.getFileStatus().getPath()));
+ assertNull(meta.get(new Path(path, "notfound")));
+ }
+
+ @Test
+ public void testGetNull() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ exception.expect(NullPointerException.class);
+ exception.expectMessage(notNullValue(String.class));
+ meta.get(null);
+ }
+
+ @Test
+ public void testGetRoot() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage(notNullValue(String.class));
+ meta.get(new Path("/"));
+ }
+
+ @Test
+ public void testGetNotChild() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage(notNullValue(String.class));
+ meta.get(new Path("/different/ancestor"));
+ }
+
+ @Test
+ public void testPut() {
+ Path path = new Path("/path");
+ PathMetadata pathMeta1 = new PathMetadata(
+ new S3AFileStatus(true, true, new Path(path, "dir1")));
+ PathMetadata pathMeta2 = new PathMetadata(
+ new S3AFileStatus(true, false, new Path(path, "dir2")));
+ PathMetadata pathMeta3 = new PathMetadata(
+ new S3AFileStatus(123, 456, new Path(path, "file1"), 789));
+ List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
+ DirListingMetadata meta = new DirListingMetadata(path, listing, false);
+ assertEquals(path, meta.getPath());
+ assertNotNull(meta.getListing());
+ assertFalse(meta.getListing().isEmpty());
+ assertTrue(meta.getListing().contains(pathMeta1));
+ assertTrue(meta.getListing().contains(pathMeta2));
+ assertTrue(meta.getListing().contains(pathMeta3));
+ assertFalse(meta.isAuthoritative());
+ PathMetadata pathMeta4 = new PathMetadata(
+ new S3AFileStatus(true, true, new Path(path, "dir3")));
+ meta.put(pathMeta4.getFileStatus());
+ assertTrue(meta.getListing().contains(pathMeta4));
+ assertEquals(pathMeta4, meta.get(pathMeta4.getFileStatus().getPath()));
+ }
+
+ @Test
+ public void testPutNull() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ exception.expect(NullPointerException.class);
+ exception.expectMessage(notNullValue(String.class));
+ meta.put(null);
+ }
+
+ @Test
+ public void testPutNullPath() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ exception.expect(NullPointerException.class);
+ exception.expectMessage(notNullValue(String.class));
+ meta.put(new S3AFileStatus(true, true, null));
+ }
+
+ @Test
+ public void testPutRoot() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage(notNullValue(String.class));
+ meta.put(new S3AFileStatus(true, true, new Path("/")));
+ }
+
+ @Test
+ public void testPutNotChild() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage(notNullValue(String.class));
+ meta.put(new S3AFileStatus(true, true, new Path("/different/ancestor")));
+ }
+
+ @Test
+ public void testRemove() {
+ Path path = new Path("/path");
+ PathMetadata pathMeta1 = new PathMetadata(
+ new S3AFileStatus(true, true, new Path(path, "dir1")));
+ PathMetadata pathMeta2 = new PathMetadata(
+ new S3AFileStatus(true, false, new Path(path, "dir2")));
+ PathMetadata pathMeta3 = new PathMetadata(
+ new S3AFileStatus(123, 456, new Path(path, "file1"), 789));
+ List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
+ DirListingMetadata meta = new DirListingMetadata(path, listing, false);
+ assertEquals(path, meta.getPath());
+ assertNotNull(meta.getListing());
+ assertFalse(meta.getListing().isEmpty());
+ assertTrue(meta.getListing().contains(pathMeta1));
+ assertTrue(meta.getListing().contains(pathMeta2));
+ assertTrue(meta.getListing().contains(pathMeta3));
+ assertFalse(meta.isAuthoritative());
+ meta.remove(pathMeta1.getFileStatus().getPath());
+ assertFalse(meta.getListing().contains(pathMeta1));
+ assertNull(meta.get(pathMeta1.getFileStatus().getPath()));
+ }
+
+ @Test
+ public void testRemoveNull() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ exception.expect(NullPointerException.class);
+ exception.expectMessage(notNullValue(String.class));
+ meta.remove(null);
+ }
+
+ @Test
+ public void testRemoveRoot() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage(notNullValue(String.class));
+ meta.remove(new Path("/"));
+ }
+
+ @Test
+ public void testRemoveNotChild() {
+ Path path = new Path("/path");
+ DirListingMetadata meta = new DirListingMetadata(path, null, false);
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage(notNullValue(String.class));
+ meta.remove(new Path("/different/ancestor"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-5805. Add isDebugEnabled
check for debug logs in nodemanager (Bibin A Chundatt via Varun Saxena)
Posted by fa...@apache.org.
YARN-5805. Add isDebugEnabled check for debug logs in nodemanager (Bibin A Chundatt via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/310aa468
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/310aa468
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/310aa468
Branch: refs/heads/HADOOP-13345
Commit: 310aa4686eb697f0700a71d33106514c404dab1b
Parents: 7ba74be
Author: Varun Saxena <va...@apache.org>
Authored: Tue Nov 1 14:51:17 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Tue Nov 1 14:51:31 2016 +0530
----------------------------------------------------------------------
.../nodemanager/DefaultContainerExecutor.java | 7 ++--
.../server/nodemanager/DeletionService.java | 13 +++++--
.../nodemanager/amrmproxy/AMRMProxyService.java | 4 ++-
.../application/ApplicationImpl.java | 6 ++--
.../container/ContainerImpl.java | 5 +--
.../launcher/ContainerLaunch.java | 36 +++++++++++---------
.../localizer/LocalizedResource.java | 5 +--
.../security/LocalizerTokenSelector.java | 4 ++-
.../monitor/ContainersMonitorImpl.java | 13 +++----
.../security/NMTokenSecretManagerInNM.java | 16 ++++++---
.../nodemanager/util/ProcessIdFileReader.java | 12 ++++---
11 files changed, 75 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 4947820..420035d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -556,9 +556,10 @@ public class DefaultContainerExecutor extends ContainerExecutor {
String user = ctx.getUser();
String pid = ctx.getPid();
Signal signal = ctx.getSignal();
-
- LOG.debug("Sending signal " + signal.getValue() + " to pid " + pid
- + " as user " + user);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sending signal " + signal.getValue() + " to pid " + pid
+ + " as user " + user);
+ }
if (!containerIsAlive(pid)) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
index db834b2..aac0af9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
@@ -239,7 +239,9 @@ public class DeletionService extends AbstractService {
boolean error = false;
if (null == user) {
if (baseDirs == null || baseDirs.size() == 0) {
- LOG.debug("NM deleting absolute path : " + subDir);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("NM deleting absolute path : " + subDir);
+ }
try {
lfs.delete(subDir, true);
} catch (IOException e) {
@@ -249,7 +251,9 @@ public class DeletionService extends AbstractService {
} else {
for (Path baseDir : baseDirs) {
Path del = subDir == null? baseDir : new Path(baseDir, subDir);
- LOG.debug("NM deleting path : " + del);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("NM deleting path : " + del);
+ }
try {
lfs.delete(del, true);
} catch (IOException e) {
@@ -260,7 +264,10 @@ public class DeletionService extends AbstractService {
}
} else {
try {
- LOG.debug("Deleting path: [" + subDir + "] as user: [" + user + "]");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Deleting path: [" + subDir + "] as user: [" + user + "]");
+ }
if (baseDirs == null || baseDirs.size() == 0) {
delService.exec.deleteAsUser(new DeletionAsUserContext.Builder()
.setUser(user)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
index 511db16..79882aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
@@ -549,7 +549,9 @@ public class AMRMProxyService extends AbstractService implements
AMRMProxyService.this.stopApplication(event.getApplicationID());
break;
default:
- LOG.debug("AMRMProxy is ignoring event: " + event.getType());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AMRMProxy is ignoring event: " + event.getType());
+ }
break;
}
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 531693e..b9197c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -591,8 +591,10 @@ public class ApplicationImpl implements Application {
try {
ApplicationId applicationID = event.getApplicationID();
- LOG.debug("Processing " + applicationID + " of type " + event.getType());
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Processing " + applicationID + " of type " + event.getType());
+ }
ApplicationState oldState = stateMachine.getCurrentState();
ApplicationState newState = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index e6b9d9f..307cf5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1641,8 +1641,9 @@ public class ContainerImpl implements Container {
this.writeLock.lock();
ContainerId containerID = event.getContainerID();
- LOG.debug("Processing " + containerID + " of type " + event.getType());
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing " + containerID + " of type " + event.getType());
+ }
ContainerState oldState = stateMachine.getCurrentState();
ContainerState newState = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index d8239ef..d774030 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -571,8 +571,9 @@ public class ContainerLaunch implements Callable<Integer> {
+ " No cleanup needed to be done");
return;
}
-
- LOG.debug("Marking container " + containerIdStr + " as inactive");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Marking container " + containerIdStr + " as inactive");
+ }
// this should ensure that if the container process has not launched
// by this time, it will never be launched
exec.deactivateContainer(containerId);
@@ -596,10 +597,10 @@ public class ContainerLaunch implements Callable<Integer> {
// kill process
if (processId != null) {
String user = container.getUser();
- LOG.debug("Sending signal to pid " + processId
- + " as user " + user
- + " for container " + containerIdStr);
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sending signal to pid " + processId + " as user " + user
+ + " for container " + containerIdStr);
+ }
final Signal signal = sleepDelayBeforeSigKill > 0
? Signal.TERM
: Signal.KILL;
@@ -611,12 +612,11 @@ public class ContainerLaunch implements Callable<Integer> {
.setPid(processId)
.setSignal(signal)
.build());
-
- LOG.debug("Sent signal " + signal + " to pid " + processId
- + " as user " + user
- + " for container " + containerIdStr
- + ", result=" + (result? "success" : "failed"));
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sent signal " + signal + " to pid " + processId
+ + " as user " + user + " for container " + containerIdStr
+ + ", result=" + (result ? "success" : "failed"));
+ }
if (sleepDelayBeforeSigKill > 0) {
new DelayedProcessKiller(container, user,
processId, sleepDelayBeforeSigKill, Signal.KILL, exec).start();
@@ -744,8 +744,10 @@ public class ContainerLaunch implements Callable<Integer> {
String containerIdStr =
container.getContainerId().toString();
String processId = null;
- LOG.debug("Accessing pid for container " + containerIdStr
- + " from pid file " + pidFilePath);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Accessing pid for container " + containerIdStr
+ + " from pid file " + pidFilePath);
+ }
int sleepCounter = 0;
final int sleepInterval = 100;
@@ -754,8 +756,10 @@ public class ContainerLaunch implements Callable<Integer> {
while (true) {
processId = ProcessIdFileReader.getProcessId(pidFilePath);
if (processId != null) {
- LOG.debug("Got pid " + processId + " for container "
- + containerIdStr);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Got pid " + processId + " for container " + containerIdStr);
+ }
break;
}
else if ((sleepCounter*sleepInterval) > maxKillWaitTime) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
index 464293a..6d67cac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
@@ -190,8 +190,9 @@ public class LocalizedResource implements EventHandler<ResourceEvent> {
this.writeLock.lock();
Path resourcePath = event.getLocalResourceRequest().getPath();
- LOG.debug("Processing " + resourcePath + " of type " + event.getType());
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing " + resourcePath + " of type " + event.getType());
+ }
ResourceState oldState = this.stateMachine.getCurrentState();
ResourceState newState = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSelector.java
index 56c6da3..7c4f982 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSelector.java
@@ -41,7 +41,9 @@ public class LocalizerTokenSelector implements
LOG.debug("Using localizerTokenSelector.");
for (Token<? extends TokenIdentifier> token : tokens) {
- LOG.debug("Token of kind " + token.getKind() + " is found");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Token of kind " + token.getKind() + " is found");
+ }
if (LocalizerTokenIdentifier.KIND.equals(token.getKind())) {
return (Token<LocalizerTokenIdentifier>) token;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index cd9d6af..432cd67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -446,9 +446,10 @@ public class ContainersMonitorImpl extends AbstractService implements
if (pId == null || !isResourceCalculatorAvailable()) {
continue; // processTree cannot be tracked
}
-
- LOG.debug("Constructing ProcessTree for : PID = " + pId
- + " ContainerId = " + containerId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Constructing ProcessTree for : PID = " + pId
+ + " ContainerId = " + containerId);
+ }
ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree();
pTree.updateProcessTree(); // update process-tree
long currentVmemUsage = pTree.getVirtualMemorySize();
@@ -527,9 +528,9 @@ public class ContainersMonitorImpl extends AbstractService implements
if (pId != null) {
// pId will be null, either if the container is not spawned yet
// or if the container's pid is removed from ContainerExecutor
- LOG.debug("Tracking ProcessTree " + pId
- + " for the first time");
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Tracking ProcessTree " + pId + " for the first time");
+ }
ResourceCalculatorProcessTree pt =
ResourceCalculatorProcessTree.
getResourceCalculatorProcessTree(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
index 86cce35..4ed6118 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
@@ -191,8 +191,10 @@ public class NMTokenSecretManagerInNM extends BaseNMTokenSecretManager {
public synchronized void appFinished(ApplicationId appId) {
List<ApplicationAttemptId> appAttemptList = appToAppAttemptMap.get(appId);
if (appAttemptList != null) {
- LOG.debug("Removing application attempts NMToken keys for application "
- + appId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removing application attempts NMToken keys for application "
+ + appId);
+ }
for (ApplicationAttemptId appAttemptId : appAttemptList) {
removeAppAttemptKey(appAttemptId);
}
@@ -226,8 +228,10 @@ public class NMTokenSecretManagerInNM extends BaseNMTokenSecretManager {
if (oldKey == null
|| oldKey.getMasterKey().getKeyId() != identifier.getKeyId()) {
// Update key only if it is modified.
- LOG.debug("NMToken key updated for application attempt : "
- + identifier.getApplicationAttemptId().toString());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("NMToken key updated for application attempt : "
+ + identifier.getApplicationAttemptId().toString());
+ }
if (identifier.getKeyId() == currentMasterKey.getMasterKey()
.getKeyId()) {
updateAppAttemptKey(appAttemptId, currentMasterKey);
@@ -243,7 +247,9 @@ public class NMTokenSecretManagerInNM extends BaseNMTokenSecretManager {
}
public synchronized void setNodeId(NodeId nodeId) {
- LOG.debug("updating nodeId : " + nodeId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("updating nodeId : " + nodeId);
+ }
this.nodeId = nodeId;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310aa468/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
index 5a7dba7..41d299e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
@@ -48,8 +48,9 @@ public class ProcessIdFileReader {
if (path == null) {
throw new IOException("Trying to access process id from a null path");
}
-
- LOG.debug("Accessing pid from pid file " + path);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Accessing pid from pid file " + path);
+ }
String processId = null;
BufferedReader bufReader = null;
@@ -97,9 +98,10 @@ public class ProcessIdFileReader {
bufReader.close();
}
}
- LOG.debug("Got pid "
- + (processId != null? processId : "null")
- + " from path " + path);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Got pid " + (processId != null ? processId : "null")
+ + " from path " + path);
+ }
return processId;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: MAPREDUCE-6796. Remove unused
properties from JTConfig.java. Contributed by Haibo Chen.
Posted by fa...@apache.org.
MAPREDUCE-6796. Remove unused properties from JTConfig.java. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de013274
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de013274
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de013274
Branch: refs/heads/HADOOP-13345
Commit: de0132748da4d5cd4ef65571987b2581ff456084
Parents: abfc15d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Sat Nov 5 03:49:19 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Sat Nov 5 03:49:19 2016 +0900
----------------------------------------------------------------------
.../src/site/markdown/DeprecatedProperties.md | 24 ------
.../mapreduce/server/jobtracker/JTConfig.java | 81 +-------------------
.../hadoop/mapreduce/util/ConfigUtil.java | 53 -------------
.../resources/job_1329348432655_0001_conf.xml | 15 ----
.../src/main/data/2jobs2min-rumen-jh.json | 52 +------------
5 files changed, 8 insertions(+), 217 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de013274/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
index 94ed68e..8de1be2 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
@@ -60,7 +60,6 @@ The following table lists the configuration property names that are deprecated i
| fs.checkpoint.period | dfs.namenode.checkpoint.period |
| fs.default.name | fs.defaultFS |
| hadoop.configured.node.mapping | net.topology.configured.node.mapping |
-| hadoop.job.history.location | mapreduce.jobtracker.jobhistory.location |
| hadoop.native.lib | io.native.lib.available |
| hadoop.net.static.resolutions | mapreduce.tasktracker.net.static.resolutions |
| hadoop.pipes.command-file.keep | mapreduce.pipes.commandfile.preserve |
@@ -102,7 +101,6 @@ The following table lists the configuration property names that are deprecated i
| mapred.cache.localArchives | mapreduce.job.cache.local.archives |
| mapred.cache.localFiles | mapreduce.job.cache.local.files |
| mapred.child.tmp | mapreduce.task.tmp.dir |
-| mapred.cluster.average.blacklist.threshold | mapreduce.jobtracker.blacklist.average.threshold |
| mapred.cluster.map.memory.mb | mapreduce.cluster.mapmemory.mb |
| mapred.cluster.max.map.memory.mb | mapreduce.jobtracker.maxmapmemory.mb |
| mapred.cluster.max.reduce.memory.mb | mapreduce.jobtracker.maxreducememory.mb |
@@ -115,9 +113,6 @@ The following table lists the configuration property names that are deprecated i
| mapred.healthChecker.script.args | mapreduce.tasktracker.healthchecker.script.args |
| mapred.healthChecker.script.path | mapreduce.tasktracker.healthchecker.script.path |
| mapred.healthChecker.script.timeout | mapreduce.tasktracker.healthchecker.script.timeout |
-| mapred.heartbeats.in.second | mapreduce.jobtracker.heartbeats.in.second |
-| mapred.hosts.exclude | mapreduce.jobtracker.hosts.exclude.filename |
-| mapred.hosts | mapreduce.jobtracker.hosts.filename |
| mapred.inmem.merge.threshold | mapreduce.reduce.merge.inmem.threshold |
| mapred.input.dir.formats | mapreduce.input.multipleinputs.dir.formats |
| mapred.input.dir.mappers | mapreduce.input.multipleinputs.dir.mappers |
@@ -127,7 +122,6 @@ The following table lists the configuration property names that are deprecated i
| mapred.job.classpath.archives | mapreduce.job.classpath.archives |
| mapred.job.classpath.files | mapreduce.job.classpath.files |
| mapred.job.id | mapreduce.job.id |
-| mapred.jobinit.threads | mapreduce.jobtracker.jobinit.threads |
| mapred.job.map.memory.mb | mapreduce.map.memory.mb |
| mapred.job.name | mapreduce.job.name |
| mapred.job.priority | mapreduce.job.priority |
@@ -139,23 +133,9 @@ The following table lists the configuration property names that are deprecated i
| mapred.job.reuse.jvm.num.tasks | mapreduce.job.jvm.numtasks |
| mapred.job.shuffle.input.buffer.percent | mapreduce.reduce.shuffle.input.buffer.percent |
| mapred.job.shuffle.merge.percent | mapreduce.reduce.shuffle.merge.percent |
-| mapred.job.tracker.handler.count | mapreduce.jobtracker.handler.count |
-| mapred.job.tracker.history.completed.location | mapreduce.jobtracker.jobhistory.completed.location |
-| mapred.job.tracker.http.address | mapreduce.jobtracker.http.address |
-| mapred.jobtracker.instrumentation | mapreduce.jobtracker.instrumentation |
-| mapred.jobtracker.job.history.block.size | mapreduce.jobtracker.jobhistory.block.size |
-| mapred.job.tracker.jobhistory.lru.cache.size | mapreduce.jobtracker.jobhistory.lru.cache.size |
| mapred.job.tracker | mapreduce.jobtracker.address |
-| mapred.jobtracker.maxtasks.per.job | mapreduce.jobtracker.maxtasks.perjob |
| mapred.job.tracker.persist.jobstatus.active | mapreduce.jobtracker.persist.jobstatus.active |
-| mapred.job.tracker.persist.jobstatus.dir | mapreduce.jobtracker.persist.jobstatus.dir |
-| mapred.job.tracker.persist.jobstatus.hours | mapreduce.jobtracker.persist.jobstatus.hours |
-| mapred.jobtracker.restart.recover | mapreduce.jobtracker.restart.recover |
-| mapred.job.tracker.retiredjobs.cache.size | mapreduce.jobtracker.retiredjobs.cache.size |
| mapred.job.tracker.retire.jobs | mapreduce.jobtracker.retirejobs |
-| mapred.jobtracker.taskalloc.capacitypad | mapreduce.jobtracker.taskscheduler.taskalloc.capacitypad |
-| mapred.jobtracker.taskScheduler | mapreduce.jobtracker.taskscheduler |
-| mapred.jobtracker.taskScheduler.maxRunningTasksPerJob | mapreduce.jobtracker.taskscheduler.maxrunningtasks.perjob |
| mapred.join.expr | mapreduce.join.expr |
| mapred.join.keycomparator | mapreduce.join.keycomparator |
| mapred.lazy.output.format | mapreduce.output.lazyoutputformat.outputformat |
@@ -179,7 +159,6 @@ The following table lists the configuration property names that are deprecated i
| mapred.max.map.failures.percent | mapreduce.map.failures.maxpercent |
| mapred.max.reduce.failures.percent | mapreduce.reduce.failures.maxpercent |
| mapred.max.split.size | mapreduce.input.fileinputformat.split.maxsize |
-| mapred.max.tracker.blacklists | mapreduce.jobtracker.tasktracker.maxblacklists |
| mapred.max.tracker.failures | mapreduce.job.maxtaskfailures.per.tracker |
| mapred.merge.recordsBeforeProgress | mapreduce.task.merge.progress.records |
| mapred.min.split.size | mapreduce.input.fileinputformat.split.minsize |
@@ -232,7 +211,6 @@ The following table lists the configuration property names that are deprecated i
| mapred.tasktracker.dns.interface | mapreduce.tasktracker.dns.interface |
| mapred.tasktracker.dns.nameserver | mapreduce.tasktracker.dns.nameserver |
| mapred.tasktracker.events.batchsize | mapreduce.tasktracker.events.batchsize |
-| mapred.tasktracker.expiry.interval | mapreduce.jobtracker.expire.trackers.interval |
| mapred.task.tracker.http.address | mapreduce.tasktracker.http.address |
| mapred.tasktracker.indexcache.mb | mapreduce.tasktracker.indexcache.mb |
| mapred.tasktracker.instrumentation | mapreduce.tasktracker.instrumentation |
@@ -251,7 +229,6 @@ The following table lists the configuration property names that are deprecated i
| mapreduce.combine.class | mapreduce.job.combine.class |
| mapreduce.inputformat.class | mapreduce.job.inputformat.class |
| mapreduce.job.counters.limit | mapreduce.job.counters.max |
-| mapreduce.jobtracker.permissions.supergroup | mapreduce.cluster.permissions.supergroup |
| mapreduce.map.class | mapreduce.job.map.class |
| mapreduce.outputformat.class | mapreduce.job.outputformat.class |
| mapreduce.partitioner.class | mapreduce.job.partitioner.class |
@@ -277,7 +254,6 @@ The following table lists the configuration property names that are deprecated i
| topology.script.file.name | net.topology.script.file.name |
| topology.script.number.args | net.topology.script.number.args |
| user.name | mapreduce.job.user.name |
-| webinterface.private.actions | mapreduce.jobtracker.webinterface.trusted |
| yarn.app.mapreduce.yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts |
| yarn.client.app-submission.poll-interval | yarn.client.application-client-protocol.poll-timeout-ms |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de013274/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java
index ef05d2a..45c0593 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java
@@ -31,92 +31,19 @@ import org.apache.hadoop.mapreduce.MRConfig;
public interface JTConfig extends MRConfig {
// JobTracker configuration parameters
public static final String JT_IPC_ADDRESS = "mapreduce.jobtracker.address";
- public static final String JT_HTTP_ADDRESS =
- "mapreduce.jobtracker.http.address";
- public static final String JT_IPC_HANDLER_COUNT =
- "mapreduce.jobtracker.handler.count";
- public static final String JT_RESTART_ENABLED =
- "mapreduce.jobtracker.restart.recover";
- public static final String JT_TASK_SCHEDULER =
- "mapreduce.jobtracker.taskscheduler";
- public static final String JT_INSTRUMENTATION =
- "mapreduce.jobtracker.instrumentation";
- public static final String JT_TASKS_PER_JOB =
- "mapreduce.jobtracker.maxtasks.perjob";
- public static final String JT_HEARTBEATS_IN_SECOND =
- "mapreduce.jobtracker.heartbeats.in.second";
- public static final String JT_HEARTBEATS_SCALING_FACTOR =
- "mapreduce.jobtracker.heartbeats.scaling.factor";
- public static final String JT_HEARTBEAT_INTERVAL_MIN =
- "mapreduce.jobtracker.heartbeat.interval.min";
- public static final int JT_HEARTBEAT_INTERVAL_MIN_DEFAULT = 300;
- public static final String JT_PERSIST_JOBSTATUS =
+ public static final String JT_PERSIST_JOBSTATUS =
"mapreduce.jobtracker.persist.jobstatus.active";
- public static final String JT_PERSIST_JOBSTATUS_HOURS =
- "mapreduce.jobtracker.persist.jobstatus.hours";
- public static final String JT_PERSIST_JOBSTATUS_DIR =
- "mapreduce.jobtracker.persist.jobstatus.dir";
- /**
- * @deprecated Use MR_SUPERGROUP instead
- */
- @Deprecated
- public static final String JT_SUPERGROUP =
- "mapreduce.jobtracker.permissions.supergroup";
- public static final String JT_RETIREJOBS =
+ public static final String JT_RETIREJOBS =
"mapreduce.jobtracker.retirejobs";
- public static final String JT_RETIREJOB_CACHE_SIZE =
- "mapreduce.jobtracker.retiredjobs.cache.size";
- public static final String JT_TASKCACHE_LEVELS =
+ public static final String JT_TASKCACHE_LEVELS =
"mapreduce.jobtracker.taskcache.levels";
- public static final String JT_TASK_ALLOC_PAD_FRACTION =
- "mapreduce.jobtracker.taskscheduler.taskalloc.capacitypad";
- public static final String JT_JOBINIT_THREADS =
- "mapreduce.jobtracker.jobinit.threads";
- public static final String JT_TRACKER_EXPIRY_INTERVAL =
- "mapreduce.jobtracker.expire.trackers.interval";
- public static final String JT_RUNNINGTASKS_PER_JOB =
- "mapreduce.jobtracker.taskscheduler.maxrunningtasks.perjob";
- public static final String JT_HOSTS_FILENAME =
- "mapreduce.jobtracker.hosts.filename";
- public static final String JT_HOSTS_EXCLUDE_FILENAME =
- "mapreduce.jobtracker.hosts.exclude.filename";
- public static final String JT_JOBHISTORY_CACHE_SIZE =
- "mapreduce.jobtracker.jobhistory.lru.cache.size";
- public static final String JT_JOBHISTORY_BLOCK_SIZE =
- "mapreduce.jobtracker.jobhistory.block.size";
- public static final String JT_JOBHISTORY_COMPLETED_LOCATION =
- "mapreduce.jobtracker.jobhistory.completed.location";
- public static final String JT_JOBHISTORY_LOCATION =
- "mapreduce.jobtracker.jobhistory.location";
- // number of partial task progress reports we retain in job history
- public static final String JT_JOBHISTORY_TASKPROGRESS_NUMBER_SPLITS =
- "mapreduce.jobtracker.jobhistory.task.numberprogresssplits";
- public static final String JT_AVG_BLACKLIST_THRESHOLD =
- "mapreduce.jobtracker.blacklist.average.threshold";
public static final String JT_SYSTEM_DIR = "mapreduce.jobtracker.system.dir";
public static final String JT_STAGING_AREA_ROOT =
"mapreduce.jobtracker.staging.root.dir";
- public static final String JT_MAX_TRACKER_BLACKLISTS =
- "mapreduce.jobtracker.tasktracker.maxblacklists";
- public static final String JT_JOBHISTORY_MAXAGE =
- "mapreduce.jobtracker.jobhistory.maxage";
- public static final String JT_MAX_MAPMEMORY_MB =
+ public static final String JT_MAX_MAPMEMORY_MB =
"mapreduce.jobtracker.maxmapmemory.mb";
public static final String JT_MAX_REDUCEMEMORY_MB =
"mapreduce.jobtracker.maxreducememory.mb";
- public static final String JT_MAX_JOB_SPLIT_METAINFO_SIZE =
- "mapreduce.jobtracker.split.metainfo.maxsize";
public static final String JT_USER_NAME = "mapreduce.jobtracker.kerberos.principal";
- public static final String JT_KEYTAB_FILE =
- "mapreduce.jobtracker.keytab.file";
- public static final String PRIVATE_ACTIONS_KEY =
- "mapreduce.jobtracker.webinterface.trusted";
- public static final String JT_PLUGINS =
- "mapreduce.jobtracker.plugins";
- public static final String SHUFFLE_EXCEPTION_STACK_REGEX =
- "mapreduce.reduce.shuffle.catch.exception.stack.regex";
- public static final String SHUFFLE_EXCEPTION_MSG_REGEX =
- "mapreduce.reduce.shuffle.catch.exception.message.regex";
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de013274/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
index 66b630a..d3f62e4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
@@ -68,64 +68,16 @@ public class ConfigUtil {
new DeprecationDelta("mapred.cluster.max.reduce.memory.mb",
JTConfig.JT_MAX_REDUCEMEMORY_MB),
- new DeprecationDelta("mapred.cluster.average.blacklist.threshold",
- JTConfig.JT_AVG_BLACKLIST_THRESHOLD),
- new DeprecationDelta("hadoop.job.history.location",
- JTConfig.JT_JOBHISTORY_LOCATION),
- new DeprecationDelta(
- "mapred.job.tracker.history.completed.location",
- JTConfig.JT_JOBHISTORY_COMPLETED_LOCATION),
- new DeprecationDelta("mapred.jobtracker.job.history.block.size",
- JTConfig.JT_JOBHISTORY_BLOCK_SIZE),
- new DeprecationDelta("mapred.job.tracker.jobhistory.lru.cache.size",
- JTConfig.JT_JOBHISTORY_CACHE_SIZE),
- new DeprecationDelta("mapred.hosts",
- JTConfig.JT_HOSTS_FILENAME),
- new DeprecationDelta("mapred.hosts.exclude",
- JTConfig.JT_HOSTS_EXCLUDE_FILENAME),
new DeprecationDelta("mapred.system.dir",
JTConfig.JT_SYSTEM_DIR),
- new DeprecationDelta("mapred.max.tracker.blacklists",
- JTConfig.JT_MAX_TRACKER_BLACKLISTS),
new DeprecationDelta("mapred.job.tracker",
JTConfig.JT_IPC_ADDRESS),
- new DeprecationDelta("mapred.job.tracker.http.address",
- JTConfig.JT_HTTP_ADDRESS),
- new DeprecationDelta("mapred.job.tracker.handler.count",
- JTConfig.JT_IPC_HANDLER_COUNT),
- new DeprecationDelta("mapred.jobtracker.restart.recover",
- JTConfig.JT_RESTART_ENABLED),
- new DeprecationDelta("mapred.jobtracker.taskScheduler",
- JTConfig.JT_TASK_SCHEDULER),
- new DeprecationDelta(
- "mapred.jobtracker.taskScheduler.maxRunningTasksPerJob",
- JTConfig.JT_RUNNINGTASKS_PER_JOB),
- new DeprecationDelta("mapred.jobtracker.instrumentation",
- JTConfig.JT_INSTRUMENTATION),
- new DeprecationDelta("mapred.jobtracker.maxtasks.per.job",
- JTConfig.JT_TASKS_PER_JOB),
- new DeprecationDelta("mapred.heartbeats.in.second",
- JTConfig.JT_HEARTBEATS_IN_SECOND),
new DeprecationDelta("mapred.job.tracker.persist.jobstatus.active",
JTConfig.JT_PERSIST_JOBSTATUS),
- new DeprecationDelta("mapred.job.tracker.persist.jobstatus.hours",
- JTConfig.JT_PERSIST_JOBSTATUS_HOURS),
- new DeprecationDelta("mapred.job.tracker.persist.jobstatus.dir",
- JTConfig.JT_PERSIST_JOBSTATUS_DIR),
new DeprecationDelta("mapred.permissions.supergroup",
MRConfig.MR_SUPERGROUP),
- new DeprecationDelta("mapreduce.jobtracker.permissions.supergroup",
- MRConfig.MR_SUPERGROUP),
new DeprecationDelta("mapred.task.cache.levels",
JTConfig.JT_TASKCACHE_LEVELS),
- new DeprecationDelta("mapred.jobtracker.taskalloc.capacitypad",
- JTConfig.JT_TASK_ALLOC_PAD_FRACTION),
- new DeprecationDelta("mapred.jobinit.threads",
- JTConfig.JT_JOBINIT_THREADS),
- new DeprecationDelta("mapred.tasktracker.expiry.interval",
- JTConfig.JT_TRACKER_EXPIRY_INTERVAL),
- new DeprecationDelta("mapred.job.tracker.retiredjobs.cache.size",
- JTConfig.JT_RETIREJOB_CACHE_SIZE),
new DeprecationDelta("mapred.job.tracker.retire.jobs",
JTConfig.JT_RETIREJOBS),
new DeprecationDelta("mapred.healthChecker.interval",
@@ -511,17 +463,12 @@ public class ConfigUtil {
new DeprecationDelta("mapred.pipes.user.inputformat",
org.apache.hadoop.mapred.pipes.Submitter.INPUT_FORMAT),
- new DeprecationDelta("webinterface.private.actions",
- JTConfig.PRIVATE_ACTIONS_KEY),
-
new DeprecationDelta("security.task.umbilical.protocol.acl",
MRJobConfig.MR_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL),
new DeprecationDelta("security.job.submission.protocol.acl",
MRJobConfig.MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT),
new DeprecationDelta("mapreduce.user.classpath.first",
MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST ),
- new DeprecationDelta(JTConfig.JT_MAX_JOB_SPLIT_METAINFO_SIZE,
- MRJobConfig.SPLIT_METAINFO_MAXSIZE),
new DeprecationDelta("mapred.input.dir.recursive",
FileInputFormat.INPUT_DIR_RECURSIVE)
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de013274/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
index cbae524..e4619d6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
@@ -46,7 +46,6 @@
<property><!--Loaded from job.xml--><name>dfs.namenode.edits.dir</name><value>${dfs.namenode.name.dir}</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.map.class</name><value>org.apache.hadoop.mapreduce.SleepJob$SleepMapper</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.http.address</name><value>0.0.0.0:9864</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.jobhistory.task.numberprogresssplits</name><value>12</value></property>
<property><!--Loaded from job.xml--><name>yarn.acl.enable</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.localizer.fetch.thread-count</name><value>4</value></property>
<property><!--Loaded from job.xml--><name>hadoop.proxyuser.user.hosts</name><value>127.0.0.1</value></property>
@@ -55,7 +54,6 @@
<property><!--Loaded from job.xml--><name>mapreduce.reduce.log.level</name><value>INFO</value></property>
<property><!--Loaded from job.xml--><name>yarn.log-aggregation-enable</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>dfs.https.server.keystore.resource</name><value>ssl-server.xml</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.instrumentation</name><value>org.apache.hadoop.mapred.JobTrackerMetricsInst</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.replication.min</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.java.opts</name><value>-Xmx500m</value></property>
<property><!--Loaded from job.xml--><name>s3native.bytes-per-checksum</name><value>512</value></property>
@@ -72,7 +70,6 @@
<property><!--Loaded from job.xml--><name>dfs.namenode.max.objects</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.safemode.threshold-pct</name><value>0.999f</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.local.dir.minspacekill</name><value>0</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.retiredjobs.cache.size</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>dfs.blocksize</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.reduce.slowstart.completedmaps</name><value>0.05</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.end-notification.retry.attempts</name><value>5</value></property>
@@ -82,7 +79,6 @@
<property><!--Loaded from Unknown--><name>mapreduce.job.user.name</name><value>user</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.outofband.heartbeat</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>io.native.lib.available</name><value>true</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.persist.jobstatus.hours</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>dfs.client-write-packet-size</name><value>65536</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.client.progressmonitor.pollinterval</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.name.dir</name><value>file:///home/user/hadoop-user/dfs/name</value></property>
@@ -98,7 +94,6 @@
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.taskcontroller</name><value>org.apache.hadoop.mapred.DefaultTaskController</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.parallelcopies</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.env-whitelist</name><value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.heartbeats.in.second</name><value>100</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.maxtaskfailures.per.tracker</name><value>4</value></property>
<property><!--Loaded from job.xml--><name>ipc.client.connection.maxidletime</name><value>10000</value></property>
<property><!--Loaded from job.xml--><name>dfs.blockreport.intervalMsec</name><value>21600000</value></property>
@@ -114,7 +109,6 @@
<property><!--Loaded from job.xml--><name>ipc.client.tcpnodelay</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.output.value.class</name><value>org.apache.hadoop.io.NullWritable</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.accesstime.precision</name><value>3600000</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.tasktracker.maxblacklists</name><value>4</value></property>
<property><!--Loaded from Unknown--><name>rpc.engine.com.google.protobuf.BlockingService</name><value>org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.jvm.numtasks</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.io.sort.mb</name><value>100</value></property>
@@ -133,7 +127,6 @@
<property><!--Loaded from job.xml--><name>fs.har.impl</name><value>org.apache.hadoop.fs.HarFileSystem</value></property>
<property><!--Loaded from job.xml--><name>hadoop.security.authentication</name><value>simple</value></property>
<property><!--Loaded from job.xml--><name>fs.s3n.buffer.dir</name><value>${hadoop.tmp.dir}/s3n</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.taskscheduler</name><value>org.apache.hadoop.mapred.JobQueueTaskScheduler</value></property>
<property><!--Loaded from job.xml--><name>yarn.app.mapreduce.am.job.task.listener.thread-count</name><value>30</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.reduces</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.sort.spill.percent</name><value>0.80</value></property>
@@ -154,7 +147,6 @@
<property><!--Loaded from job.xml--><name>fs.file.impl</name><value>org.apache.hadoop.fs.LocalFileSystem</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.https-address</name><value>0.0.0.0:9871</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.skip.start.attempts</name><value>2</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.persist.jobstatus.dir</name><value>/jobtracker/jobsInfo</value></property>
<property><!--Loaded from job.xml--><name>ipc.client.kill.max</name><value>10</value></property>
<property><!--Loaded from mapred-default.xml--><name>mapreduce.job.end-notification.max.attempts</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobhistory.max-age-ms</name><value>10000000000</value></property>
@@ -175,9 +167,7 @@
<property><!--Loaded from job.xml--><name>io.mapfile.bloom.size</name><value>1048576</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.container-executor.class</name><value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.maxattempts</name><value>4</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.jobhistory.block.size</name><value>3145728</value></property>
<property><!--Loaded from job.xml--><name>ftp.replication</name><value>3</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.http.address</name><value>0.0.0.0:50030</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.health-checker.script.timeout-ms</name><value>1200000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobhistory.address</name><value>0.0.0.0:10020</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.dns.nameserver</name><value>default</value></property>
@@ -317,7 +307,6 @@
<property><!--Loaded from job.xml--><name>dfs.datanode.https.address</name><value>0.0.0.0:9865</value></property>
<property><!--Loaded from job.xml--><name>file.replication</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.resource-tracker.address</name><value>0.0.0.0:8025</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.restart.recover</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>hadoop.work.around.non.threadsafe.getpwuid</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.client.genericoptionsparser.used</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.indexcache.mb</name><value>10</value></property>
@@ -331,7 +320,6 @@
<property><!--Loaded from job.xml--><name>file.bytes-per-checksum</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.input.fileinputformat.inputdir</name><value>hdfs://localhost:8021/user/user/ignored</value></property>
<property><!--Loaded from job.xml--><name>dfs.client.block.write.replace-datanode-on-failure.enable</name><value>ture</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.handler.count</name><value>10</value></property>
<property><!--Loaded from job.xml--><name>net.topology.script.number.args</name><value>100</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.profile.maps</name><value>0-2</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.decommission.interval</name><value>30</value></property>
@@ -342,7 +330,6 @@
<property><!--Loaded from job.xml--><name>yarn.nodemanager.vmem-pmem-ratio</name><value>2.1</value></property>
<property><!--Loaded from job.xml--><name>io.mapfile.bloom.error.rate</name><value>0.005</value></property>
<property><!--Loaded from job.xml--><name>dfs.permissions.superusergroup</name><value>supergroup</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.expire.trackers.interval</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.cluster.acls.enabled</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.remote-app-log-dir-suffix</name><value>logs</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.checkpoint.check.period</name><value>60</value></property>
@@ -351,8 +338,6 @@
<property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.merge.percent</name><value>0.66</value></property>
<property><!--Loaded from job.xml--><name>tfile.io.chunk.size</name><value>1048576</value></property>
<property><!--Loaded from job.xml--><name>file.blocksize</name><value>67108864</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.jobhistory.lru.cache.size</name><value>5</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.maxtasks.perjob</name><value>-1</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.nm.liveness-monitor.interval-ms</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.webapp.address</name><value>0.0.0.0:9999</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.acl-modify-job</name><value> </value></property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de013274/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
index cc3dcea..095cfd5 100644
--- a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
+++ b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
@@ -1,4 +1,4 @@
-{
+[{
"priority" : "NORMAL",
"jobID" : "job_1369942127770_1205",
"user" : "jenkins",
@@ -4585,7 +4585,6 @@
"dfs.datanode.http.address" : "0.0.0.0:9864",
"mapreduce.terasort.num-rows" : "400000000",
"mapreduce.job.map.class" : "org.apache.hadoop.examples.terasort.TeraGen$SortGenMapper",
- "mapreduce.jobtracker.jobhistory.task.numberprogresssplits" : "12",
"dfs.namenode.write.stale.datanode.ratio" : "0.5f",
"dfs.client.use.datanode.hostname" : "false",
"yarn.acl.enable" : "true",
@@ -4602,7 +4601,6 @@
"mapreduce.reduce.log.level" : "INFO",
"yarn.log-aggregation-enable" : "false",
"dfs.datanode.sync.behind.writes" : "false",
- "mapreduce.jobtracker.instrumentation" : "org.apache.hadoop.mapred.JobTrackerMetricsInst",
"dfs.https.server.keystore.resource" : "ssl-server.xml",
"hadoop.security.group.mapping.ldap.search.attr.group.name" : "cn",
"dfs.namenode.replication.min" : "1",
@@ -4625,7 +4623,6 @@
"hadoop.ssl.client.conf" : "ssl-client.xml",
"dfs.namenode.safemode.threshold-pct" : "0.999f",
"mapreduce.tasktracker.local.dir.minspacekill" : "0",
- "mapreduce.jobtracker.retiredjobs.cache.size" : "1000",
"dfs.blocksize" : "134217728",
"yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler",
"mapreduce.job.reduce.slowstart.completedmaps" : "0.8",
@@ -4635,7 +4632,6 @@
"mapreduce.job.user.name" : "jenkins",
"mapreduce.tasktracker.outofband.heartbeat" : "false",
"io.native.lib.available" : "true",
- "mapreduce.jobtracker.persist.jobstatus.hours" : "0",
"dfs.client-write-packet-size" : "65536",
"mapreduce.client.progressmonitor.pollinterval" : "1000",
"dfs.namenode.name.dir" : "file://${hadoop.tmp.dir}/dfs/name",
@@ -4653,7 +4649,6 @@
"yarn.scheduler.fair.preemption" : "true",
"mapreduce.reduce.shuffle.parallelcopies" : "5",
"yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME",
- "mapreduce.jobtracker.heartbeats.in.second" : "100",
"mapreduce.job.maxtaskfailures.per.tracker" : "3",
"ipc.client.connection.maxidletime" : "10000",
"mapreduce.shuffle.ssl.enabled" : "false",
@@ -4673,7 +4668,6 @@
"dfs.replication" : "3",
"ipc.client.tcpnodelay" : "false",
"dfs.namenode.accesstime.precision" : "3600000",
- "mapreduce.jobtracker.tasktracker.maxblacklists" : "4",
"dfs.client.read.shortcircuit.skip.checksum" : "false",
"mapreduce.job.jvm.numtasks" : "1",
"mapreduce.task.io.sort.mb" : "100",
@@ -4692,7 +4686,6 @@
"hadoop.security.authentication" : "simple",
"fs.s3n.buffer.dir" : "${hadoop.tmp.dir}/s3n",
"rpc.engine.org.apache.hadoop.yarn.api.AMRMProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
- "mapreduce.jobtracker.taskscheduler" : "org.apache.hadoop.mapred.JobQueueTaskScheduler",
"yarn.app.mapreduce.am.job.task.listener.thread-count" : "30",
"dfs.namenode.avoid.read.stale.datanode" : "false",
"mapreduce.job.reduces" : "0",
@@ -4718,7 +4711,6 @@
"dfs.namenode.replication.interval" : "3",
"mapreduce.task.skip.start.attempts" : "2",
"dfs.namenode.https-address" : "a2115.smile.com:20102",
- "mapreduce.jobtracker.persist.jobstatus.dir" : "/jobtracker/jobsInfo",
"ipc.client.kill.max" : "10",
"dfs.ha.automatic-failover.enabled" : "false",
"mapreduce.jobhistory.keytab" : "/etc/security/keytab/jhs.service.keytab",
@@ -4741,11 +4733,9 @@
"io.mapfile.bloom.size" : "1048576",
"yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
"mapreduce.map.maxattempts" : "4",
- "mapreduce.jobtracker.jobhistory.block.size" : "3145728",
"yarn.log-aggregation.retain-seconds" : "-1",
"yarn.app.mapreduce.am.job.committer.cancel-timeout" : "60000",
"ftp.replication" : "3",
- "mapreduce.jobtracker.http.address" : "0.0.0.0:50030",
"yarn.nodemanager.health-checker.script.timeout-ms" : "1200000",
"mapreduce.jobhistory.address" : "a2115.smile.com:10020",
"mapreduce.jobtracker.taskcache.levels" : "2",
@@ -4903,7 +4893,6 @@
"dfs.datanode.drop.cache.behind.reads" : "false",
"hadoop.fuse.connection.timeout" : "300",
"hadoop.work.around.non.threadsafe.getpwuid" : "false",
- "mapreduce.jobtracker.restart.recover" : "false",
"hadoop.tmp.dir" : "/tmp/hadoop-${user.name}",
"mapreduce.output.fileoutputformat.compress" : "false",
"mapreduce.tasktracker.indexcache.mb" : "10",
@@ -4915,7 +4904,6 @@
"dfs.namenode.fs-limits.max-blocks-per-file" : "1048576",
"file.bytes-per-checksum" : "512",
"mapreduce.task.profile.reduces" : "0-2",
- "mapreduce.jobtracker.handler.count" : "10",
"dfs.client.block.write.replace-datanode-on-failure.enable" : "true",
"mapreduce.job.output.value.class" : "org.apache.hadoop.io.Text",
"yarn.dispatcher.exit-on-error" : "true",
@@ -4933,7 +4921,6 @@
"mapreduce.shuffle.ssl.file.buffer.size" : "65536",
"dfs.permissions.superusergroup" : "supergroup",
"dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold" : "10737418240",
- "mapreduce.jobtracker.expire.trackers.interval" : "600000",
"mapreduce.cluster.acls.enabled" : "false",
"yarn.nodemanager.remote-app-log-dir-suffix" : "logs",
"ha.failover-controller.graceful-fence.connection.retries" : "1",
@@ -4943,8 +4930,6 @@
"io.seqfile.local.dir" : "${hadoop.tmp.dir}/io/local",
"dfs.namenode.checkpoint.check.period" : "60",
"yarn.resourcemanager.nm.liveness-monitor.interval-ms" : "1000",
- "mapreduce.jobtracker.maxtasks.perjob" : "-1",
- "mapreduce.jobtracker.jobhistory.lru.cache.size" : "5",
"file.blocksize" : "67108864",
"tfile.io.chunk.size" : "1048576",
"mapreduce.job.acl-modify-job" : " ",
@@ -5095,7 +5080,7 @@
"clusterReduceMB" : -1,
"jobMapMB" : 200,
"jobReduceMB" : 200
-} {
+}, {
"priority" : "NORMAL",
"jobID" : "job_1369942127770_1206",
"user" : "jenkins",
@@ -9682,7 +9667,6 @@
"dfs.datanode.http.address" : "0.0.0.0:9864",
"mapreduce.terasort.num-rows" : "400000000",
"mapreduce.job.map.class" : "org.apache.hadoop.examples.terasort.TeraGen$SortGenMapper",
- "mapreduce.jobtracker.jobhistory.task.numberprogresssplits" : "12",
"dfs.namenode.write.stale.datanode.ratio" : "0.5f",
"dfs.client.use.datanode.hostname" : "false",
"yarn.acl.enable" : "true",
@@ -9699,7 +9683,6 @@
"mapreduce.reduce.log.level" : "INFO",
"yarn.log-aggregation-enable" : "false",
"dfs.datanode.sync.behind.writes" : "false",
- "mapreduce.jobtracker.instrumentation" : "org.apache.hadoop.mapred.JobTrackerMetricsInst",
"dfs.https.server.keystore.resource" : "ssl-server.xml",
"hadoop.security.group.mapping.ldap.search.attr.group.name" : "cn",
"dfs.namenode.replication.min" : "1",
@@ -9722,7 +9705,6 @@
"hadoop.ssl.client.conf" : "ssl-client.xml",
"dfs.namenode.safemode.threshold-pct" : "0.999f",
"mapreduce.tasktracker.local.dir.minspacekill" : "0",
- "mapreduce.jobtracker.retiredjobs.cache.size" : "1000",
"dfs.blocksize" : "134217728",
"yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler",
"mapreduce.job.reduce.slowstart.completedmaps" : "0.8",
@@ -9732,7 +9714,6 @@
"mapreduce.job.user.name" : "jenkins",
"mapreduce.tasktracker.outofband.heartbeat" : "false",
"io.native.lib.available" : "true",
- "mapreduce.jobtracker.persist.jobstatus.hours" : "0",
"dfs.client-write-packet-size" : "65536",
"mapreduce.client.progressmonitor.pollinterval" : "1000",
"dfs.namenode.name.dir" : "file://${hadoop.tmp.dir}/dfs/name",
@@ -9750,7 +9731,6 @@
"yarn.scheduler.fair.preemption" : "true",
"mapreduce.reduce.shuffle.parallelcopies" : "5",
"yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME",
- "mapreduce.jobtracker.heartbeats.in.second" : "100",
"mapreduce.job.maxtaskfailures.per.tracker" : "3",
"ipc.client.connection.maxidletime" : "10000",
"mapreduce.shuffle.ssl.enabled" : "false",
@@ -9770,7 +9750,6 @@
"dfs.replication" : "3",
"ipc.client.tcpnodelay" : "false",
"dfs.namenode.accesstime.precision" : "3600000",
- "mapreduce.jobtracker.tasktracker.maxblacklists" : "4",
"dfs.client.read.shortcircuit.skip.checksum" : "false",
"mapreduce.job.jvm.numtasks" : "1",
"mapreduce.task.io.sort.mb" : "100",
@@ -9789,7 +9768,6 @@
"hadoop.security.authentication" : "simple",
"fs.s3n.buffer.dir" : "${hadoop.tmp.dir}/s3",
"rpc.engine.org.apache.hadoop.yarn.api.AMRMProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
- "mapreduce.jobtracker.taskscheduler" : "org.apache.hadoop.mapred.JobQueueTaskScheduler",
"yarn.app.mapreduce.am.job.task.listener.thread-count" : "30",
"dfs.namenode.avoid.read.stale.datanode" : "false",
"mapreduce.job.reduces" : "0",
@@ -9815,7 +9793,6 @@
"dfs.namenode.replication.interval" : "3",
"mapreduce.task.skip.start.attempts" : "2",
"dfs.namenode.https-address" : "a2115.smile.com:20102",
- "mapreduce.jobtracker.persist.jobstatus.dir" : "/jobtracker/jobsInfo",
"ipc.client.kill.max" : "10",
"dfs.ha.automatic-failover.enabled" : "false",
"mapreduce.jobhistory.keytab" : "/etc/security/keytab/jhs.service.keytab",
@@ -9838,11 +9815,9 @@
"io.mapfile.bloom.size" : "1048576",
"yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
"mapreduce.map.maxattempts" : "4",
- "mapreduce.jobtracker.jobhistory.block.size" : "3145728",
"yarn.log-aggregation.retain-seconds" : "-1",
"yarn.app.mapreduce.am.job.committer.cancel-timeout" : "60000",
"ftp.replication" : "3",
- "mapreduce.jobtracker.http.address" : "0.0.0.0:50030",
"yarn.nodemanager.health-checker.script.timeout-ms" : "1200000",
"mapreduce.jobhistory.address" : "a2115.smile.com:10020",
"mapreduce.jobtracker.taskcache.levels" : "2",
@@ -10000,7 +9975,6 @@
"dfs.datanode.drop.cache.behind.reads" : "false",
"hadoop.fuse.connection.timeout" : "300",
"hadoop.work.around.non.threadsafe.getpwuid" : "false",
- "mapreduce.jobtracker.restart.recover" : "false",
"hadoop.tmp.dir" : "/tmp/hadoop-${user.name}",
"mapreduce.output.fileoutputformat.compress" : "false",
"mapreduce.tasktracker.indexcache.mb" : "10",
@@ -10012,7 +9986,6 @@
"dfs.namenode.fs-limits.max-blocks-per-file" : "1048576",
"file.bytes-per-checksum" : "512",
"mapreduce.task.profile.reduces" : "0-2",
- "mapreduce.jobtracker.handler.count" : "10",
"dfs.client.block.write.replace-datanode-on-failure.enable" : "true",
"mapreduce.job.output.value.class" : "org.apache.hadoop.io.Text",
"yarn.dispatcher.exit-on-error" : "true",
@@ -10030,7 +10003,6 @@
"mapreduce.shuffle.ssl.file.buffer.size" : "65536",
"dfs.permissions.superusergroup" : "supergroup",
"dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold" : "10737418240",
- "mapreduce.jobtracker.expire.trackers.interval" : "600000",
"mapreduce.cluster.acls.enabled" : "false",
"yarn.nodemanager.remote-app-log-dir-suffix" : "logs",
"ha.failover-controller.graceful-fence.connection.retries" : "1",
@@ -10040,8 +10012,6 @@
"io.seqfile.local.dir" : "${hadoop.tmp.dir}/io/local",
"dfs.namenode.checkpoint.check.period" : "60",
"yarn.resourcemanager.nm.liveness-monitor.interval-ms" : "1000",
- "mapreduce.jobtracker.maxtasks.perjob" : "-1",
- "mapreduce.jobtracker.jobhistory.lru.cache.size" : "5",
"file.blocksize" : "67108864",
"tfile.io.chunk.size" : "1048576",
"mapreduce.job.acl-modify-job" : " ",
@@ -10192,7 +10162,7 @@
"clusterReduceMB" : -1,
"jobMapMB" : 200,
"jobReduceMB" : 200
-} {
+}, {
"priority" : "NORMAL",
"jobID" : "job_1369942127770_1207",
"user" : "jenkins",
@@ -10279,7 +10249,6 @@
"dfs.datanode.http.address" : "0.0.0.0:9864",
"mapreduce.terasort.num-rows" : "400000000",
"mapreduce.job.map.class" : "org.apache.hadoop.examples.terasort.TeraGen$SortGenMapper",
-"mapreduce.jobtracker.jobhistory.task.numberprogresssplits" : "12",
"dfs.namenode.write.stale.datanode.ratio" : "0.5f",
"dfs.client.use.datanode.hostname" : "false",
"yarn.acl.enable" : "true",
@@ -10296,7 +10265,6 @@
"mapreduce.reduce.log.level" : "INFO",
"yarn.log-aggregation-enable" : "false",
"dfs.datanode.sync.behind.writes" : "false",
-"mapreduce.jobtracker.instrumentation" : "org.apache.hadoop.mapred.JobTrackerMetricsInst",
"dfs.https.server.keystore.resource" : "ssl-server.xml",
"hadoop.security.group.mapping.ldap.search.attr.group.name" : "cn",
"dfs.namenode.replication.min" : "1",
@@ -10319,7 +10287,6 @@
"hadoop.ssl.client.conf" : "ssl-client.xml",
"dfs.namenode.safemode.threshold-pct" : "0.999f",
"mapreduce.tasktracker.local.dir.minspacekill" : "0",
-"mapreduce.jobtracker.retiredjobs.cache.size" : "1000",
"dfs.blocksize" : "134217728",
"yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler",
"mapreduce.job.reduce.slowstart.completedmaps" : "0.8",
@@ -10329,7 +10296,6 @@
"mapreduce.job.user.name" : "jenkins",
"mapreduce.tasktracker.outofband.heartbeat" : "false",
"io.native.lib.available" : "true",
-"mapreduce.jobtracker.persist.jobstatus.hours" : "0",
"dfs.client-write-packet-size" : "65536",
"mapreduce.client.progressmonitor.pollinterval" : "1000",
"dfs.namenode.name.dir" : "file://${hadoop.tmp.dir}/dfs/name",
@@ -10348,7 +10314,6 @@
"mapreduce.reduce.shuffle.parallelcopies" : "5",
"dfs.support.append" : "true",
"yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME",
-"mapreduce.jobtracker.heartbeats.in.second" : "100",
"mapreduce.job.maxtaskfailures.per.tracker" : "3",
"ipc.client.connection.maxidletime" : "10000",
"mapreduce.shuffle.ssl.enabled" : "false",
@@ -10368,7 +10333,6 @@
"dfs.replication" : "3",
"ipc.client.tcpnodelay" : "false",
"dfs.namenode.accesstime.precision" : "3600000",
-"mapreduce.jobtracker.tasktracker.maxblacklists" : "4",
"dfs.client.read.shortcircuit.skip.checksum" : "false",
"mapreduce.job.jvm.numtasks" : "1",
"mapreduce.task.io.sort.mb" : "100",
@@ -10387,7 +10351,6 @@
"hadoop.security.authentication" : "simple",
"fs.s3n.buffer.dir" : "${hadoop.tmp.dir}/s3n",
"rpc.engine.org.apache.hadoop.yarn.api.AMRMProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
-"mapreduce.jobtracker.taskscheduler" : "org.apache.hadoop.mapred.JobQueueTaskScheduler",
"yarn.app.mapreduce.am.job.task.listener.thread-count" : "30",
"dfs.namenode.avoid.read.stale.datanode" : "false",
"mapreduce.job.reduces" : "0",
@@ -10413,7 +10376,6 @@
"dfs.namenode.replication.interval" : "3",
"mapreduce.task.skip.start.attempts" : "2",
"dfs.namenode.https-address" : "a2115.smile.com:20102",
-"mapreduce.jobtracker.persist.jobstatus.dir" : "/jobtracker/jobsInfo",
"ipc.client.kill.max" : "10",
"dfs.ha.automatic-failover.enabled" : "false",
"mapreduce.jobhistory.keytab" : "/etc/security/keytab/jhs.service.keytab",
@@ -10436,11 +10398,9 @@
"io.mapfile.bloom.size" : "1048576",
"yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
"mapreduce.map.maxattempts" : "4",
-"mapreduce.jobtracker.jobhistory.block.size" : "3145728",
"yarn.log-aggregation.retain-seconds" : "-1",
"yarn.app.mapreduce.am.job.committer.cancel-timeout" : "60000",
"ftp.replication" : "3",
-"mapreduce.jobtracker.http.address" : "0.0.0.0:50030",
"yarn.nodemanager.health-checker.script.timeout-ms" : "1200000",
"mapreduce.jobhistory.address" : "a2115.smile.com:10020",
"mapreduce.jobtracker.taskcache.levels" : "2",
@@ -10598,7 +10558,6 @@
"dfs.datanode.drop.cache.behind.reads" : "false",
"hadoop.fuse.connection.timeout" : "300",
"hadoop.work.around.non.threadsafe.getpwuid" : "false",
-"mapreduce.jobtracker.restart.recover" : "false",
"hadoop.tmp.dir" : "/tmp/hadoop-${user.name}",
"mapreduce.output.fileoutputformat.compress" : "false",
"mapreduce.tasktracker.indexcache.mb" : "10",
@@ -10610,7 +10569,6 @@
"dfs.namenode.fs-limits.max-blocks-per-file" : "1048576",
"file.bytes-per-checksum" : "512",
"mapreduce.task.profile.reduces" : "0-2",
-"mapreduce.jobtracker.handler.count" : "10",
"dfs.client.block.write.replace-datanode-on-failure.enable" : "true",
"mapreduce.job.output.value.class" : "org.apache.hadoop.io.Text",
"yarn.dispatcher.exit-on-error" : "true",
@@ -10628,7 +10586,6 @@
"mapreduce.shuffle.ssl.file.buffer.size" : "65536",
"dfs.permissions.superusergroup" : "supergroup",
"dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold" : "10737418240",
-"mapreduce.jobtracker.expire.trackers.interval" : "600000",
"mapreduce.cluster.acls.enabled" : "false",
"yarn.nodemanager.remote-app-log-dir-suffix" : "logs",
"ha.failover-controller.graceful-fence.connection.retries" : "1",
@@ -10638,8 +10595,6 @@
"io.seqfile.local.dir" : "${hadoop.tmp.dir}/io/local",
"dfs.namenode.checkpoint.check.period" : "60",
"yarn.resourcemanager.nm.liveness-monitor.interval-ms" : "1000",
-"mapreduce.jobtracker.maxtasks.perjob" : "-1",
-"mapreduce.jobtracker.jobhistory.lru.cache.size" : "5",
"file.blocksize" : "67108864",
"tfile.io.chunk.size" : "1048576",
"mapreduce.job.acl-modify-job" : " ",
@@ -10791,3 +10746,4 @@
"jobMapMB" : 200,
"jobReduceMB" : 200
}
+]
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org