You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by kk...@apache.org on 2017/03/07 23:10:15 UTC
[01/50] hadoop git commit: HADOOP-12556. KafkaSink jar files are
created but not copied to target dist (Babak Behzad via aw) [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-5972 3a68608c7 -> 6999ad2e0 (forced update)
HADOOP-12556. KafkaSink jar files are created but not copied to target dist (Babak Behzad via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/989bd56b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/989bd56b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/989bd56b
Branch: refs/heads/YARN-5972
Commit: 989bd56b9fd521a67afbc691c9915be13d7c8e3a
Parents: 235203d
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Feb 28 16:15:40 2017 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Feb 28 16:15:40 2017 -0800
----------------------------------------------------------------------
hadoop-tools/hadoop-tools-dist/pom.xml | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989bd56b/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 00f07ef..dd28404 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -95,6 +95,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-kafka</artifactId>
+ <scope>compile</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
<scope>compile</scope>
<version>${project.version}</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] hadoop git commit: YARN-6287.
RMCriticalThreadUncaughtExceptionHandler.rmContext should be final
(Contributed by Corey Barker via Daniel Templeton)
Posted by kk...@apache.org.
YARN-6287. RMCriticalThreadUncaughtExceptionHandler.rmContext should be final (Contributed by Corey Barker via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0c239cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0c239cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0c239cd
Branch: refs/heads/YARN-5972
Commit: e0c239cdbda336e09a35d112d451c2e17d74a3fc
Parents: 1f9848d
Author: Daniel Templeton <te...@apache.org>
Authored: Tue Mar 7 11:58:48 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Tue Mar 7 11:58:48 2017 -0800
----------------------------------------------------------------------
.../resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0c239cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
index c5c6087..a67f81a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
@@ -37,7 +37,7 @@ public class RMCriticalThreadUncaughtExceptionHandler
implements UncaughtExceptionHandler {
private static final Log LOG = LogFactory.getLog(
RMCriticalThreadUncaughtExceptionHandler.class);
- private RMContext rmContext;
+ private final RMContext rmContext;
public RMCriticalThreadUncaughtExceptionHandler(RMContext rmContext) {
this.rmContext = rmContext;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] hadoop git commit: Merge branch 'trunk' of
https://git-wip-us.apache.org/repos/asf/hadoop into trunk
Posted by kk...@apache.org.
Merge branch 'trunk' of https://git-wip-us.apache.org/repos/asf/hadoop into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e14eade
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e14eade
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e14eade
Branch: refs/heads/YARN-5972
Commit: 4e14eaded21f244d280b1ffe90a89959fd753047
Parents: 25c84d2 555d0c3
Author: Ming Ma <mi...@apache.org>
Authored: Wed Mar 1 20:24:49 2017 -0800
Committer: Ming Ma <mi...@apache.org>
Committed: Wed Mar 1 20:24:49 2017 -0800
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] hadoop git commit: HDFS-8741. Proper error msg to be printed
when invalid operation type is given to WebHDFS operations. Contributed by
Surendra Singh Lilhore.
Posted by kk...@apache.org.
HDFS-8741. Proper error msg to be printed when invalid operation type is given to WebHDFS operations. Contributed by Surendra Singh Lilhore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3536ce03
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3536ce03
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3536ce03
Branch: refs/heads/YARN-5972
Commit: 3536ce031ca780d6de83cf67779f571a0142ccc8
Parents: fad766e
Author: Yiqun Lin <yq...@apache.org>
Authored: Mon Mar 6 19:04:03 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Mon Mar 6 19:04:03 2017 +0800
----------------------------------------------------------------------
.../hdfs/web/resources/DeleteOpParam.java | 11 +++++-
.../hadoop/hdfs/web/resources/GetOpParam.java | 11 +++++-
.../hadoop/hdfs/web/resources/PostOpParam.java | 11 +++++-
.../hadoop/hdfs/web/resources/PutOpParam.java | 11 +++++-
.../hadoop/hdfs/web/resources/TestParam.java | 41 ++++++++++++++++++++
5 files changed, 81 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3536ce03/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java
index 25bed1c..e765498 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java
@@ -72,7 +72,16 @@ public class DeleteOpParam extends HttpOpParam<DeleteOpParam.Op> {
* @param str a string representation of the parameter value.
*/
public DeleteOpParam(final String str) {
- super(DOMAIN, DOMAIN.parse(str));
+ super(DOMAIN, getOp(str));
+ }
+
+ private static Op getOp(String str) {
+ try {
+ return DOMAIN.parse(str);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(str + " is not a valid " + Type.DELETE
+ + " operation.");
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3536ce03/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
index 1321bf6..d32af33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
@@ -111,7 +111,16 @@ public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
* @param str a string representation of the parameter value.
*/
public GetOpParam(final String str) {
- super(DOMAIN, DOMAIN.parse(str));
+ super(DOMAIN, getOp(str));
+ }
+
+ private static Op getOp(String str) {
+ try {
+ return DOMAIN.parse(str);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(str + " is not a valid " + Type.GET
+ + " operation.");
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3536ce03/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java
index 56a14c7..305db46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java
@@ -80,7 +80,16 @@ public class PostOpParam extends HttpOpParam<PostOpParam.Op> {
* @param str a string representation of the parameter value.
*/
public PostOpParam(final String str) {
- super(DOMAIN, DOMAIN.parse(str));
+ super(DOMAIN, getOp(str));
+ }
+
+ private static Op getOp(String str) {
+ try {
+ return DOMAIN.parse(str);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(str + " is not a valid " + Type.POST
+ + " operation.");
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3536ce03/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
index 4bb48a6..558bb53 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
@@ -107,7 +107,16 @@ public class PutOpParam extends HttpOpParam<PutOpParam.Op> {
* @param str a string representation of the parameter value.
*/
public PutOpParam(final String str) {
- super(DOMAIN, DOMAIN.parse(str));
+ super(DOMAIN, getOp(str));
+ }
+
+ private static Op getOp(String str) {
+ try {
+ return DOMAIN.parse(str);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(str + " is not a valid " + Type.PUT
+ + " operation.");
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3536ce03/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
index d444cb4..6c145a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -503,4 +504,44 @@ public class TestParam {
p = new StoragePolicyParam("COLD");
Assert.assertEquals("COLD", p.getValue());
}
+
+ @Test
+ public void testHttpOpParams() {
+ try {
+ new PostOpParam("TEST");
+ Assert
+ .fail("Construct the PostOpParam with param value 'TEST' should be"
+ + " failed.");
+ } catch (IllegalArgumentException e) {
+ GenericTestUtils.assertExceptionContains(
+ "TEST is not a valid POST operation.", e);
+ }
+ try {
+ new PutOpParam("TEST");
+ Assert
+ .fail("Construct the PutOpParam with param value 'TEST' should be"
+ + " failed.");
+ } catch (IllegalArgumentException e) {
+ GenericTestUtils.assertExceptionContains(
+ "TEST is not a valid PUT operation.", e);
+ }
+ try {
+ new DeleteOpParam("TEST");
+ Assert
+ .fail("Construct the DeleteOpParam with param value 'TEST' should be"
+ + " failed.");
+ } catch (IllegalArgumentException e) {
+ GenericTestUtils.assertExceptionContains(
+ "TEST is not a valid DELETE operation.", e);
+ }
+ try {
+ new GetOpParam("TEST");
+ Assert
+ .fail("Construct the GetOpParam with param value 'TEST' should be"
+ + " failed.");
+ } catch (IllegalArgumentException e) {
+ GenericTestUtils.assertExceptionContains(
+ "TEST is not a valid GET operation.", e);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] hadoop git commit: HADOOP-14108. CLI MiniCluster: add an
option to specify NameNode HTTP port. Contributed by Takanobu Asanuma.
Posted by kk...@apache.org.
HADOOP-14108. CLI MiniCluster: add an option to specify NameNode HTTP port. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/209ecd1a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/209ecd1a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/209ecd1a
Branch: refs/heads/YARN-5972
Commit: 209ecd1a5c056e096a5d804759f88302608d503b
Parents: 3536ce0
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Mar 7 01:32:47 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Mar 7 01:34:51 2017 +0900
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/CLIMiniCluster.md.vm | 1 +
.../org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java | 7 ++++++-
2 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209ecd1a/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
index 74c2414..806df0a 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
@@ -55,6 +55,7 @@ There are a number of command line arguments that the users can use to control w
$ -namenode <arg> URL of the namenode (default is either the DFS
$ cluster or a temporary dir)
$ -nnport <arg> NameNode port (default 0--we choose)
+ $ -nnhttpport <arg> NameNode HTTP port (default 0--we choose)
$ -nodemanagers <arg> How many nodemanagers to start (default 1)
$ -nodfs Don't start a mini DFS cluster
$ -nomr Don't start a mini MR cluster
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209ecd1a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
index 3cc73b5..324f0ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
@@ -69,6 +69,7 @@ public class MiniHadoopClusterManager {
private int numNodeManagers;
private int numDataNodes;
private int nnPort;
+ private int nnHttpPort;
private int rmPort;
private int jhsPort;
private StartupOption dfsOpts;
@@ -92,6 +93,8 @@ public class MiniHadoopClusterManager {
.addOption("datanodes", true, "How many datanodes to start (default 1)")
.addOption("format", false, "Format the DFS (default false)")
.addOption("nnport", true, "NameNode port (default 0--we choose)")
+ .addOption("nnhttpport", true,
+ "NameNode HTTP port (default 0--we choose)")
.addOption(
"namenode",
true,
@@ -152,7 +155,8 @@ public class MiniHadoopClusterManager {
URISyntaxException {
if (!noDFS) {
dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort)
- .numDataNodes(numDataNodes).startupOption(dfsOpts).build();
+ .nameNodeHttpPort(nnHttpPort).numDataNodes(numDataNodes)
+ .startupOption(dfsOpts).build();
LOG.info("Started MiniDFSCluster -- namenode on port "
+ dfs.getNameNodePort());
}
@@ -254,6 +258,7 @@ public class MiniHadoopClusterManager {
noDFS = cli.hasOption("nodfs");
numDataNodes = intArgument(cli, "datanodes", 1);
nnPort = intArgument(cli, "nnport", 0);
+ nnHttpPort = intArgument(cli, "nnhttpport", 0);
dfsOpts = cli.hasOption("format") ? StartupOption.FORMAT
: StartupOption.REGULAR;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] hadoop git commit: HDFS-11461. DataNode Disk Outlier
Detection. Contributed by Hanisha Koneru.
Posted by kk...@apache.org.
HDFS-11461. DataNode Disk Outlier Detection. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3ec531f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3ec531f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3ec531f
Branch: refs/heads/YARN-5972
Commit: b3ec531f400dd0a6506dc71233d38ae57b764a43
Parents: 747bafa
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Mar 2 12:45:48 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Mar 2 12:45:48 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +-
.../server/blockmanagement/SlowPeerTracker.java | 4 +-
.../hdfs/server/datanode/BPServiceActor.java | 2 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 16 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 10 +
.../datanode/metrics/DataNodeDiskMetrics.java | 181 +++++++++++++++++
.../datanode/metrics/DataNodePeerMetrics.java | 6 +-
.../datanode/metrics/OutlierDetector.java | 182 +++++++++++++++++
.../datanode/metrics/SlowNodeDetector.java | 194 -------------------
.../src/main/resources/hdfs-default.xml | 2 +-
.../TestDataNodeOutlierDetectionViaMetrics.java | 6 +-
.../datanode/metrics/TestSlowNodeDetector.java | 30 +--
12 files changed, 412 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 68cef36..be20829 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -677,9 +677,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT = "dfs.block.misreplication.processing.limit";
public static final int DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT = 10000;
- public static final String DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_KEY =
- "dfs.datanode.slow.peers.report.interval";
- public static final String DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_DEFAULT =
+ public static final String DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY =
+ "dfs.datanode.outliers.report.interval";
+ public static final String DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT =
"30m";
// property for fsimage compression
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
index cf3a20c..c72a621 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
@@ -94,8 +94,8 @@ public class SlowPeerTracker {
this.timer = timer;
this.allReports = new ConcurrentHashMap<>();
this.reportValidityMs = conf.getTimeDuration(
- DFSConfigKeys.DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_KEY,
- DFSConfigKeys.DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_DEFAULT,
+ DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS) * 3;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 644a8ab..a0ba627 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -129,7 +129,7 @@ class BPServiceActor implements Runnable {
prevBlockReportId = ThreadLocalRandom.current().nextLong();
scheduler = new Scheduler(dnConf.heartBeatInterval,
dnConf.getLifelineIntervalMs(), dnConf.blockReportInterval,
- dnConf.slowPeersReportIntervalMs);
+ dnConf.outliersReportIntervalMs);
// get the value of maxDataLength.
this.maxDataLength = dnConf.getMaxDataLength();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 2723677..3275ba8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -30,8 +30,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_LIFELINE_INTERVAL_SECONDS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_NON_LOCAL_LAZY_PERSIST;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_NON_LOCAL_LAZY_PERSIST_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY;
@@ -95,7 +95,8 @@ public class DNConf {
final long blockReportInterval;
final long blockReportSplitThreshold;
final boolean peerStatsEnabled;
- final long slowPeersReportIntervalMs;
+ final boolean diskStatsEnabled;
+ final long outliersReportIntervalMs;
final long ibrInterval;
final long initialBlockReportDelayMs;
final long cacheReportInterval;
@@ -173,9 +174,12 @@ public class DNConf {
this.peerStatsEnabled = getConf().getBoolean(
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
- this.slowPeersReportIntervalMs = getConf().getTimeDuration(
- DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_KEY,
- DFS_DATANODE_SLOW_PEERS_REPORT_INTERVAL_DEFAULT,
+ this.diskStatsEnabled = getConf().getBoolean(
+ DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY,
+ DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
+ this.outliersReportIntervalMs = getConf().getTimeDuration(
+ DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
+ DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
this.ibrInterval = getConf().getLong(
DFSConfigKeys.DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 5db41bd..6f24858 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -164,6 +164,7 @@ import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResour
import org.apache.hadoop.hdfs.server.datanode.erasurecode.ErasureCodingWorker;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeDiskMetrics;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodePeerMetrics;
import org.apache.hadoop.hdfs.server.datanode.web.DatanodeHttpServer;
@@ -336,6 +337,7 @@ public class DataNode extends ReconfigurableBase
DataNodeMetrics metrics;
@Nullable
private DataNodePeerMetrics peerMetrics;
+ private DataNodeDiskMetrics diskMetrics;
private InetSocketAddress streamingAddr;
// See the note below in incrDatanodeNetworkErrors re: concurrency.
@@ -1390,6 +1392,11 @@ public class DataNode extends ReconfigurableBase
dnConf.saslPropsResolver, dnConf.trustedChannelResolver);
saslServer = new SaslDataTransferServer(dnConf, blockPoolTokenSecretManager);
startMetricsLogger();
+
+ if (dnConf.diskStatsEnabled) {
+ diskMetrics = new DataNodeDiskMetrics(this,
+ dnConf.outliersReportIntervalMs);
+ }
}
/**
@@ -2046,6 +2053,9 @@ public class DataNode extends ReconfigurableBase
if (metrics != null) {
metrics.shutdown();
}
+ if (diskMetrics != null) {
+ diskMetrics.shutdownAndWait();
+ }
if (dataNodeInfoBeanName != null) {
MBeans.unregister(dataNodeInfoBeanName);
dataNodeInfoBeanName = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
new file mode 100644
index 0000000..85e2bd9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.metrics;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.util.Daemon;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This class detects and maintains DataNode disk outliers and their
+ * latencies for different ops (metadata, read, write).
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class DataNodeDiskMetrics {
+
+ public static final Logger LOG = LoggerFactory.getLogger(
+ DataNodeDiskMetrics.class);
+
+ private DataNode dn;
+ private final long MIN_OUTLIER_DETECTION_DISKS = 5;
+ private final long SLOW_DISK_LOW_THRESHOLD_MS = 20;
+ private final long detectionInterval;
+ private volatile boolean shouldRun;
+ private OutlierDetector slowDiskDetector;
+ private Daemon slowDiskDetectionDaemon;
+ private volatile Map<String, Map<DiskOutlierDetectionOp, Double>> diskOutliersStats;
+
+ public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) {
+ this.dn = dn;
+ this.detectionInterval = diskOutlierDetectionIntervalMs;
+ slowDiskDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_DISKS,
+ SLOW_DISK_LOW_THRESHOLD_MS);
+ shouldRun = true;
+ startDiskOutlierDetectionThread();
+ }
+
+ private void startDiskOutlierDetectionThread() {
+ slowDiskDetectionDaemon = new Daemon(new Runnable() {
+ @Override
+ public void run() {
+ while (shouldRun) {
+ Map<String, Double> metadataOpStats = Maps.newHashMap();
+ Map<String, Double> readIoStats = Maps.newHashMap();
+ Map<String, Double> writeIoStats = Maps.newHashMap();
+ FsDatasetSpi.FsVolumeReferences fsVolumeReferences = null;
+ try {
+ fsVolumeReferences = dn.getFSDataset().getFsVolumeReferences();
+ Iterator<FsVolumeSpi> volumeIterator = fsVolumeReferences
+ .iterator();
+ while (volumeIterator.hasNext()) {
+ FsVolumeSpi volume = volumeIterator.next();
+ DataNodeVolumeMetrics metrics = volumeIterator.next().getMetrics();
+ String volumeName = volume.getBaseURI().getPath();
+
+ metadataOpStats.put(volumeName,
+ metrics.getMetadataOperationMean());
+ readIoStats.put(volumeName, metrics.getReadIoMean());
+ writeIoStats.put(volumeName, metrics.getWriteIoMean());
+ }
+ } finally {
+ if (fsVolumeReferences != null) {
+ try {
+ fsVolumeReferences.close();
+ } catch (IOException e) {
+ LOG.error("Error in releasing FS Volume references", e);
+ }
+ }
+ }
+ if (metadataOpStats.isEmpty() && readIoStats.isEmpty() &&
+ writeIoStats.isEmpty()) {
+ LOG.debug("No disk stats available for detecting outliers.");
+ return;
+ }
+
+ detectAndUpdateDiskOutliers(metadataOpStats, readIoStats,
+ writeIoStats);
+
+ try {
+ Thread.sleep(detectionInterval);
+ } catch (InterruptedException e) {
+ LOG.error("Disk Outlier Detection thread interrupted", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ });
+ slowDiskDetectionDaemon.start();
+ }
+
+ private void detectAndUpdateDiskOutliers(Map<String, Double> metadataOpStats,
+ Map<String, Double> readIoStats, Map<String, Double> writeIoStats) {
+ Set<String> diskOutliersSet = Sets.newHashSet();
+
+ // Get MetadataOp Outliers
+ Map<String, Double> metadataOpOutliers = slowDiskDetector
+ .getOutliers(metadataOpStats);
+ if (!metadataOpOutliers.isEmpty()) {
+ diskOutliersSet.addAll(metadataOpOutliers.keySet());
+ }
+
+ // Get ReadIo Outliers
+ Map<String, Double> readIoOutliers = slowDiskDetector
+ .getOutliers(readIoStats);
+ if (!readIoOutliers.isEmpty()) {
+ diskOutliersSet.addAll(readIoOutliers.keySet());
+ }
+
+ // Get WriteIo Outliers
+ Map<String, Double> writeIoOutliers = slowDiskDetector
+ .getOutliers(writeIoStats);
+ if (!readIoOutliers.isEmpty()) {
+ diskOutliersSet.addAll(writeIoOutliers.keySet());
+ }
+
+ Map<String, Map<DiskOutlierDetectionOp, Double>> diskStats =
+ Maps.newHashMap();
+ for (String disk : diskOutliersSet) {
+ Map<DiskOutlierDetectionOp, Double> diskStat = Maps.newHashMap();
+ diskStat.put(DiskOutlierDetectionOp.METADATA, metadataOpStats.get(disk));
+ diskStat.put(DiskOutlierDetectionOp.READ, readIoStats.get(disk));
+ diskStat.put(DiskOutlierDetectionOp.WRITE, writeIoStats.get(disk));
+ diskStats.put(disk, diskStat);
+ }
+
+ diskOutliersStats = diskStats;
+ LOG.debug("Updated disk outliers.");
+ }
+
+ /**
+ * Lists the types of operations on which disk latencies are measured.
+ */
+ public enum DiskOutlierDetectionOp {
+ METADATA,
+ READ,
+ WRITE
+ }
+
+ public Map<String,
+ Map<DiskOutlierDetectionOp, Double>> getDiskOutliersStats() {
+ return diskOutliersStats;
+ }
+
+ public void shutdownAndWait() {
+ shouldRun = false;
+ slowDiskDetectionDaemon.interrupt();
+ try {
+ slowDiskDetectionDaemon.join();
+ } catch (InterruptedException e) {
+ LOG.error("Disk Outlier Detection daemon did not shutdown", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
index 5241c78..827bdd2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
@@ -52,8 +52,9 @@ public class DataNodePeerMetrics {
* Threshold in milliseconds below which a DataNode is definitely not slow.
*/
private static final long LOW_THRESHOLD_MS = 5;
+ private static final long MIN_OUTLIER_DETECTION_NODES = 10;
- private final SlowNodeDetector slowNodeDetector;
+ private final OutlierDetector slowNodeDetector;
/**
* Minimum number of packet send samples which are required to qualify
@@ -68,7 +69,8 @@ public class DataNodePeerMetrics {
final long windowSizeMs,
final int numWindows) {
this.name = name;
- this.slowNodeDetector = new SlowNodeDetector(LOW_THRESHOLD_MS);
+ this.slowNodeDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_NODES,
+ LOW_THRESHOLD_MS);
sendPacketDownstreamRollingAvgerages = new RollingAverages(
windowSizeMs, numWindows);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
new file mode 100644
index 0000000..771a17b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.metrics;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * A utility class to help detect resources (nodes/ disks) whose aggregate
+ * latency is an outlier within a given set.
+ *
+ * We use the median absolute deviation for outlier detection as
+ * described in the following publication:
+ *
+ * Leys, C., et al., Detecting outliers: Do not use standard deviation
+ * around the mean, use absolute deviation around the median.
+ * http://dx.doi.org/10.1016/j.jesp.2013.03.013
+ *
+ * We augment the above scheme with the following heuristics to be even
+ * more conservative:
+ *
+ * 1. Skip outlier detection if the sample size is too small.
+ * 2. Never flag resources whose aggregate latency is below a low threshold.
+ * 3. Never flag resources whose aggregate latency is less than a small
+ * multiple of the median.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OutlierDetector {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(OutlierDetector.class);
+
+ /**
+ * Minimum number of resources to run outlier detection.
+ */
+ private final long minNumResources;
+
+ /**
+ * The multiplier is from Leys, C. et al.
+ */
+ private static final double MAD_MULTIPLIER = (double) 1.4826;
+
+ /**
+ * Threshold in milliseconds below which a node/ disk is definitely not slow.
+ */
+ private final long lowThresholdMs;
+
+ /**
+ * Deviation multiplier. A sample is considered to be an outlier if it
+ * exceeds the median by (multiplier * median abs. deviation). 3 is a
+ * conservative choice.
+ */
+ private static final int DEVIATION_MULTIPLIER = 3;
+
+ /**
+ * If most of the samples are clustered together, the MAD can be
+ * low. The median multiplier introduces another safeguard to avoid
+ * overaggressive outlier detection.
+ */
+ @VisibleForTesting
+ static final int MEDIAN_MULTIPLIER = 3;
+
+ public OutlierDetector(long minNumResources, long lowThresholdMs) {
+ this.minNumResources = minNumResources;
+ this.lowThresholdMs = lowThresholdMs;
+ }
+
+ /**
+ * Return a set of nodes/ disks whose latency is much higher than
+ * their counterparts. The input is a map of (resource -> aggregate latency)
+ * entries.
+ *
+ * The aggregate may be an arithmetic mean or a percentile e.g.
+ * 90th percentile. Percentiles are a better choice than median
+ * since latency is usually not a normal distribution.
+ *
+ * This method allocates temporary memory O(n) and
+ * has run time O(n.log(n)), where n = stats.size().
+ *
+ * @return
+ */
+ public Map<String, Double> getOutliers(Map<String, Double> stats) {
+ if (stats.size() < minNumResources) {
+ LOG.debug("Skipping statistical outlier detection as we don't have " +
+ "latency data for enough resources. Have {}, need at least {}",
+ stats.size(), minNumResources);
+ return ImmutableMap.of();
+ }
+ // Compute the median absolute deviation of the aggregates.
+ final List<Double> sorted = new ArrayList<>(stats.values());
+ Collections.sort(sorted);
+ final Double median = computeMedian(sorted);
+ final Double mad = computeMad(sorted);
+ Double upperLimitLatency = Math.max(
+ lowThresholdMs, median * MEDIAN_MULTIPLIER);
+ upperLimitLatency = Math.max(
+ upperLimitLatency, median + (DEVIATION_MULTIPLIER * mad));
+
+ final Map<String, Double> slowResources = new HashMap<>();
+
+ LOG.trace("getOutliers: List={}, MedianLatency={}, " +
+ "MedianAbsoluteDeviation={}, upperLimitLatency={}",
+ sorted, median, mad, upperLimitLatency);
+
+ // Find resources whose latency exceeds the threshold.
+ for (Map.Entry<String, Double> entry : stats.entrySet()) {
+ if (entry.getValue() > upperLimitLatency) {
+ slowResources.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ return slowResources;
+ }
+
+ /**
+ * Compute the Median Absolute Deviation of a sorted list.
+ */
+ public static Double computeMad(List<Double> sortedValues) {
+ if (sortedValues.size() == 0) {
+ throw new IllegalArgumentException(
+ "Cannot compute the Median Absolute Deviation " +
+ "of an empty list.");
+ }
+
+ // First get the median of the values.
+ Double median = computeMedian(sortedValues);
+ List<Double> deviations = new ArrayList<>(sortedValues);
+
+ // Then update the list to store deviation from the median.
+ for (int i = 0; i < sortedValues.size(); ++i) {
+ deviations.set(i, Math.abs(sortedValues.get(i) - median));
+ }
+
+ // Finally get the median absolute deviation.
+ Collections.sort(deviations);
+ return computeMedian(deviations) * MAD_MULTIPLIER;
+ }
+
+ /**
+ * Compute the median of a sorted list.
+ */
+ public static Double computeMedian(List<Double> sortedValues) {
+ if (sortedValues.size() == 0) {
+ throw new IllegalArgumentException(
+ "Cannot compute the median of an empty list.");
+ }
+
+ Double median = sortedValues.get(sortedValues.size() / 2);
+ if (sortedValues.size() % 2 == 0) {
+ median += sortedValues.get((sortedValues.size() / 2) - 1);
+ median /= 2;
+ }
+ return median;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/SlowNodeDetector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/SlowNodeDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/SlowNodeDetector.java
deleted file mode 100644
index b6278ce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/SlowNodeDetector.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.datanode.metrics;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-
-/**
- * A utility class to help detect nodes whose aggregate latency
- * is an outlier within a given set.
- *
- * We use the median absolute deviation for outlier detection as
- * described in the following publication:
- *
- * Leys, C., et al., Detecting outliers: Do not use standard deviation
- * around the mean, use absolute deviation around the median.
- * http://dx.doi.org/10.1016/j.jesp.2013.03.013
- *
- * We augment the above scheme with the following heuristics to be even
- * more conservative:
- *
- * 1. Skip outlier detection if the sample size is too small.
- * 2. Never flag nodes whose aggregate latency is below a low threshold.
- * 3. Never flag nodes whose aggregate latency is less than a small
- * multiple of the median.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class SlowNodeDetector {
- public static final Logger LOG =
- LoggerFactory.getLogger(SlowNodeDetector.class);
-
- /**
- * Minimum number of peers to run outlier detection.
- */
- private static long minOutlierDetectionPeers = 10;
-
- /**
- * The multiplier is from Leys, C. et al.
- */
- private static final double MAD_MULTIPLIER = (double) 1.4826;
-
- /**
- * Threshold in milliseconds below which a DataNode is definitely not slow.
- */
- private final long lowThresholdMs;
-
- /**
- * Deviation multiplier. A sample is considered to be an outlier if it
- * exceeds the median by (multiplier * median abs. deviation). 3 is a
- * conservative choice.
- */
- private static final int DEVIATION_MULTIPLIER = 3;
-
- /**
- * If most of the samples are clustered together, the MAD can be
- * low. The median multiplier introduces another safeguard to avoid
- * overaggressive outlier detection.
- */
- @VisibleForTesting
- static final int MEDIAN_MULTIPLIER = 3;
-
- public SlowNodeDetector(long lowThresholdMs) {
- this.lowThresholdMs = lowThresholdMs;
- }
-
- /**
- * Return a set of DataNodes whose latency is much higher than
- * their peers. The input is a map of (node -> aggregate latency)
- * entries.
- *
- * The aggregate may be an arithmetic mean or a percentile e.g.
- * 90th percentile. Percentiles are a better choice than median
- * since latency is usually not a normal distribution.
- *
- * This method allocates temporary memory O(n) and
- * has run time O(n.log(n)), where n = stats.size().
- *
- * @return
- */
- public Map<String, Double> getOutliers(Map<String, Double> stats) {
- if (stats.size() < minOutlierDetectionPeers) {
- LOG.debug("Skipping statistical outlier detection as we don't have " +
- "latency data for enough peers. Have {}, need at least {}",
- stats.size(), minOutlierDetectionPeers);
- return ImmutableMap.of();
- }
- // Compute the median absolute deviation of the aggregates.
- final List<Double> sorted = new ArrayList<>(stats.values());
- Collections.sort(sorted);
- final Double median = computeMedian(sorted);
- final Double mad = computeMad(sorted);
- Double upperLimitLatency = Math.max(
- lowThresholdMs, median * MEDIAN_MULTIPLIER);
- upperLimitLatency = Math.max(
- upperLimitLatency, median + (DEVIATION_MULTIPLIER * mad));
-
- final Map<String, Double> slowNodes = new HashMap<>();
-
- LOG.trace("getOutliers: List={}, MedianLatency={}, " +
- "MedianAbsoluteDeviation={}, upperLimitLatency={}",
- sorted, median, mad, upperLimitLatency);
-
- // Find nodes whose latency exceeds the threshold.
- for (Map.Entry<String, Double> entry : stats.entrySet()) {
- if (entry.getValue() > upperLimitLatency) {
- slowNodes.put(entry.getKey(), entry.getValue());
- }
- }
-
- return slowNodes;
- }
-
- /**
- * Compute the Median Absolute Deviation of a sorted list.
- */
- public static Double computeMad(List<Double> sortedValues) {
- if (sortedValues.size() == 0) {
- throw new IllegalArgumentException(
- "Cannot compute the Median Absolute Deviation " +
- "of an empty list.");
- }
-
- // First get the median of the values.
- Double median = computeMedian(sortedValues);
- List<Double> deviations = new ArrayList<>(sortedValues);
-
- // Then update the list to store deviation from the median.
- for (int i = 0; i < sortedValues.size(); ++i) {
- deviations.set(i, Math.abs(sortedValues.get(i) - median));
- }
-
- // Finally get the median absolute deviation.
- Collections.sort(deviations);
- return computeMedian(deviations) * MAD_MULTIPLIER;
- }
-
- /**
- * Compute the median of a sorted list.
- */
- public static Double computeMedian(List<Double> sortedValues) {
- if (sortedValues.size() == 0) {
- throw new IllegalArgumentException(
- "Cannot compute the median of an empty list.");
- }
-
- Double median = sortedValues.get(sortedValues.size() / 2);
- if (sortedValues.size() % 2 == 0) {
- median += sortedValues.get((sortedValues.size() / 2) - 1);
- median /= 2;
- }
- return median;
- }
-
- /**
- * This method *must not* be used outside of unit tests.
- */
- @VisibleForTesting
- static void setMinOutlierDetectionPeers(long minOutlierDetectionPeers) {
- SlowNodeDetector.minOutlierDetectionPeers = minOutlierDetectionPeers;
- }
-
- @VisibleForTesting
- static long getMinOutlierDetectionPeers() {
- return minOutlierDetectionPeers;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index c220025..36b93b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2009,7 +2009,7 @@
</property>
<property>
- <name>dfs.datanode.slow.peers.report.interval</name>
+ <name>dfs.datanode.outliers.report.interval</name>
<value>30m</value>
<description>
This setting controls how frequently DataNodes will report their peer
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
index 34e15e5..eb7769e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
@@ -54,13 +54,14 @@ public class TestDataNodeOutlierDetectionViaMetrics {
private static final int ROLLING_AVERAGE_WINDOWS = 10;
private static final int SLOW_NODE_LATENCY_MS = 20_000;
private static final int FAST_NODE_MAX_LATENCY_MS = 5;
+ private static final long MIN_OUTLIER_DETECTION_PEERS = 10;
private Random random = new Random(System.currentTimeMillis());
@Before
public void setup() {
GenericTestUtils.setLogLevel(DataNodePeerMetrics.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SlowNodeDetector.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(OutlierDetector.LOG, Level.ALL);
}
/**
@@ -111,8 +112,7 @@ public class TestDataNodeOutlierDetectionViaMetrics {
*/
public void injectFastNodesSamples(DataNodePeerMetrics peerMetrics) {
for (int nodeIndex = 0;
- nodeIndex < SlowNodeDetector.getMinOutlierDetectionPeers();
- ++nodeIndex) {
+ nodeIndex < MIN_OUTLIER_DETECTION_PEERS; ++nodeIndex) {
final String nodeName = "FastNode-" + nodeIndex;
LOG.info("Generating stats for node {}", nodeName);
for (int i = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ec531f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestSlowNodeDetector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestSlowNodeDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestSlowNodeDetector.java
index 7b368c4..f06a87a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestSlowNodeDetector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestSlowNodeDetector.java
@@ -40,7 +40,7 @@ import java.util.Set;
import static org.junit.Assert.assertTrue;
/**
- * Unit tests for {@link SlowNodeDetector}.
+ * Unit tests for {@link OutlierDetector}.
*/
public class TestSlowNodeDetector {
public static final Logger LOG =
@@ -183,7 +183,7 @@ public class TestSlowNodeDetector {
.put(ImmutableMap.of(
"n1", LOW_THRESHOLD + 0.1,
"n2", LOW_THRESHOLD + 0.1,
- "n3", LOW_THRESHOLD * SlowNodeDetector.MEDIAN_MULTIPLIER - 0.1),
+ "n3", LOW_THRESHOLD * OutlierDetector.MEDIAN_MULTIPLIER - 0.1),
ImmutableSet.of())
// A statistical outlier must be returned if it is outside a
@@ -192,7 +192,7 @@ public class TestSlowNodeDetector {
"n1", LOW_THRESHOLD + 0.1,
"n2", LOW_THRESHOLD + 0.1,
"n3", (LOW_THRESHOLD + 0.1) *
- SlowNodeDetector.MEDIAN_MULTIPLIER + 0.1),
+ OutlierDetector.MEDIAN_MULTIPLIER + 0.1),
ImmutableSet.of("n3"))
// Only the statistical outliers n3 and n11 should be returned.
@@ -233,13 +233,13 @@ public class TestSlowNodeDetector {
.build();
- private SlowNodeDetector slowNodeDetector;
+ private OutlierDetector slowNodeDetector;
@Before
public void setup() {
- slowNodeDetector = new SlowNodeDetector((long) LOW_THRESHOLD);
- SlowNodeDetector.setMinOutlierDetectionPeers(MIN_OUTLIER_DETECTION_PEERS);
- GenericTestUtils.setLogLevel(SlowNodeDetector.LOG, Level.ALL);
+ slowNodeDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_PEERS,
+ (long) LOW_THRESHOLD);
+ GenericTestUtils.setLogLevel(OutlierDetector.LOG, Level.ALL);
}
@Test
@@ -259,7 +259,7 @@ public class TestSlowNodeDetector {
}
/**
- * Unit test for {@link SlowNodeDetector#computeMedian(List)}.
+ * Unit test for {@link OutlierDetector#computeMedian(List)}.
*/
@Test
public void testMediansFromTestMatrix() {
@@ -267,7 +267,7 @@ public class TestSlowNodeDetector {
medianTestMatrix.entrySet()) {
final List<Double> inputList = new ArrayList<>(entry.getKey());
Collections.sort(inputList);
- final Double median = SlowNodeDetector.computeMedian(inputList);
+ final Double median = OutlierDetector.computeMedian(inputList);
final Double expectedMedian = entry.getValue().getLeft();
// Ensure that the median is within 0.001% of expected.
@@ -283,7 +283,7 @@ public class TestSlowNodeDetector {
}
/**
- * Unit test for {@link SlowNodeDetector#computeMad(List)}.
+ * Unit test for {@link OutlierDetector#computeMad(List)}.
*/
@Test
public void testMadsFromTestMatrix() {
@@ -291,7 +291,7 @@ public class TestSlowNodeDetector {
medianTestMatrix.entrySet()) {
final List<Double> inputList = new ArrayList<>(entry.getKey());
Collections.sort(inputList);
- final Double mad = SlowNodeDetector.computeMad(inputList);
+ final Double mad = OutlierDetector.computeMad(inputList);
final Double expectedMad = entry.getValue().getRight();
// Ensure that the MAD is within 0.001% of expected.
@@ -316,20 +316,20 @@ public class TestSlowNodeDetector {
}
/**
- * Verify that {@link SlowNodeDetector#computeMedian(List)} throws when
+ * Verify that {@link OutlierDetector#computeMedian(List)} throws when
* passed an empty list.
*/
@Test(expected=IllegalArgumentException.class)
public void testMedianOfEmptyList() {
- SlowNodeDetector.computeMedian(Collections.emptyList());
+ OutlierDetector.computeMedian(Collections.emptyList());
}
/**
- * Verify that {@link SlowNodeDetector#computeMad(List)} throws when
+ * Verify that {@link OutlierDetector#computeMad(List)} throws when
* passed an empty list.
*/
@Test(expected=IllegalArgumentException.class)
public void testMadOfEmptyList() {
- SlowNodeDetector.computeMedian(Collections.emptyList());
+ OutlierDetector.computeMedian(Collections.emptyList());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] hadoop git commit: HADOOP-13930. Azure: Add Authorization
support to WASB. Contributed by Sivaguru Sankaridurg and Dushyanth
Posted by kk...@apache.org.
HADOOP-13930. Azure: Add Authorization support to WASB. Contributed by Sivaguru Sankaridurg and Dushyanth
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b7cd62b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b7cd62b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b7cd62b
Branch: refs/heads/YARN-5972
Commit: 6b7cd62b8cf12616b13142f2eb2cfc2f25796f0f
Parents: 279d187
Author: Mingliang Liu <li...@apache.org>
Authored: Fri Mar 3 17:06:03 2017 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Fri Mar 3 17:09:12 2017 -0800
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 10 +
.../conf/TestCommonConfigurationFields.java | 2 +
.../fs/azure/AzureNativeFileSystemStore.java | 4 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 155 ++++++++++-
.../fs/azure/RemoteSASKeyGeneratorImpl.java | 183 ++++++++----
.../fs/azure/RemoteWasbAuthorizerImpl.java | 247 +++++++++++++++++
.../fs/azure/SecureStorageInterfaceImpl.java | 6 +-
.../fs/azure/WasbAuthorizationException.java | 40 +++
.../fs/azure/WasbAuthorizationOperations.java | 44 +++
.../fs/azure/WasbAuthorizerInterface.java | 47 ++++
.../hadoop/fs/azure/security/Constants.java | 54 ++++
.../security/WasbDelegationTokenIdentifier.java | 48 ++++
.../fs/azure/security/WasbTokenRenewer.java | 124 +++++++++
.../hadoop/fs/azure/security/package.html | 28 ++
...apache.hadoop.security.token.TokenIdentifier | 16 ++
...rg.apache.hadoop.security.token.TokenRenewer | 16 ++
.../hadoop-azure/src/site/markdown/index.md | 34 +++
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 102 +++++++
.../TestNativeAzureFileSystemAuthorization.java | 277 +++++++++++++++++++
19 files changed, 1373 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 35be56b..52b58ed 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1292,6 +1292,16 @@
to specify the time (such as 2s, 2m, 1h, etc.).
</description>
</property>
+<property>
+ <name>fs.azure.authorization</name>
+ <value>false</value>
+ <description>
+ Config flag to enable authorization support in WASB. Setting it to "true" enables
+ authorization support to WASB. Currently WASB authorization requires a remote service
+ to provide authorization that needs to be specified via fs.azure.authorization.remote.service.url
+ configuration
+ </description>
+</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 966a8ac..7410d29 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -181,6 +181,8 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPropsToSkipCompare.add("io.compression.codec.bzip2.library");
// - org.apache.hadoop.io.SequenceFile
xmlPropsToSkipCompare.add("io.seqfile.local.dir");
+ // - org.apache.hadoop.fs.azure.NativeAzureFileSystem
+ xmlPropsToSkipCompare.add("fs.azure.authorization");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 07c389c..9d7ac80 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -303,7 +303,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
private boolean useSecureMode = false;
private boolean useLocalSasKeyMode = false;
- private String delegationToken;
+
/**
* A test hook interface that can modify the operation context we use for
* Azure Storage operations, e.g. to inject errors.
@@ -478,7 +478,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
this.storageInteractionLayer = new StorageInterfaceImpl();
} else {
this.storageInteractionLayer = new SecureStorageInterfaceImpl(
- useLocalSasKeyMode, conf, delegationToken);
+ useLocalSasKeyMode, conf);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index b742e53..0dfefaf 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -25,9 +25,12 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URL;
import java.nio.charset.Charset;
+import java.security.PrivilegedExceptionAction;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -61,10 +64,15 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
+import org.apache.hadoop.fs.azure.security.Constants;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
+import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
@@ -1096,7 +1104,39 @@ public class NativeAzureFileSystem extends FileSystem {
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
private boolean appendSupportEnabled = false;
-
+ private DelegationTokenAuthenticatedURL authURL;
+ private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
+ private String credServiceUrl;
+
+ /**
+ * Configuration key to enable authorization support in WASB.
+ */
+ public static final String KEY_AZURE_AUTHORIZATION =
+ "fs.azure.authorization";
+
+ /**
+ * Default value for the authorization support in WASB.
+ */
+ private static final boolean DEFAULT_AZURE_AUTHORIZATION = false;
+
+ /**
+ * Flag controlling authorization support in WASB.
+ */
+ private boolean azureAuthorization = false;
+
+ /**
+ * Flag controlling Kerberos support in WASB.
+ */
+ private boolean kerberosSupportEnabled = false;
+
+ /**
+ * Authorizer to use when authorization support is enabled in
+ * WASB.
+ */
+ private WasbAuthorizerInterface authorizer = null;
+
+ private String delegationToken = null;
+
public NativeAzureFileSystem() {
// set store in initialize()
}
@@ -1227,6 +1267,31 @@ public class NativeAzureFileSystem extends FileSystem {
// Initialize thread counts from user configuration
deleteThreadCount = conf.getInt(AZURE_DELETE_THREADS, DEFAULT_AZURE_DELETE_THREADS);
renameThreadCount = conf.getInt(AZURE_RENAME_THREADS, DEFAULT_AZURE_RENAME_THREADS);
+
+ this.azureAuthorization = conf.getBoolean(KEY_AZURE_AUTHORIZATION,
+ DEFAULT_AZURE_AUTHORIZATION);
+ this.kerberosSupportEnabled = conf.getBoolean(
+ Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
+
+ if (this.azureAuthorization) {
+
+ this.authorizer =
+ new RemoteWasbAuthorizerImpl();
+ authorizer.init(conf);
+ }
+ if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
+ DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator();
+ authURL = new DelegationTokenAuthenticatedURL(authenticator);
+ credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, String
+ .format("http://%s:%s",
+ InetAddress.getLocalHost().getCanonicalHostName(),
+ Constants.DEFAULT_CRED_SERVICE_PORT));
+ }
+ }
+
+ @VisibleForTesting
+ public void updateWasbAuthorizer(WasbAuthorizerInterface authorizer) {
+ this.authorizer = authorizer;
}
private NativeFileSystemStore createDefaultStore(Configuration conf) {
@@ -1340,6 +1405,15 @@ public class NativeAzureFileSystem extends FileSystem {
return store;
}
+ private void performAuthCheck(String path, String accessType,
+ String operation) throws WasbAuthorizationException, IOException {
+
+ if (azureAuthorization && !this.authorizer.authorize(path, accessType)) {
+ throw new WasbAuthorizationException(operation
+ + " operation for Path : " + path + " not allowed");
+ }
+ }
+
/**
* Gets the metrics source for this file system.
* This is mainly here for unit testing purposes.
@@ -1362,6 +1436,10 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Opening file: {} for append", f);
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.WRITE.toString(), "append");
+
String key = pathToKey(absolutePath);
FileMetadata meta = null;
try {
@@ -1562,6 +1640,10 @@ public class NativeAzureFileSystem extends FileSystem {
}
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.WRITE.toString(), "create");
+
String key = pathToKey(absolutePath);
FileMetadata existingMetadata = store.retrieveMetadata(key);
@@ -1684,6 +1766,10 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Deleting file: {}", f.toString());
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "delete");
+
String key = pathToKey(absolutePath);
// Capture the metadata for the path.
@@ -1954,6 +2040,10 @@ public class NativeAzureFileSystem extends FileSystem {
// Capture the absolute path and the path to key.
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "getFileStatus");
+
String key = pathToKey(absolutePath);
if (key.length() == 0) { // root always exists
return newDirectory(null, absolutePath);
@@ -2052,6 +2142,10 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Listing status for {}", f.toString());
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "list");
+
String key = pathToKey(absolutePath);
Set<FileStatus> status = new TreeSet<FileStatus>();
FileMetadata meta = null;
@@ -2274,6 +2368,10 @@ public class NativeAzureFileSystem extends FileSystem {
}
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "mkdirs");
+
PermissionStatus permissionStatus = null;
if(noUmask) {
// ensure owner still has wx permissions at the minimum
@@ -2327,6 +2425,10 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Opening file: {}", f.toString());
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.READ.toString(), "read");
+
String key = pathToKey(absolutePath);
FileMetadata meta = null;
try {
@@ -2383,7 +2485,12 @@ public class NativeAzureFileSystem extends FileSystem {
+ " through WASB that has colons in the name");
}
- String srcKey = pathToKey(makeAbsolute(src));
+ Path absolutePath = makeAbsolute(src);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "rename");
+
+ String srcKey = pathToKey(absolutePath);
if (srcKey.length() == 0) {
// Cannot rename root of file system
@@ -2685,6 +2792,10 @@ public class NativeAzureFileSystem extends FileSystem {
@Override
public void setPermission(Path p, FsPermission permission) throws FileNotFoundException, IOException {
Path absolutePath = makeAbsolute(p);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "setPermission");
+
String key = pathToKey(absolutePath);
FileMetadata metadata = null;
try {
@@ -2723,6 +2834,10 @@ public class NativeAzureFileSystem extends FileSystem {
public void setOwner(Path p, String username, String groupname)
throws IOException {
Path absolutePath = makeAbsolute(p);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "setOwner");
+
String key = pathToKey(absolutePath);
FileMetadata metadata = null;
@@ -2785,6 +2900,42 @@ public class NativeAzureFileSystem extends FileSystem {
isClosed = true;
}
+ @Override
+ public Token<?> getDelegationToken(final String renewer) throws IOException {
+ if(kerberosSupportEnabled) {
+ try {
+ final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ final UserGroupInformation proxyUser = connectUgi;
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ }
+ if(!connectUgi.hasKerberosCredentials()){
+ connectUgi = UserGroupInformation.getLoginUser();
+ }
+ connectUgi.checkTGTAndReloginFromKeytab();
+ return connectUgi.doAs(new PrivilegedExceptionAction<Token<?>>() {
+ @Override
+ public Token<?> run() throws Exception {
+ return authURL.getDelegationToken(new URL(credServiceUrl
+ + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT),
+ authToken, renewer, (proxyUser != null)? ugi.getShortUserName(): null);
+ }
+ });
+ } catch (Exception ex) {
+ LOG.error("Error in fetching the delegation token from remote service",
+ ex);
+ if (ex instanceof IOException) {
+ throw (IOException) ex;
+ } else {
+ throw new IOException(ex);
+ }
+ }
+ } else {
+ return super.getDelegationToken(renewer);
+ }
+ }
+
/**
* A handler that defines what to do with blobs whose upload was
* interrupted.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
index 404419d..43672b2 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
@@ -19,10 +19,23 @@
package org.apache.hadoop.fs.azure;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Iterator;
+import org.apache.commons.lang.Validate;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azure.security.Constants;
+import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.Authenticator;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
@@ -43,12 +56,6 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
public static final Logger LOG =
LoggerFactory.getLogger(AzureNativeFileSystemStore.class);
- /**
- * Configuration parameter name expected in the Configuration
- * object to provide the url of the remote service {@value}
- */
- private static final String KEY_CRED_SERVICE_URL =
- "fs.azure.cred.service.url";
/**
* Container SAS Key generation OP name. {@value}
@@ -82,7 +89,7 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
* Query parameter name for user info {@value}
*/
private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME =
- "delegation_token";
+ "delegation";
/**
* Query parameter name for the relative path inside the storage
@@ -94,24 +101,40 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
private String delegationToken = "";
private String credServiceUrl = "";
private WasbRemoteCallHelper remoteCallHelper = null;
+ private boolean isSecurityEnabled;
+ private boolean isKerberosSupportEnabled;
public RemoteSASKeyGeneratorImpl(Configuration conf) {
super(conf);
}
- public boolean initialize(Configuration conf, String delegationToken) {
+ public boolean initialize(Configuration conf) {
LOG.debug("Initializing RemoteSASKeyGeneratorImpl instance");
- credServiceUrl = conf.get(KEY_CRED_SERVICE_URL);
+ Iterator<Token<? extends TokenIdentifier>> tokenIterator = null;
+ try {
+ tokenIterator = UserGroupInformation.getCurrentUser().getCredentials()
+ .getAllTokens().iterator();
+ while (tokenIterator.hasNext()) {
+ Token<? extends TokenIdentifier> iteratedToken = tokenIterator.next();
+ if (iteratedToken.getKind().equals(WasbDelegationTokenIdentifier.TOKEN_KIND)) {
+ delegationToken = iteratedToken.encodeToUrlString();
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("Error in fetching the WASB delegation token");
+ }
- if (delegationToken == null || delegationToken.isEmpty()) {
- LOG.error("Delegation Token not provided for initialization"
- + " of RemoteSASKeyGenerator");
+ try {
+ credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, String
+ .format("http://%s:%s",
+ InetAddress.getLocalHost().getCanonicalHostName(),
+ Constants.DEFAULT_CRED_SERVICE_PORT));
+ } catch (UnknownHostException e) {
+ LOG.error("Invalid CredService Url, configure it correctly.");
return false;
}
- this.delegationToken = delegationToken;
-
if (credServiceUrl == null || credServiceUrl.isEmpty()) {
LOG.error("CredService Url not found in configuration to initialize"
+ " RemoteSASKeyGenerator");
@@ -119,16 +142,17 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
}
remoteCallHelper = new WasbRemoteCallHelper();
- LOG.debug("Initialization of RemoteSASKeyGenerator instance successfull");
+ this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
+ this.isKerberosSupportEnabled = conf.getBoolean(
+ Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
+ LOG.debug("Initialization of RemoteSASKeyGenerator instance successful");
return true;
}
@Override
public URI getContainerSASUri(String storageAccount, String container)
throws SASKeyGenerationException {
-
try {
-
LOG.debug("Generating Container SAS Key for Container {} "
+ "inside Storage Account {} ", container, storageAccount);
URIBuilder uriBuilder = new URIBuilder(credServiceUrl);
@@ -139,84 +163,131 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
container);
uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME, ""
+ getSasKeyExpiryPeriod());
- uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
- this.delegationToken);
-
- RemoteSASKeyGenerationResponse sasKeyResponse =
- makeRemoteRequest(uriBuilder.build());
-
- if (sasKeyResponse == null) {
- throw new SASKeyGenerationException("RemoteSASKeyGenerationResponse"
- + " object null from remote call");
- } else if (sasKeyResponse.getResponseCode()
- == REMOTE_CALL_SUCCESS_CODE) {
- return new URI(sasKeyResponse.getSasKey());
+ if (isSecurityEnabled && (delegationToken != null && !delegationToken
+ .isEmpty())) {
+ uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
+ this.delegationToken);
+ }
+
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ if (connectUgi == null) {
+ connectUgi = ugi;
} else {
- throw new SASKeyGenerationException("Remote Service encountered error"
- + " in SAS Key generation : "
- + sasKeyResponse.getResponseMessage());
+ uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName());
+ }
+
+ if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){
+ connectUgi = UserGroupInformation.getLoginUser();
}
+ return getSASKey(uriBuilder.build(), connectUgi);
} catch (URISyntaxException uriSyntaxEx) {
throw new SASKeyGenerationException("Encountered URISyntaxException "
+ "while building the HttpGetRequest to remote cred service",
uriSyntaxEx);
+ } catch (IOException e) {
+ throw new SASKeyGenerationException("Encountered IOException"
+ + " while building the HttpGetRequest to remote service", e);
}
}
@Override
public URI getRelativeBlobSASUri(String storageAccount, String container,
String relativePath) throws SASKeyGenerationException {
-
try {
-
LOG.debug("Generating RelativePath SAS Key for relativePath {} inside"
+ " Container {} inside Storage Account {} ",
relativePath, container, storageAccount);
URIBuilder uriBuilder = new URIBuilder(credServiceUrl);
uriBuilder.setPath("/" + BLOB_SAS_OP);
- uriBuilder.addParameter(STORAGE_ACCOUNT_QUERY_PARAM_NAME,
- storageAccount);
- uriBuilder.addParameter(CONTAINER_QUERY_PARAM_NAME,
- container);
+ uriBuilder.addParameter(STORAGE_ACCOUNT_QUERY_PARAM_NAME, storageAccount);
+ uriBuilder.addParameter(CONTAINER_QUERY_PARAM_NAME, container);
uriBuilder.addParameter(RELATIVE_PATH_QUERY_PARAM_NAME,
relativePath);
uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME, ""
+ getSasKeyExpiryPeriod());
- uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
- this.delegationToken);
-
- RemoteSASKeyGenerationResponse sasKeyResponse =
- makeRemoteRequest(uriBuilder.build());
-
- if (sasKeyResponse == null) {
- throw new SASKeyGenerationException("RemoteSASKeyGenerationResponse"
- + " object null from remote call");
- } else if (sasKeyResponse.getResponseCode()
- == REMOTE_CALL_SUCCESS_CODE) {
- return new URI(sasKeyResponse.getSasKey());
- } else {
- throw new SASKeyGenerationException("Remote Service encountered error"
- + " in SAS Key generation : "
- + sasKeyResponse.getResponseMessage());
+
+ if (isSecurityEnabled && (delegationToken != null && !delegationToken
+ .isEmpty())) {
+ uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
+ this.delegationToken);
+ }
+
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ } else{
+ uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName());
+ }
+
+ if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){
+ connectUgi = UserGroupInformation.getLoginUser();
}
+ return getSASKey(uriBuilder.build(), connectUgi);
} catch (URISyntaxException uriSyntaxEx) {
throw new SASKeyGenerationException("Encountered URISyntaxException"
+ " while building the HttpGetRequest to " + " remote service",
uriSyntaxEx);
+ } catch (IOException e) {
+ throw new SASKeyGenerationException("Encountered IOException"
+ + " while building the HttpGetRequest to remote service", e);
+ }
+ }
+
+ private URI getSASKey(final URI uri, UserGroupInformation connectUgi)
+ throws URISyntaxException, SASKeyGenerationException {
+ RemoteSASKeyGenerationResponse sasKeyResponse = null;
+ try {
+ connectUgi.checkTGTAndReloginFromKeytab();
+ sasKeyResponse = connectUgi.doAs(new PrivilegedExceptionAction<RemoteSASKeyGenerationResponse>() {
+ @Override
+ public RemoteSASKeyGenerationResponse run() throws Exception {
+ AuthenticatedURL.Token token = null;
+ if (isKerberosSupportEnabled && UserGroupInformation.isSecurityEnabled() && (
+ delegationToken == null || delegationToken.isEmpty())) {
+ token = new AuthenticatedURL.Token();
+ final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator();
+ kerberosAuthenticator.authenticate(uri.toURL(), token);
+ Validate.isTrue(token.isSet(),
+ "Authenticated Token is NOT present. The request cannot proceed.");
+ }
+ return makeRemoteRequest(uri, (token != null ? token.toString() : null));
+ }
+ });
+ } catch (InterruptedException e) {
+ LOG.error("Error fetching the SAS Key from Remote Service", e);
+ } catch (IOException e) {
+ LOG.error("Error fetching the SAS Key from Remote Service", e);
+ }
+
+ if (sasKeyResponse == null) {
+ throw new SASKeyGenerationException(
+ "RemoteSASKeyGenerationResponse" + " object null from remote call");
+ } else if (sasKeyResponse.getResponseCode() == REMOTE_CALL_SUCCESS_CODE) {
+ return new URI(sasKeyResponse.getSasKey());
+ } else {
+ throw new SASKeyGenerationException("Remote Service encountered error"
+ + " in SAS Key generation : " + sasKeyResponse.getResponseMessage());
}
}
/**
* Helper method to make a remote request.
* @param uri - Uri to use for the remote request
+ * @param token - hadoop.auth token for the remote request
* @return RemoteSASKeyGenerationResponse
*/
- private RemoteSASKeyGenerationResponse makeRemoteRequest(URI uri)
+ private RemoteSASKeyGenerationResponse makeRemoteRequest(URI uri, String token)
throws SASKeyGenerationException {
try {
+ HttpGet httpGet = new HttpGet(uri);
+ if(token != null){
+ httpGet.setHeader("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + token);
+ }
String responseBody =
- remoteCallHelper.makeRemoteGetRequest(new HttpGet(uri));
+ remoteCallHelper.makeRemoteGetRequest(httpGet);
ObjectMapper objectMapper = new ObjectMapper();
return objectMapper.readValue(responseBody,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
new file mode 100644
index 0000000..bb1093d
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Iterator;
+
+import org.apache.commons.lang.Validate;
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.fs.azure.security.Constants;
+import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.Authenticator;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.azure.WasbRemoteCallHelper.REMOTE_CALL_SUCCESS_CODE;
+
+/**
+ * Class implementing WasbAuthorizerInterface using a remote
+ * service that implements the authorization operation. This
+ * class expects the url of the remote service to be passed
+ * via config.
+ */
+public class RemoteWasbAuthorizerImpl implements WasbAuthorizerInterface {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(RemoteWasbAuthorizerImpl.class);
+ private String remoteAuthorizerServiceUrl = "";
+
+ /**
+ * Configuration parameter name expected in the Configuration object to
+ * provide the url of the remote service. {@value}
+ */
+ public static final String KEY_REMOTE_AUTH_SERVICE_URL =
+ "fs.azure.authorization.remote.service.url";
+
+ /**
+ * Authorization operation OP name in the remote service {@value}
+ */
+ private static final String CHECK_AUTHORIZATION_OP =
+ "CHECK_AUTHORIZATION";
+
+ /**
+ * Query parameter specifying the access operation type. {@value}
+ */
+ private static final String ACCESS_OPERATION_QUERY_PARAM_NAME =
+ "operation_type";
+
+ /**
+ * Query parameter specifying the wasb absolute path. {@value}
+ */
+ private static final String WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME =
+ "wasb_absolute_path";
+
+ /**
+ * Query parameter name for user info {@value}
+ */
+ private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME =
+ "delegation";
+
+ private WasbRemoteCallHelper remoteCallHelper = null;
+ private String delegationToken;
+ private boolean isSecurityEnabled;
+ private boolean isKerberosSupportEnabled;
+
+ @Override
+ public void init(Configuration conf)
+ throws WasbAuthorizationException, IOException {
+ LOG.debug("Initializing RemoteWasbAuthorizerImpl instance");
+ delegationToken = UserGroupInformation.getCurrentUser().getCredentials().getToken(WasbDelegationTokenIdentifier.TOKEN_KIND).encodeToUrlString();
+
+ remoteAuthorizerServiceUrl = conf.get(KEY_REMOTE_AUTH_SERVICE_URL, String
+ .format("http://%s:%s",
+ InetAddress.getLocalHost().getCanonicalHostName(),
+ Constants.DEFAULT_CRED_SERVICE_PORT));
+
+ if (remoteAuthorizerServiceUrl == null
+ || remoteAuthorizerServiceUrl.isEmpty()) {
+ throw new WasbAuthorizationException(
+ "fs.azure.authorization.remote.service.url config not set"
+ + " in configuration.");
+ }
+
+ this.remoteCallHelper = new WasbRemoteCallHelper();
+ this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
+ this.isKerberosSupportEnabled = conf.getBoolean(
+ Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
+ }
+
+ @Override
+ public boolean authorize(String wasbAbsolutePath, String accessType)
+ throws WasbAuthorizationException, IOException {
+ try {
+ final URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl);
+ uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP);
+ uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME,
+ wasbAbsolutePath);
+ uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME,
+ accessType);
+ if (isSecurityEnabled && (delegationToken != null && !delegationToken
+ .isEmpty())) {
+ uriBuilder
+ .addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, delegationToken);
+ }
+ String responseBody = null;
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ } else{
+ uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName());
+ }
+ if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){
+ connectUgi = UserGroupInformation.getLoginUser();
+ }
+ connectUgi.checkTGTAndReloginFromKeytab();
+ try {
+ responseBody = connectUgi.doAs(new PrivilegedExceptionAction<String>(){
+ @Override
+ public String run() throws Exception {
+ AuthenticatedURL.Token token = null;
+ HttpGet httpGet = new HttpGet(uriBuilder.build());
+ if (isKerberosSupportEnabled && UserGroupInformation.isSecurityEnabled() && (
+ delegationToken == null || delegationToken.isEmpty())) {
+ token = new AuthenticatedURL.Token();
+ final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator();
+ kerberosAuthenticator
+ .authenticate(uriBuilder.build().toURL(), token);
+ Validate.isTrue(token.isSet(),
+ "Authenticated Token is NOT present. The request cannot proceed.");
+ if(token != null){
+ httpGet.setHeader("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + token);
+ }
+ }
+ return remoteCallHelper.makeRemoteGetRequest(httpGet);
+ }});
+ } catch (InterruptedException e) {
+ LOG.error("Error in check authorization", e);
+ }
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ RemoteAuthorizerResponse authorizerResponse =
+ objectMapper.readValue(responseBody, RemoteAuthorizerResponse.class);
+
+ if (authorizerResponse == null) {
+ throw new WasbAuthorizationException(
+ "RemoteAuthorizerResponse object null from remote call");
+ } else if (authorizerResponse.getResponseCode()
+ == REMOTE_CALL_SUCCESS_CODE) {
+ return authorizerResponse.getAuthorizationResult();
+ } else {
+ throw new WasbAuthorizationException("Remote authorization"
+ + " serivce encountered an error "
+ + authorizerResponse.getResponseMessage());
+ }
+ } catch (URISyntaxException | WasbRemoteCallException
+ | JsonParseException | JsonMappingException ex) {
+ throw new WasbAuthorizationException(ex);
+ }
+ }
+}
+
+/**
+ * POJO representing the response expected from a remote
+ * authorization service.
+ * The remote service is expected to return the authorization
+ * response in the following JSON format
+ * {
+ * "responseCode" : 0 or non-zero <int>,
+ * "responseMessage" : relavant message of failure <String>
+ * "authorizationResult" : authorization result <boolean>
+ * true - if auhorization allowed
+ * false - otherwise.
+ *
+ * }
+ */
+class RemoteAuthorizerResponse {
+
+ private int responseCode;
+ private boolean authorizationResult;
+ private String responseMessage;
+
+ public RemoteAuthorizerResponse(){
+ }
+
+ public RemoteAuthorizerResponse(int responseCode,
+ boolean authorizationResult, String message) {
+ this.responseCode = responseCode;
+ this.authorizationResult = authorizationResult;
+ this.responseMessage = message;
+ }
+
+ public int getResponseCode() {
+ return responseCode;
+ }
+
+ public void setResponseCode(int responseCode) {
+ this.responseCode = responseCode;
+ }
+
+ public boolean getAuthorizationResult() {
+ return authorizationResult;
+ }
+
+ public void setAuthorizationResult(boolean authorizationResult) {
+ this.authorizationResult = authorizationResult;
+ }
+
+ public String getResponseMessage() {
+ return responseMessage;
+ }
+
+ public void setResponseMessage(String message) {
+ this.responseMessage = message;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
index 6749a76..5ec9136 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
@@ -69,19 +69,17 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
public static final String SAS_ERROR_CODE = "SAS Error";
private SASKeyGeneratorInterface sasKeyGenerator;
private String storageAccount;
- private String delegationToken;
public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode,
- Configuration conf, String delegationToken)
+ Configuration conf)
throws SecureModeException {
- this.delegationToken = delegationToken;
if (useLocalSASKeyMode) {
this.sasKeyGenerator = new LocalSASKeyGeneratorImpl(conf);
} else {
RemoteSASKeyGeneratorImpl remoteSasKeyGenerator =
new RemoteSASKeyGeneratorImpl(conf);
- if (!remoteSasKeyGenerator.initialize(conf, this.delegationToken)) {
+ if (!remoteSasKeyGenerator.initialize(conf)) {
throw new SecureModeException("Remote SAS Key mode could"
+ " not be initialized");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
new file mode 100644
index 0000000..eff9248
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+/**
+ * Exception that gets thrown during the authorization failures
+ * in WASB.
+ */
+public class WasbAuthorizationException extends AzureException {
+
+ private static final long serialVersionUID = 1L;
+
+ public WasbAuthorizationException(String message) {
+ super(message);
+ }
+
+ public WasbAuthorizationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public WasbAuthorizationException(Throwable t) {
+ super(t);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
new file mode 100644
index 0000000..bd76837
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+/**
+ * Different authorization operations supported
+ * in WASB.
+ */
+
+public enum WasbAuthorizationOperations {
+
+ READ, WRITE, EXECUTE;
+
+ @Override
+ public String toString() {
+ switch(this) {
+ case READ:
+ return "read";
+ case WRITE:
+ return "write";
+ case EXECUTE:
+ return "execute";
+ default:
+ throw new IllegalArgumentException(
+ "Invalid Auhtorization Operation");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
new file mode 100644
index 0000000..0c61997
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Interface to implement authorization support in WASB.
+ * API's of this interface will be implemented in the
+ * StorageInterface Layer before making calls to Azure
+ * Storage.
+ */
+public interface WasbAuthorizerInterface {
+ /**
+ * Initializer method
+ * @param conf - Configuration object
+ */
+ public void init(Configuration conf)
+ throws WasbAuthorizationException, IOException;
+
+ /**
+ * Authorizer API to authorize access in WASB.
+ * @param wasbAbsolutePath : Absolute WASB Path used for access.
+ * @param accessType : Type of access
+ * @return : true - If access allowed false - If access is not allowed.
+ */
+ public boolean authorize(String wasbAbsolutePath, String accessType)
+ throws WasbAuthorizationException, IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
new file mode 100644
index 0000000..1050687
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure.security;
+
+/**
+ * Constants for used with WASB security implementation.
+ */
+public final class Constants {
+
+ private Constants() {
+ }
+
+ /**
+ * Configuration parameter name expected in the Configuration
+ * object to provide the url of the remote service {@value}
+ */
+ public static final String KEY_CRED_SERVICE_URL = "fs.azure.cred.service.url";
+ /**
+ * Default port of the remote service used as delegation token manager and Azure storage SAS key generator.
+ */
+ public static final int DEFAULT_CRED_SERVICE_PORT = 50911;
+
+ /**
+ * Default remote delegation token manager endpoint.
+ */
+ public static final String DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT = "/tokenmanager/v1";
+
+ /**
+ * The configuration property to enable Kerberos support.
+ */
+
+ public static final String AZURE_KERBEROS_SUPPORT_PROPERTY_NAME = "fs.azure.enable.kerberos.support";
+
+ /**
+ * Parameter to be used for impersonation.
+ */
+ public static final String DOAS_PARAM="doas";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java
new file mode 100644
index 0000000..530e045
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure.security;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
+
+/**
+ * Delegation token Identifier for WASB delegation tokens.
+ */
+public class WasbDelegationTokenIdentifier extends DelegationTokenIdentifier {
+ public static final Text TOKEN_KIND = new Text("WASB delegation");
+
+ public WasbDelegationTokenIdentifier(){
+ super(TOKEN_KIND);
+ }
+
+ public WasbDelegationTokenIdentifier(Text kind) {
+ super(kind);
+ }
+
+ public WasbDelegationTokenIdentifier(Text kind, Text owner, Text renewer,
+ Text realUser) {
+ super(kind, owner, renewer, realUser);
+ }
+
+ @Override
+ public Text getKind() {
+ return TOKEN_KIND;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java
new file mode 100644
index 0000000..09b7349
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure.security;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
+import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URL;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * Token Renewer for renewing WASB delegation tokens with remote service.
+ */
+public class WasbTokenRenewer extends TokenRenewer {
+ public static final Logger LOG = LoggerFactory
+ .getLogger(WasbTokenRenewer.class);
+
+ @Override
+ public boolean handleKind(Text kind) {
+ return WasbDelegationTokenIdentifier.TOKEN_KIND.equals(kind);
+ }
+
+ @Override
+ public boolean isManaged(Token<?> token) throws IOException {
+ return true;
+ }
+
+ @Override
+ public long renew(final Token<?> token, Configuration conf)
+ throws IOException, InterruptedException {
+ LOG.debug("Renewing the delegation token");
+ final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ final UserGroupInformation proxyUser = connectUgi;
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ }
+ if(!connectUgi.hasKerberosCredentials()){
+ connectUgi = UserGroupInformation.getLoginUser();
+ }
+ connectUgi.checkTGTAndReloginFromKeytab();
+ final DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
+ authToken
+ .setDelegationToken((Token<AbstractDelegationTokenIdentifier>) token);
+ final String credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL,
+ String.format("http://%s:%s",
+ InetAddress.getLocalHost().getCanonicalHostName(),
+ Constants.DEFAULT_CRED_SERVICE_PORT));
+ DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator();
+ final DelegationTokenAuthenticatedURL authURL = new DelegationTokenAuthenticatedURL(
+ authenticator);
+
+ return connectUgi.doAs(new PrivilegedExceptionAction<Long>() {
+ @Override
+ public Long run() throws Exception {
+ return authURL.renewDelegationToken(new URL(credServiceUrl
+ + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT),
+ authToken, (proxyUser != null) ? ugi.getShortUserName() : null);
+ }
+ });
+ }
+
+ @Override
+ public void cancel(final Token<?> token, Configuration conf)
+ throws IOException, InterruptedException {
+ LOG.debug("Cancelling the delegation token");
+ final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ final UserGroupInformation proxyUser = connectUgi;
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ }
+ if(!connectUgi.hasKerberosCredentials()){
+ connectUgi = UserGroupInformation.getLoginUser();
+ }
+ connectUgi.checkTGTAndReloginFromKeytab();
+ final DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
+ authToken
+ .setDelegationToken((Token<AbstractDelegationTokenIdentifier>) token);
+ final String credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL,
+ String.format("http://%s:%s",
+ InetAddress.getLocalHost().getCanonicalHostName(),
+ Constants.DEFAULT_CRED_SERVICE_PORT));
+ DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator();
+ final DelegationTokenAuthenticatedURL authURL = new DelegationTokenAuthenticatedURL(
+ authenticator);
+ connectUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ authURL.cancelDelegationToken(new URL(credServiceUrl
+ + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT),
+ authToken, (proxyUser != null) ? ugi.getShortUserName() : null);
+ return null;
+ }
+ });
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html
new file mode 100644
index 0000000..fe58c0a
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html
@@ -0,0 +1,28 @@
+<html>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<body>
+
+<p>
+ Infrastructure for WASB client Security to work with Kerberos and Delegation
+ tokens.
+</p>
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
new file mode 100644
index 0000000..7ec8216
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
new file mode 100644
index 0000000..f9c590a
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.fs.azure.security.WasbTokenRenewer
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index 2865223..a06d297 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -333,6 +333,40 @@ The service is expected to return a response in JSON format:
"sasKey" : Requested SAS Key <String>
}
```
+
+## <a name="WASB Authorization" />Authorization Support in WASB.
+
+Authorization support can be enabled in WASB using the following configuration:
+
+```
+ <property>
+ <name>fs.azure.authorization</name>
+ <value>true</value>
+ </property>
+```
+ The current implementation of authorization relies on the presence of an external service that can enforce
+ the authorization. The service is expected to be running on a URL provided by the following config.
+
+```
+ <property>
+ <name>fs.azure.authorization.remote.service.url</name>
+ <value>{URL}</value>
+ </property>
+```
+
+ The remote service is expected to provide support for the following REST call: ```{URL}/CHECK_AUTHORIZATION```
+ An example request:
+ ```{URL}/CHECK_AUTHORIZATION?wasb_absolute_path=<absolute_path>&operation_type=<operation type>&delegation_token=<delegation token>```
+
+ The service is expected to return a response in JSON format:
+ ```
+ {
+ "responseCode" : 0 or non-zero <int>,
+ "responseMessage" : relavant message on failure <String>,
+ "authorizationResult" : true/false <boolean>
+ }
+ ```
+
## <a name="Testing_the_hadoop-azure_Module" />Testing the hadoop-azure Module
The hadoop-azure module includes a full suite of unit tests. Most of the tests
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
new file mode 100644
index 0000000..af5a537
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * A mock wasb authorizer implementation.
+ */
+
+public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface {
+
+ private Map<AuthorizationComponent, Boolean> authRules;
+
+ @Override
+ public void init(Configuration conf) {
+ authRules = new HashMap<AuthorizationComponent, Boolean>();
+ }
+
+ public void addAuthRule(String wasbAbsolutePath,
+ String accessType, boolean access) {
+ AuthorizationComponent component =
+ new AuthorizationComponent(wasbAbsolutePath, accessType);
+ this.authRules.put(component, access);
+ }
+
+ @Override
+ public boolean authorize(String wasbAbsolutePath, String accessType)
+ throws WasbAuthorizationException {
+
+ AuthorizationComponent component =
+ new AuthorizationComponent(wasbAbsolutePath, accessType);
+
+ if (authRules.containsKey(component)) {
+ return authRules.get(component);
+ } else {
+ return false;
+ }
+ }
+}
+
+class AuthorizationComponent {
+
+ private String wasbAbsolutePath;
+ private String accessType;
+
+ public AuthorizationComponent(String wasbAbsolutePath,
+ String accessType) {
+ this.wasbAbsolutePath = wasbAbsolutePath;
+ this.accessType = accessType;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.wasbAbsolutePath.hashCode() ^ this.accessType.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj == null
+ || !(obj instanceof AuthorizationComponent)) {
+ return false;
+ }
+
+ return ((AuthorizationComponent)obj).
+ getWasbAbsolutePath().equals(this.wasbAbsolutePath)
+ && ((AuthorizationComponent)obj).
+ getAccessType().equals(this.accessType);
+ }
+
+ public String getWasbAbsolutePath() {
+ return this.wasbAbsolutePath;
+ }
+
+ public String getAccessType() {
+ return accessType;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7cd62b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
new file mode 100644
index 0000000..e765335
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
@@ -0,0 +1,277 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+import com.sun.tools.javac.util.Assert;
+
+/**
+ * Test class to hold all WASB authorization tests.
+ */
+public class TestNativeAzureFileSystemAuthorization
+ extends AbstractWasbTestBase {
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true");
+ conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URL, "test_url");
+ return AzureBlobStorageTestAccount.create(conf);
+ }
+
+ /**
+ * Positive test to verify Create and delete access check
+ * @throws Throwable
+ */
+ @Test
+ public void testCreateAccessCheckPositive() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ String testFile = "test.dat";
+ Path testPath = new Path(fs.getWorkingDirectory(), testFile);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+ authorizer.addAuthRule(fs.getWorkingDirectory().toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), true);
+
+ fs.create(testPath);
+ Assert.check(fs.exists(testPath));
+ fs.delete(testPath, false);
+ }
+
+ /**
+ * Negative test to verify Create access check
+ * @throws Throwable
+ */
+
+ @Test(expected=WasbAuthorizationException.class)
+ public void testCreateAccessCheckNegative() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ String testFile = "test.dat";
+ Path testPath = new Path(fs.getWorkingDirectory(), testFile);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.WRITE.toString(), false);
+ fs.updateWasbAuthorizer(authorizer);
+
+ fs.create(new Path(testFile));
+ }
+
+ /**
+ * Positive test to verify Create and delete access check
+ * @throws Throwable
+ */
+ @Test
+ public void testListAccessCheckPositive() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ String testFolder = "\\";
+ Path testPath = new Path(fs.getWorkingDirectory(), testFolder);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ fs.listStatus(testPath);
+ }
+
+ /**
+ * Negative test to verify Create access check
+ * @throws Throwable
+ */
+
+ @Test(expected=WasbAuthorizationException.class)
+ public void testListAccessCheckNegative() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ String testFolder = "\\";
+ Path testPath = new Path(fs.getWorkingDirectory(), testFolder);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), false);
+ fs.updateWasbAuthorizer(authorizer);
+
+ fs.listStatus(testPath);
+ }
+
+ /**
+ * Positive test to verify rename access check.
+ * @throws Throwable
+ */
+ @Test
+ public void testRenameAccessCheckPositive() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ String testFile = "test.dat";
+ Path testPath = new Path(fs.getWorkingDirectory(), testFile);
+ String renameFile = "test2.dat";
+ Path renamePath = new Path(fs.getWorkingDirectory(), renameFile);
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(renamePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(fs.getWorkingDirectory().toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+ fs.create(testPath);
+
+ Assert.check(fs.exists(testPath));
+ fs.rename(testPath, renamePath);
+ Assert.check(fs.exists(renamePath));
+ fs.delete(renamePath, false);
+ }
+
+ /**
+ * Negative test to verify rename access check.
+ * @throws Throwable
+ */
+ @Test(expected=WasbAuthorizationException.class)
+ public void testRenameAccessCheckNegative() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+ String testFile = "test.dat";
+ Path testPath = new Path(fs.getWorkingDirectory(), testFile);
+ Path renamePath = new Path("test2.dat");
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), false);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testPath);
+
+ Assert.check(fs.exists(testPath));
+ fs.rename(testPath, renamePath);
+ Assert.check(fs.exists(renamePath));
+ fs.delete(renamePath, false);
+ } catch (WasbAuthorizationException ex) {
+ throw ex;
+ } finally {
+ authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), false);
+ fs.updateWasbAuthorizer(authorizer);
+ Assert.check(fs.exists(testPath));
+ fs.delete(testPath, false);
+ }
+ }
+
+ /**
+ * Positive test for read access check.
+ * @throws Throwable
+ */
+ @Test
+ public void testReadAccessCheckPositive() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+ String testFile = "test.dat";
+ Path testPath = new Path(fs.getWorkingDirectory(), testFile);
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.READ.toString(), true);
+ authorizer.addAuthRule(fs.getWorkingDirectory().toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+ fs.create(testPath);
+ Assert.check(fs.exists(testPath));
+ FSDataInputStream inputStream = fs.open(testPath);
+ inputStream.close();
+ fs.delete(testPath, false);
+ }
+
+ /**
+ * Negative test to verify read access check.
+ * @throws Throwable
+ */
+ @Test(expected=WasbAuthorizationException.class)
+ public void testReadAccessCheckNegative() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+ String testFile = "test.dat";
+ Path testPath = new Path(fs.getWorkingDirectory(), testFile);
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(),
+ WasbAuthorizationOperations.READ.toString(), false);
+ fs.updateWasbAuthorizer(authorizer);
+
+ fs.create(new Path(testFile));
+ Assert.check(fs.exists(testPath));
+ FSDataInputStream inputStream = null;
+ try {
+ inputStream = fs.open(new Path(testFile));
+ } catch (WasbAuthorizationException ex) {
+ throw ex;
+ } finally {
+ fs.delete(new Path(testFile), false);
+ if (inputStream != null) {
+ inputStream.close();
+ }
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] hadoop git commit: HDFS-11416. Refactor out system default
erasure coding policy. Contributed by Andrew Wang.
Posted by kk...@apache.org.
HDFS-11416. Refactor out system default erasure coding policy. Contributed by Andrew Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3749152b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3749152b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3749152b
Branch: refs/heads/YARN-5972
Commit: 3749152b661d0359b3b941ab1d17177230f3b8dc
Parents: 8f4817f
Author: Rakesh Radhakrishnan <ra...@apache.org>
Authored: Fri Mar 3 09:28:03 2017 +0530
Committer: Rakesh Radhakrishnan <ra...@apache.org>
Committed: Fri Mar 3 09:28:03 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/util/StringUtils.java | 11 ++++++
.../server/namenode/FSDirErasureCodingOp.java | 38 +++++++++-----------
.../hdfs/server/namenode/FSEditLogLoader.java | 3 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 16 +++++----
.../hdfs/server/namenode/NamenodeFsck.java | 6 +---
.../server/namenode/TestStripedINodeFile.java | 13 +++++++
6 files changed, 52 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 11a93de..d2b572b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -178,6 +178,17 @@ public class StringUtils {
}
/**
+ * Convert a byte to a hex string.
+ * @see #byteToHexString(byte[])
+ * @see #byteToHexString(byte[], int, int)
+ * @param b byte
+ * @return byte's hex value as a String
+ */
+ public static String byteToHexString(byte b) {
+ return byteToHexString(new byte[] {b});
+ }
+
+ /**
* Given a hexstring this will return the byte array corresponding to the
* string
* @param hex the hex String array
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index f4c4ad7..5240c7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -102,6 +102,7 @@ final class FSDirErasureCodingOp {
FSDirectory fsd = fsn.getFSDirectory();
assert fsd.hasWriteLock();
Preconditions.checkNotNull(srcIIP, "INodes cannot be null");
+ Preconditions.checkNotNull(ecPolicy, "EC policy cannot be null");
String src = srcIIP.getPath();
final INode inode = srcIIP.getLastINode();
if (inode == null) {
@@ -112,29 +113,24 @@ final class FSDirErasureCodingOp {
"for a file " + src);
}
- // System default erasure coding policy will be used since no specified.
- if (ecPolicy == null) {
- ecPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy();
- } else {
- // If ecPolicy is specified check if it is one among active policies.
- boolean validPolicy = false;
- ErasureCodingPolicy[] activePolicies =
- FSDirErasureCodingOp.getErasureCodingPolicies(fsd.getFSNamesystem());
- for (ErasureCodingPolicy activePolicy : activePolicies) {
- if (activePolicy.equals(ecPolicy)) {
- validPolicy = true;
- break;
- }
+ // Check that the EC policy is one of the active policies.
+ boolean validPolicy = false;
+ ErasureCodingPolicy[] activePolicies =
+ FSDirErasureCodingOp.getErasureCodingPolicies(fsd.getFSNamesystem());
+ for (ErasureCodingPolicy activePolicy : activePolicies) {
+ if (activePolicy.equals(ecPolicy)) {
+ validPolicy = true;
+ break;
}
- if (!validPolicy) {
- List<String> ecPolicyNames = new ArrayList<String>();
- for (ErasureCodingPolicy activePolicy : activePolicies) {
- ecPolicyNames.add(activePolicy.getName());
- }
- throw new HadoopIllegalArgumentException("Policy [ " +
- ecPolicy.getName() + " ] does not match any of the " +
- "supported policies. Please select any one of " + ecPolicyNames);
+ }
+ if (!validPolicy) {
+ List<String> ecPolicyNames = new ArrayList<String>();
+ for (ErasureCodingPolicy activePolicy : activePolicies) {
+ ecPolicyNames.add(activePolicy.getName());
}
+ throw new HadoopIllegalArgumentException("Policy [ " +
+ ecPolicy.getName() + " ] does not match any of the " +
+ "supported policies. Please select any one of " + ecPolicyNames);
}
final XAttr ecXAttr;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 2990344..e3ad076 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -1108,8 +1108,7 @@ public class FSEditLogLoader {
// versions of Hadoop. Current versions always log
// OP_ADD operations as each block is allocated.
if (isStriped) {
- newBI = new BlockInfoStriped(newBlock,
- ErasureCodingPolicyManager.getSystemDefaultPolicy());
+ newBI = new BlockInfoStriped(newBlock, ecPolicy);
} else {
newBI = new BlockInfoContiguous(newBlock,
file.getFileReplication());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 0315150..19480e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hdfs.util.LongBitFormat;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.util.StringUtils;
/** I-node for closed file. */
@InterfaceAudience.Private
@@ -190,9 +191,10 @@ public class INodeFile extends INodeWithAdditionalFields
if (blockType == STRIPED) {
Preconditions.checkArgument(replication == null &&
erasureCodingPolicyID != null);
- Preconditions.checkArgument(
- ErasureCodingPolicyManager.getPolicyByPolicyID(
- erasureCodingPolicyID) != null);
+ Preconditions.checkArgument(ErasureCodingPolicyManager
+ .getPolicyByPolicyID(erasureCodingPolicyID) != null,
+ "Could not find EC policy with ID 0x" + StringUtils
+ .byteToHexString(erasureCodingPolicyID));
layoutRedundancy |= BLOCK_TYPE_MASK_STRIPED;
// Following bitwise OR with signed byte erasureCodingPolicyID is safe
// as the PolicyID can never be in negative.
@@ -201,7 +203,8 @@ public class INodeFile extends INodeWithAdditionalFields
Preconditions.checkArgument(replication != null &&
erasureCodingPolicyID == null);
Preconditions.checkArgument(replication >= 0 &&
- replication <= MAX_REDUNDANCY);
+ replication <= MAX_REDUNDANCY,
+ "Invalid replication value " + replication);
layoutRedundancy |= replication;
}
return layoutRedundancy;
@@ -513,9 +516,8 @@ public class INodeFile extends INodeWithAdditionalFields
ErasureCodingPolicy ecPolicy =
ErasureCodingPolicyManager.getPolicyByPolicyID(
getErasureCodingPolicyID());
- if (ecPolicy == null){
- ecPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy();
- }
+ Preconditions.checkNotNull(ecPolicy, "Could not find EC policy with ID 0x"
+ + StringUtils.byteToHexString(getErasureCodingPolicyID()));
return (short) (ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 7e91935..eddab3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -1311,11 +1311,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
@VisibleForTesting
static class ErasureCodingResult extends Result {
- final String defaultECPolicy;
ErasureCodingResult(Configuration conf) {
- defaultECPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy()
- .getName();
}
@Override
@@ -1392,8 +1389,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
((float) (numMisReplicatedBlocks * 100) / (float) totalBlocks))
.append(" %)");
}
- res.append("\n Default ecPolicy:\t\t").append(defaultECPolicy)
- .append("\n Average block group size:\t").append(
+ res.append("\n Average block group size:\t").append(
getReplicationFactor()).append("\n Missing block groups:\t\t").append(
missingIds.size()).append("\n Corrupt block groups:\t\t").append(
corruptBlocks).append("\n Missing internal blocks:\t").append(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
index a712498..ae9793a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout;
/**
@@ -83,6 +84,18 @@ public class TestStripedINodeFile {
HdfsConstants.COLD_STORAGE_POLICY_ID, BlockType.STRIPED);
}
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void testInvalidECPolicy() throws IllegalArgumentException {
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("Could not find EC policy with ID 0xbb");
+ new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
+ null, null, (byte) 0xBB, 1024L,
+ HdfsConstants.COLD_STORAGE_POLICY_ID, BlockType.STRIPED);
+ }
+
@Test
public void testBlockStripedFeature()
throws IOException, InterruptedException{
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] hadoop git commit: HDFS-8112. Relax permission checking for
EC related operations.
Posted by kk...@apache.org.
HDFS-8112. Relax permission checking for EC related operations.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3085a604
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3085a604
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3085a604
Branch: refs/heads/YARN-5972
Commit: 3085a604300ed76d06a0011bd5555e419897b6cd
Parents: 490abfb
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Mar 3 13:00:22 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Mar 3 13:00:22 2017 -0800
----------------------------------------------------------------------
.../server/namenode/FSDirErasureCodingOp.java | 74 ++++-----
.../server/namenode/FSDirStatAndListingOp.java | 4 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 12 +-
.../hdfs/server/namenode/FSDirectory.java | 3 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 20 ++-
.../hdfs/server/namenode/FSNamesystem.java | 11 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 153 +++++++++++++++++--
7 files changed, 203 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3085a604/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 5240c7e..30a7b8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.security.AccessControlException;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_ERASURECODING_POLICY;
@@ -66,15 +67,15 @@ final class FSDirErasureCodingOp {
* @return {@link HdfsFileStatus}
* @throws IOException
* @throws HadoopIllegalArgumentException if the policy is not enabled
+ * @throws AccessControlException if the user does not have write access
*/
static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn,
final String srcArg, final String ecPolicyName,
- final boolean logRetryCache) throws IOException {
+ final FSPermissionChecker pc, final boolean logRetryCache)
+ throws IOException, AccessControlException {
assert fsn.hasWriteLock();
String src = srcArg;
- FSPermissionChecker pc = null;
- pc = fsn.getPermissionChecker();
FSDirectory fsd = fsn.getFSDirectory();
final INodesInPath iip;
List<XAttr> xAttrs;
@@ -88,6 +89,10 @@ final class FSDirErasureCodingOp {
"policies.");
}
iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
+ // Write access is required to set erasure coding policy
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+ }
src = iip.getPath();
xAttrs = setErasureCodingPolicyXAttr(fsn, iip, ecPolicy);
} finally {
@@ -97,7 +102,7 @@ final class FSDirErasureCodingOp {
return fsd.getAuditFileInfo(iip);
}
- static List<XAttr> setErasureCodingPolicyXAttr(final FSNamesystem fsn,
+ private static List<XAttr> setErasureCodingPolicyXAttr(final FSNamesystem fsn,
final INodesInPath srcIIP, ErasureCodingPolicy ecPolicy) throws IOException {
FSDirectory fsd = fsn.getFSDirectory();
assert fsd.hasWriteLock();
@@ -165,19 +170,24 @@ final class FSDirErasureCodingOp {
* cache rebuilding
* @return {@link HdfsFileStatus}
* @throws IOException
+ * @throws AccessControlException if the user does not have write access
*/
static HdfsFileStatus unsetErasureCodingPolicy(final FSNamesystem fsn,
- final String srcArg, final boolean logRetryCache) throws IOException {
+ final String srcArg, final FSPermissionChecker pc,
+ final boolean logRetryCache) throws IOException {
assert fsn.hasWriteLock();
String src = srcArg;
- FSPermissionChecker pc = fsn.getPermissionChecker();
FSDirectory fsd = fsn.getFSDirectory();
final INodesInPath iip;
List<XAttr> xAttrs;
fsd.writeLock();
try {
iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
+ // Write access is required to unset erasure coding policy
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+ }
src = iip.getPath();
xAttrs = removeErasureCodingPolicyXAttr(fsn, iip);
} finally {
@@ -225,29 +235,23 @@ final class FSDirErasureCodingOp {
* @return {@link ErasureCodingPolicy}
* @throws IOException
* @throws FileNotFoundException if the path does not exist.
+ * @throws AccessControlException if no read access
*/
static ErasureCodingPolicy getErasureCodingPolicy(final FSNamesystem fsn,
- final String src) throws IOException {
+ final String src, FSPermissionChecker pc)
+ throws IOException, AccessControlException {
assert fsn.hasReadLock();
- final INodesInPath iip = getINodesInPath(fsn, src);
+ FSDirectory fsd = fsn.getFSDirectory();
+ final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.READ);
+ if (fsn.isPermissionEnabled()) {
+ fsn.getFSDirectory().checkPathAccess(pc, iip, FsAction.READ);
+ }
+
if (iip.getLastINode() == null) {
throw new FileNotFoundException("Path not found: " + iip.getPath());
}
- return getErasureCodingPolicyForPath(fsn, iip);
- }
-
- /**
- * Check if the file or directory has an erasure coding policy.
- *
- * @param fsn namespace
- * @param srcArg path
- * @return Whether the file or directory has an erasure coding policy.
- * @throws IOException
- */
- static boolean hasErasureCodingPolicy(final FSNamesystem fsn,
- final String srcArg) throws IOException {
- return hasErasureCodingPolicy(fsn, getINodesInPath(fsn, srcArg));
+ return getErasureCodingPolicyForPath(fsd, iip);
}
/**
@@ -260,22 +264,22 @@ final class FSDirErasureCodingOp {
*/
static boolean hasErasureCodingPolicy(final FSNamesystem fsn,
final INodesInPath iip) throws IOException {
- return getErasureCodingPolicy(fsn, iip) != null;
+ return unprotectedGetErasureCodingPolicy(fsn, iip) != null;
}
/**
- * Get the erasure coding policy.
+ * Get the erasure coding policy. This does not do any permission checking.
*
* @param fsn namespace
* @param iip inodes in the path containing the file
* @return {@link ErasureCodingPolicy}
* @throws IOException
*/
- static ErasureCodingPolicy getErasureCodingPolicy(final FSNamesystem fsn,
- final INodesInPath iip) throws IOException {
+ static ErasureCodingPolicy unprotectedGetErasureCodingPolicy(
+ final FSNamesystem fsn, final INodesInPath iip) throws IOException {
assert fsn.hasReadLock();
- return getErasureCodingPolicyForPath(fsn, iip);
+ return getErasureCodingPolicyForPath(fsn.getFSDirectory(), iip);
}
/**
@@ -290,21 +294,9 @@ final class FSDirErasureCodingOp {
return fsn.getErasureCodingPolicyManager().getPolicies();
}
- private static INodesInPath getINodesInPath(final FSNamesystem fsn,
- final String srcArg) throws IOException {
- final FSDirectory fsd = fsn.getFSDirectory();
- final FSPermissionChecker pc = fsn.getPermissionChecker();
- INodesInPath iip = fsd.resolvePath(pc, srcArg, DirOp.READ);
- if (fsn.isPermissionEnabled()) {
- fsn.getFSDirectory().checkPathAccess(pc, iip, FsAction.READ);
- }
- return iip;
- }
-
- private static ErasureCodingPolicy getErasureCodingPolicyForPath(FSNamesystem fsn,
- INodesInPath iip) throws IOException {
+ private static ErasureCodingPolicy getErasureCodingPolicyForPath(
+ FSDirectory fsd, INodesInPath iip) throws IOException {
Preconditions.checkNotNull(iip, "INodes cannot be null");
- FSDirectory fsd = fsn.getFSDirectory();
fsd.readLock();
try {
List<INode> inodes = iip.getReadOnlyINodes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3085a604/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index ba7deec..d0431ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -172,7 +172,7 @@ class FSDirStatAndListingOp {
final FileEncryptionInfo feInfo =
FSDirEncryptionZoneOp.getFileEncryptionInfo(fsd, iip);
final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
- getErasureCodingPolicy(fsd.getFSNamesystem(), iip);
+ unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), iip);
final LocatedBlocks blocks = bm.createLocatedBlocks(
inode.getBlocks(iip.getPathSnapshotId()), fileSize, isUc, offset,
@@ -413,7 +413,7 @@ class FSDirStatAndListingOp {
FileEncryptionInfo feInfo = null;
final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp
- .getErasureCodingPolicy(fsd.getFSNamesystem(), iip);
+ .unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), iip);
if (node.isFile()) {
final INodeFile fileNode = node.asFile();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3085a604/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 0a4d3b0..bb92004 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -190,7 +190,8 @@ class FSDirWriteFileOp {
blockType = pendingFile.getBlockType();
ErasureCodingPolicy ecPolicy = null;
if (blockType == BlockType.STRIPED) {
- ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(fsn, src);
+ ecPolicy =
+ FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(fsn, iip);
numTargets = (short) (ecPolicy.getSchema().getNumDataUnits()
+ ecPolicy.getSchema().getNumParityUnits());
} else {
@@ -418,7 +419,7 @@ class FSDirWriteFileOp {
// check if the file has an EC policy
boolean isStriped = false;
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
- getErasureCodingPolicy(fsd.getFSNamesystem(), existing);
+ unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), existing);
if (ecPolicy != null) {
isStriped = true;
}
@@ -475,8 +476,9 @@ class FSDirWriteFileOp {
// associate new last block for the file
final BlockInfo blockInfo;
if (blockType == BlockType.STRIPED) {
- ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
- fsd.getFSNamesystem(), inodesInPath);
+ ErasureCodingPolicy ecPolicy =
+ FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
+ fsd.getFSNamesystem(), inodesInPath);
short numDataUnits = (short) ecPolicy.getNumDataUnits();
short numParityUnits = (short) ecPolicy.getNumParityUnits();
short numLocations = (short) (numDataUnits + numParityUnits);
@@ -529,7 +531,7 @@ class FSDirWriteFileOp {
try {
boolean isStriped = false;
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
- getErasureCodingPolicy(fsd.getFSNamesystem(), existing);
+ unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), existing);
if (ecPolicy != null) {
isStriped = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3085a604/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index b21442d..9ac6149 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -957,7 +957,8 @@ public class FSDirectory implements Closeable {
final short replicationFactor;
if (fileINode.isStriped()) {
final ErasureCodingPolicy ecPolicy =
- FSDirErasureCodingOp.getErasureCodingPolicy(namesystem, iip);
+ FSDirErasureCodingOp
+ .unprotectedGetErasureCodingPolicy(namesystem, iip);
final short numDataUnits = (short) ecPolicy.getNumDataUnits();
final short numParityUnits = (short) ecPolicy.getNumParityUnits();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3085a604/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index e3ad076..ff36f18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -415,8 +415,9 @@ public class FSEditLogLoader {
// Update the salient file attributes.
newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
- ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
- fsDir.getFSNamesystem(), iip);
+ ErasureCodingPolicy ecPolicy =
+ FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
+ fsDir.getFSNamesystem(), iip);
updateBlocks(fsDir, addCloseOp, iip, newFile, ecPolicy);
break;
}
@@ -437,8 +438,9 @@ public class FSEditLogLoader {
// Update the salient file attributes.
file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
- ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
- fsDir.getFSNamesystem(), iip);
+ ErasureCodingPolicy ecPolicy =
+ FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
+ fsDir.getFSNamesystem(), iip);
updateBlocks(fsDir, addCloseOp, iip, file, ecPolicy);
// Now close the file
@@ -496,8 +498,9 @@ public class FSEditLogLoader {
INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ);
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// Update in-memory data structures
- ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
- fsDir.getFSNamesystem(), iip);
+ ErasureCodingPolicy ecPolicy =
+ FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
+ fsDir.getFSNamesystem(), iip);
updateBlocks(fsDir, updateOp, iip, oldFile, ecPolicy);
if (toAddRetryCache) {
@@ -515,8 +518,9 @@ public class FSEditLogLoader {
INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ);
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// add the new block to the INodeFile
- ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
- fsDir.getFSNamesystem(), iip);
+ ErasureCodingPolicy ecPolicy =
+ FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
+ fsDir.getFSNamesystem(), iip);
addNewBlock(addBlockOp, oldFile, ecPolicy);
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3085a604/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 254ccfa..81c5759 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -6782,16 +6782,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final boolean logRetryCache) throws IOException,
UnresolvedLinkException, SafeModeException, AccessControlException {
final String operationName = "setErasureCodingPolicy";
- checkSuperuserPrivilege();
checkOperation(OperationCategory.WRITE);
HdfsFileStatus resultingStat = null;
+ final FSPermissionChecker pc = getPermissionChecker();
boolean success = false;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg);
resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this,
- srcArg, ecPolicyName, logRetryCache);
+ srcArg, ecPolicyName, pc, logRetryCache);
success = true;
} catch (AccessControlException ace) {
logAuditEvent(success, operationName, srcArg, null,
@@ -6818,16 +6818,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final boolean logRetryCache) throws IOException,
UnresolvedLinkException, SafeModeException, AccessControlException {
final String operationName = "unsetErasureCodingPolicy";
- checkSuperuserPrivilege();
checkOperation(OperationCategory.WRITE);
HdfsFileStatus resultingStat = null;
+ final FSPermissionChecker pc = getPermissionChecker();
boolean success = false;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot unset erasure coding policy on " + srcArg);
resultingStat = FSDirErasureCodingOp.unsetErasureCodingPolicy(this,
- srcArg, logRetryCache);
+ srcArg, pc, logRetryCache);
success = true;
} catch (AccessControlException ace) {
logAuditEvent(success, operationName, srcArg, null,
@@ -6849,10 +6849,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
ErasureCodingPolicy getErasureCodingPolicy(String src)
throws AccessControlException, UnresolvedLinkException, IOException {
checkOperation(OperationCategory.READ);
+ FSPermissionChecker pc = getPermissionChecker();
readLock();
try {
checkOperation(OperationCategory.READ);
- return FSDirErasureCodingOp.getErasureCodingPolicy(this, src);
+ return FSDirErasureCodingOp.getErasureCodingPolicy(this, src, pc);
} finally {
readUnlock("getErasureCodingPolicy");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3085a604/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 8608626..e6a8227 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
@@ -30,12 +31,17 @@ import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.Timeout;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collection;
@@ -51,6 +57,9 @@ public class TestErasureCodingPolicies {
ErasureCodingPolicyManager.getSystemDefaultPolicy();
private FSNamesystem namesystem;
+ @Rule
+ public Timeout timeout = new Timeout(60 * 1000);
+
@Before
public void setupCluster() throws IOException {
conf = new HdfsConfiguration();
@@ -74,7 +83,7 @@ public class TestErasureCodingPolicies {
* for pre-existing files (with replicated blocks) in an EC dir, getListing
* should report them as non-ec.
*/
- @Test(timeout=60000)
+ @Test
public void testReplicatedFileUnderECDir() throws IOException {
final Path dir = new Path("/ec");
final Path replicatedFile = new Path(dir, "replicatedFile");
@@ -128,7 +137,7 @@ public class TestErasureCodingPolicies {
assertNotNull(files[1].getErasureCodingPolicy());
}
- @Test(timeout = 60000)
+ @Test
public void testBasicSetECPolicy()
throws IOException, InterruptedException {
final Path testDir = new Path("/ec");
@@ -182,7 +191,7 @@ public class TestErasureCodingPolicies {
}
}
- @Test(timeout = 60000)
+ @Test
public void testMoveValidity() throws IOException, InterruptedException {
final Path srcECDir = new Path("/srcEC");
final Path dstECDir = new Path("/dstEC");
@@ -219,7 +228,7 @@ public class TestErasureCodingPolicies {
fs.rename(nonECFile, dstECDir);
}
- @Test(timeout = 60000)
+ @Test
public void testReplication() throws IOException {
final Path testDir = new Path("/ec");
fs.mkdir(testDir, FsPermission.getDirDefault());
@@ -237,7 +246,7 @@ public class TestErasureCodingPolicies {
assertEquals(policy, fs.getErasureCodingPolicy(fooFile));
}
- @Test(timeout = 60000)
+ @Test
public void testGetErasureCodingPolicyWithSystemDefaultECPolicy() throws Exception {
String src = "/ec";
final Path ecDir = new Path(src);
@@ -254,7 +263,7 @@ public class TestErasureCodingPolicies {
verifyErasureCodingInfo(src + "/child1", sysDefaultECPolicy);
}
- @Test(timeout = 60000)
+ @Test
public void testGetErasureCodingPolicy() throws Exception {
ErasureCodingPolicy[] sysECPolicies =
ErasureCodingPolicyManager.getSystemPolicies();
@@ -284,13 +293,13 @@ public class TestErasureCodingPolicies {
usingECPolicy, ecPolicy);
}
- @Test(timeout = 60000)
- public void testCreationErasureCodingZoneWithInvalidPolicy()
+ @Test
+ public void testSetInvalidPolicy()
throws IOException {
ECSchema rsSchema = new ECSchema("rs", 4, 2);
String policyName = "RS-4-2-128k";
int cellSize = 128 * 1024;
- ErasureCodingPolicy ecPolicy=
+ ErasureCodingPolicy ecPolicy =
new ErasureCodingPolicy(policyName, rsSchema, cellSize, (byte) -1);
String src = "/ecDir4-2";
final Path ecDir = new Path(src);
@@ -305,7 +314,7 @@ public class TestErasureCodingPolicies {
}
}
- @Test(timeout = 60000)
+ @Test
public void testGetAllErasureCodingPolicies() throws Exception {
ErasureCodingPolicy[] sysECPolicies = ErasureCodingPolicyManager
.getSystemPolicies();
@@ -315,7 +324,7 @@ public class TestErasureCodingPolicies {
allECPolicies.containsAll(Arrays.asList(sysECPolicies)));
}
- @Test(timeout = 60000)
+ @Test
public void testGetErasureCodingPolicyOnANonExistentFile() throws Exception {
Path path = new Path("/ecDir");
try {
@@ -335,7 +344,7 @@ public class TestErasureCodingPolicies {
}
}
- @Test(timeout = 60000)
+ @Test
public void testMultiplePoliciesCoExist() throws Exception {
ErasureCodingPolicy[] sysPolicies =
ErasureCodingPolicyManager.getSystemPolicies();
@@ -355,4 +364,124 @@ public class TestErasureCodingPolicies {
}
}
}
+
+ @Test
+ public void testPermissions() throws Exception {
+ UserGroupInformation user =
+ UserGroupInformation.createUserForTesting("ecuser",
+ new String[]{"ecgroup"});
+ FileSystem userfs = user.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws Exception {
+ return FileSystem.get(conf);
+ }
+ });
+ HdfsAdmin useradmin = user.doAs(new PrivilegedExceptionAction<HdfsAdmin>() {
+ @Override
+ public HdfsAdmin run() throws Exception {
+ return new HdfsAdmin(userfs.getUri(), conf);
+ }
+ });
+
+ // Create dir and set an EC policy, create an EC file
+ Path ecdir = new Path("/ecdir");
+ Path ecfile = new Path(ecdir, "ecfile");
+ fs.setPermission(new Path("/"), new FsPermission((short)0777));
+ userfs.mkdirs(ecdir);
+ final String ecPolicyName =
+ ErasureCodingPolicyManager.getSystemPolicies()[0].getName();
+ useradmin.setErasureCodingPolicy(ecdir, ecPolicyName);
+ assertEquals("Policy not present on dir",
+ ecPolicyName,
+ useradmin.getErasureCodingPolicy(ecdir).getName());
+ userfs.create(ecfile).close();
+ assertEquals("Policy not present on file",
+ ecPolicyName,
+ useradmin.getErasureCodingPolicy(ecfile).getName());
+
+ // Unset and re-set
+ useradmin.unsetErasureCodingPolicy(ecdir);
+ useradmin.setErasureCodingPolicy(ecdir, ecPolicyName);
+
+ // Change write permissions and make sure set and unset are denied
+ userfs.setPermission(ecdir, new FsPermission((short)0555));
+ try {
+ useradmin.setErasureCodingPolicy(ecdir, ecPolicyName);
+ fail("Should not be able to setECPolicy without write permissions");
+ } catch (AccessControlException e) {
+ // pass
+ }
+ try {
+ useradmin.unsetErasureCodingPolicy(ecdir);
+ fail("Should not be able to unsetECPolicy without write permissions");
+ } catch (AccessControlException e) {
+ // pass
+ }
+
+ // Change the permissions again, check that set and unset work
+ userfs.setPermission(ecdir, new FsPermission((short)0640));
+ useradmin.unsetErasureCodingPolicy(ecdir);
+ useradmin.setErasureCodingPolicy(ecdir, ecPolicyName);
+
+ // Set, unset, and get with another user should be unauthorized
+ UserGroupInformation nobody =
+ UserGroupInformation.createUserForTesting("nobody",
+ new String[]{"nogroup"});
+ HdfsAdmin noadmin = nobody.doAs(new PrivilegedExceptionAction<HdfsAdmin>() {
+ @Override
+ public HdfsAdmin run() throws Exception {
+ return new HdfsAdmin(userfs.getUri(), conf);
+ }
+ });
+ try {
+ noadmin.setErasureCodingPolicy(ecdir, ecPolicyName);
+ fail("Should not be able to setECPolicy without write permissions");
+ } catch (AccessControlException e) {
+ // pass
+ }
+ try {
+ noadmin.unsetErasureCodingPolicy(ecdir);
+ fail("Should not be able to unsetECPolicy without write permissions");
+ } catch (AccessControlException e) {
+ // pass
+ }
+ try {
+ noadmin.getErasureCodingPolicy(ecdir);
+ fail("Should not be able to getECPolicy without write permissions");
+ } catch (AccessControlException e) {
+ // pass
+ }
+
+ // superuser can do whatever it wants
+ userfs.setPermission(ecdir, new FsPermission((short)0000));
+ HdfsAdmin superadmin = new HdfsAdmin(fs.getUri(), conf);
+ superadmin.unsetErasureCodingPolicy(ecdir);
+ superadmin.setErasureCodingPolicy(ecdir, ecPolicyName);
+ superadmin.getErasureCodingPolicy(ecdir);
+
+ // Normal user no longer has access
+ try {
+ useradmin.getErasureCodingPolicy(ecdir);
+ fail("Normal user should not have access");
+ } catch (AccessControlException e) {
+ // pass
+ }
+ try {
+ useradmin.setErasureCodingPolicy(ecfile, ecPolicyName);
+ fail("Normal user should not have access");
+ } catch (AccessControlException e) {
+ // pass
+ }
+ try {
+ useradmin.unsetErasureCodingPolicy(ecfile);
+ fail("Normal user should not have access");
+ } catch (AccessControlException e) {
+ // pass
+ }
+
+ // Everyone has access to getting the list of EC policies
+ useradmin.getErasureCodingPolicies();
+ noadmin.getErasureCodingPolicies();
+ superadmin.getErasureCodingPolicies();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] hadoop git commit: HDFS-11498. Make RestCsrfPreventionHandler
and WebHdfsHandler compatible with Netty 4.0.
Posted by kk...@apache.org.
HDFS-11498. Make RestCsrfPreventionHandler and WebHdfsHandler compatible with Netty 4.0.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e74196e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e74196e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e74196e
Branch: refs/heads/YARN-5972
Commit: 5e74196ede9bfc20eb6d6fe3aa6a0e5c47a40fdd
Parents: d9dc444
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Mar 6 15:04:13 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Mar 6 15:04:13 2017 -0800
----------------------------------------------------------------------
.../web/RestCsrfPreventionFilterHandler.java | 6 ++--
.../datanode/web/webhdfs/WebHdfsHandler.java | 33 ++++++++++----------
hadoop-project/pom.xml | 2 +-
3 files changed, 20 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e74196e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
index f2f0533..4958bb5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.datanode.web;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
@@ -119,7 +119,7 @@ final class RestCsrfPreventionFilterHandler
@Override
public String getMethod() {
- return req.method().name();
+ return req.getMethod().name();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e74196e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index d2b2ec2..c5fc7ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -29,17 +29,6 @@ import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.QueryStringDecoder;
import io.netty.handler.stream.ChunkedStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.security.PrivilegedExceptionAction;
-import java.util.EnumSet;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -63,17 +52,27 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.LimitInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.PrivilegedExceptionAction;
+import java.util.EnumSet;
+
+import static io.netty.handler.codec.http.HttpHeaders.Names.ACCEPT;
+import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_HEADERS;
import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_METHODS;
import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN;
+import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_MAX_AGE;
import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
import static io.netty.handler.codec.http.HttpHeaders.Names.LOCATION;
import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
-import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT;
-import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS;
-import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_MAX_AGE;
-import static io.netty.handler.codec.http.HttpHeaderValues.KEEP_ALIVE;
+import static io.netty.handler.codec.http.HttpHeaders.Values.KEEP_ALIVE;
import static io.netty.handler.codec.http.HttpMethod.GET;
import static io.netty.handler.codec.http.HttpMethod.OPTIONS;
import static io.netty.handler.codec.http.HttpMethod.POST;
@@ -145,7 +144,7 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
LOG.warn("Error retrieving hostname: ", e);
host = "unknown";
}
- REQLOG.info(host + " " + req.method() + " " + req.uri() + " " +
+ REQLOG.info(host + " " + req.getMethod() + " " + req.getUri() + " " +
getResponseCode());
}
return null;
@@ -155,7 +154,7 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
int getResponseCode() {
return (resp == null) ? INTERNAL_SERVER_ERROR.code() :
- resp.status().code();
+ resp.getStatus().code();
}
public void handle(ChannelHandlerContext ctx, HttpRequest req)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e74196e/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 4c8b900..572f0e6 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -719,7 +719,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
- <version>4.1.0.Beta5</version>
+ <version>4.0.23.Final</version>
</dependency>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] hadoop git commit: HADOOP-14048. REDO operation of
WASB#AtomicRename should create placeholder blob for destination folder.
Contributed by NITIN VERMA
Posted by kk...@apache.org.
HADOOP-14048. REDO operation of WASB#AtomicRename should create placeholder blob for destination folder. Contributed by NITIN VERMA
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c571cda5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c571cda5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c571cda5
Branch: refs/heads/YARN-5972
Commit: c571cda5c7d929477961dfff4176d7de4944d874
Parents: b5adc5c
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Mar 6 16:53:30 2017 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Mar 6 17:00:13 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/fs/azure/NativeAzureFileSystem.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c571cda5/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 0dfefaf..b1956a7 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -566,6 +566,16 @@ public class NativeAzureFileSystem extends FileSystem {
// Remove the source folder. Don't check explicitly if it exists,
// to avoid triggering redo recursively.
try {
+ // Rename the source folder 0-byte root file
+ // as destination folder 0-byte root file.
+ FileMetadata srcMetaData = this.getSourceMetadata();
+ if (srcMetaData.getBlobMaterialization() == BlobMaterialization.Explicit) {
+ // We already have a lease. So let's just rename the source blob
+ // as destination blob under same lease.
+ fs.getStoreInterface().rename(this.getSrcKey(), this.getDstKey(), false, lease);
+ }
+
+ // Now we can safely delete the source folder.
fs.getStoreInterface().delete(srcKey, lease);
} catch (Exception e) {
LOG.info("Unable to delete source folder during folder rename redo. "
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] hadoop git commit: HDFS-11412. Maintenance minimum
replication config value allowable range should be [0,
DefaultReplication]. (Manoj Govindassamy via mingma)
Posted by kk...@apache.org.
HDFS-11412. Maintenance minimum replication config value allowable range should be [0, DefaultReplication]. (Manoj Govindassamy via mingma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25c84d27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25c84d27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25c84d27
Branch: refs/heads/YARN-5972
Commit: 25c84d279bcefb72a3dd8058f25bba1713504849
Parents: 6f6dfe0
Author: Ming Ma <mi...@apache.org>
Authored: Wed Mar 1 20:23:52 2017 -0800
Committer: Ming Ma <mi...@apache.org>
Committed: Wed Mar 1 20:23:52 2017 -0800
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 9 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 ++
.../hadoop/hdfs/TestMaintenanceState.java | 111 +++++++++++++++++++
3 files changed, 124 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25c84d27/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5125b33..5ca0fa7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -484,12 +484,12 @@ public class BlockManager implements BlockStatsMXBean {
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY
+ " = " + minMaintenanceR + " < 0");
}
- if (minMaintenanceR > minR) {
+ if (minMaintenanceR > defaultReplication) {
throw new IOException("Unexpected configuration parameters: "
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY
+ " = " + minMaintenanceR + " > "
- + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY
- + " = " + minR);
+ + DFSConfigKeys.DFS_REPLICATION_KEY
+ + " = " + defaultReplication);
}
this.minReplicationToBeInMaintenance = (short)minMaintenanceR;
@@ -825,7 +825,8 @@ public class BlockManager implements BlockStatsMXBean {
if (block.isStriped()) {
return ((BlockInfoStriped) block).getRealDataBlockNum();
} else {
- return minReplicationToBeInMaintenance;
+ return (short) Math.min(minReplicationToBeInMaintenance,
+ block.getReplication());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25c84d27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 51dca41..f9908fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -807,6 +807,14 @@ public class MiniDFSCluster implements AutoCloseable {
int replication = conf.getInt(DFS_REPLICATION_KEY, 3);
conf.setInt(DFS_REPLICATION_KEY, Math.min(replication, numDataNodes));
+ int maintenanceMinReplication = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT);
+ if (maintenanceMinReplication ==
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT) {
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY,
+ Math.min(maintenanceMinReplication, numDataNodes));
+ }
int safemodeExtension = conf.getInt(
DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 0);
conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, safemodeExtension);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25c84d27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
index f3e2a0b..2432153 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
@@ -30,6 +31,7 @@ import java.util.List;
import java.util.Map;
import com.google.common.collect.Lists;
+import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileSystem;
@@ -67,6 +69,38 @@ public class TestMaintenanceState extends AdminStatesBaseTest {
}
/**
+ * Test valid value range for the config namenode.maintenance.replication.min.
+ */
+ @Test (timeout = 60000)
+ public void testMaintenanceMinReplConfigRange() {
+ LOG.info("Setting testMaintenanceMinReplConfigRange");
+
+ // Case 1: Maintenance min replication less allowed minimum 0
+ setMinMaintenanceR(-1);
+ try {
+ startCluster(1, 1);
+ fail("Cluster start should fail when 'dfs.namenode.maintenance" +
+ ".replication.min=-1'");
+ } catch (IOException e) {
+ LOG.info("Expected exception: " + e);
+ }
+
+ // Case 2: Maintenance min replication greater
+ // allowed max of DFSConfigKeys.DFS_REPLICATION_KEY
+ int defaultRepl = getConf().getInt(
+ DFSConfigKeys.DFS_REPLICATION_KEY,
+ DFSConfigKeys.DFS_REPLICATION_DEFAULT);
+ setMinMaintenanceR(defaultRepl + 1);
+ try {
+ startCluster(1, 1);
+ fail("Cluster start should fail when 'dfs.namenode.maintenance" +
+ ".replication.min > " + defaultRepl + "'");
+ } catch (IOException e) {
+ LOG.info("Expected exception: " + e);
+ }
+ }
+
+ /**
* Verify a node can transition from AdminStates.ENTERING_MAINTENANCE to
* AdminStates.NORMAL.
*/
@@ -407,6 +441,83 @@ public class TestMaintenanceState extends AdminStatesBaseTest {
}
/**
+ * Test file block replication lesser than maintenance minimum.
+ */
+ @Test(timeout = 360000)
+ public void testFileBlockReplicationAffectingMaintenance()
+ throws Exception {
+ int defaultReplication = getConf().getInt(DFSConfigKeys
+ .DFS_REPLICATION_KEY, DFSConfigKeys.DFS_REPLICATION_DEFAULT);
+ int defaultMaintenanceMinRepl = getConf().getInt(DFSConfigKeys
+ .DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT);
+
+ // Case 1:
+ // * Maintenance min larger than default min replication
+ // * File block replication larger than maintenance min
+ // * Initial data nodes not sufficient to remove all maintenance nodes
+ // as file block replication is greater than maintenance min.
+ // * Data nodes added later for the state transition to progress
+ int maintenanceMinRepl = defaultMaintenanceMinRepl + 1;
+ int fileBlockReplication = maintenanceMinRepl + 1;
+ int numAddedDataNodes = 1;
+ int numInitialDataNodes = (maintenanceMinRepl * 2 - numAddedDataNodes);
+ Assert.assertTrue(maintenanceMinRepl <= defaultReplication);
+ testFileBlockReplicationImpl(maintenanceMinRepl,
+ numInitialDataNodes, numAddedDataNodes, fileBlockReplication);
+
+ // Case 2:
+ // * Maintenance min larger than default min replication
+ // * File block replication lesser than maintenance min
+ // * Initial data nodes after removal of maintenance nodes is still
+ // sufficient for the file block replication.
+ // * No new data nodes to be added, still the state transition happens
+ maintenanceMinRepl = defaultMaintenanceMinRepl + 1;
+ fileBlockReplication = maintenanceMinRepl - 1;
+ numAddedDataNodes = 0;
+ numInitialDataNodes = (maintenanceMinRepl * 2 - numAddedDataNodes);
+ testFileBlockReplicationImpl(maintenanceMinRepl,
+ numInitialDataNodes, numAddedDataNodes, fileBlockReplication);
+ }
+
+ private void testFileBlockReplicationImpl(
+ int maintenanceMinRepl, int numDataNodes, int numNewDataNodes,
+ int fileBlockRepl)
+ throws Exception {
+ setup();
+ LOG.info("Starting testLargerMinMaintenanceReplication - maintMinRepl: "
+ + maintenanceMinRepl + ", numDNs: " + numDataNodes + ", numNewDNs: "
+ + numNewDataNodes + ", fileRepl: " + fileBlockRepl);
+ LOG.info("Setting maintenance minimum replication: " + maintenanceMinRepl);
+ setMinMaintenanceR(maintenanceMinRepl);
+ startCluster(1, numDataNodes);
+
+ final Path file = new Path("/testLargerMinMaintenanceReplication.dat");
+
+ FileSystem fileSys = getCluster().getFileSystem(0);
+ writeFile(fileSys, file, fileBlockRepl, 1);
+ final DatanodeInfo[] nodes = getFirstBlockReplicasDatanodeInfos(fileSys,
+ file);
+
+ ArrayList<String> nodeUuids = new ArrayList<>();
+ for (int i = 0; i < maintenanceMinRepl && i < nodes.length; i++) {
+ nodeUuids.add(nodes[i].getDatanodeUuid());
+ }
+
+ List<DatanodeInfo> maintenanceDNs = takeNodeOutofService(0, nodeUuids,
+ Long.MAX_VALUE, null, null, AdminStates.ENTERING_MAINTENANCE);
+
+ for (int i = 0; i < numNewDataNodes; i++) {
+ getCluster().startDataNodes(getConf(), 1, true, null, null);
+ }
+ getCluster().waitActive();
+ refreshNodes(0);
+ waitNodeState(maintenanceDNs, AdminStates.IN_MAINTENANCE);
+ cleanupFile(fileSys, file);
+ teardown();
+ }
+
+ /**
* Transition from IN_MAINTENANCE to DECOMMISSIONED.
*/
@Test(timeout = 360000)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] hadoop git commit: Revert "HADOOP-13930. Azure: Add
Authorization support to WASB. Contributed by Sivaguru Sankaridurg and
Dushyanth"
Posted by kk...@apache.org.
Revert "HADOOP-13930. Azure: Add Authorization support to WASB. Contributed by Sivaguru Sankaridurg and Dushyanth"
This reverts commit 6b7cd62b8cf12616b13142f2eb2cfc2f25796f0f.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52d7d5aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52d7d5aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52d7d5aa
Branch: refs/heads/YARN-5972
Commit: 52d7d5aa1a303cf70519a61487641211f4267c6f
Parents: c571cda
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Mar 6 17:10:11 2017 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Mar 6 17:10:11 2017 -0800
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 10 -
.../conf/TestCommonConfigurationFields.java | 2 -
.../fs/azure/AzureNativeFileSystemStore.java | 4 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 155 +----------
.../fs/azure/RemoteSASKeyGeneratorImpl.java | 183 ++++--------
.../fs/azure/RemoteWasbAuthorizerImpl.java | 247 -----------------
.../fs/azure/SecureStorageInterfaceImpl.java | 6 +-
.../fs/azure/WasbAuthorizationException.java | 40 ---
.../fs/azure/WasbAuthorizationOperations.java | 44 ---
.../fs/azure/WasbAuthorizerInterface.java | 47 ----
.../hadoop/fs/azure/security/Constants.java | 54 ----
.../security/WasbDelegationTokenIdentifier.java | 48 ----
.../fs/azure/security/WasbTokenRenewer.java | 124 ---------
.../hadoop/fs/azure/security/package.html | 28 --
...apache.hadoop.security.token.TokenIdentifier | 16 --
...rg.apache.hadoop.security.token.TokenRenewer | 16 --
.../hadoop-azure/src/site/markdown/index.md | 34 ---
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 102 -------
.../TestNativeAzureFileSystemAuthorization.java | 277 -------------------
19 files changed, 64 insertions(+), 1373 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 52b58ed..35be56b 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1292,16 +1292,6 @@
to specify the time (such as 2s, 2m, 1h, etc.).
</description>
</property>
-<property>
- <name>fs.azure.authorization</name>
- <value>false</value>
- <description>
- Config flag to enable authorization support in WASB. Setting it to "true" enables
- authorization support to WASB. Currently WASB authorization requires a remote service
- to provide authorization that needs to be specified via fs.azure.authorization.remote.service.url
- configuration
- </description>
-</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 7410d29..966a8ac 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -181,8 +181,6 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPropsToSkipCompare.add("io.compression.codec.bzip2.library");
// - org.apache.hadoop.io.SequenceFile
xmlPropsToSkipCompare.add("io.seqfile.local.dir");
- // - org.apache.hadoop.fs.azure.NativeAzureFileSystem
- xmlPropsToSkipCompare.add("fs.azure.authorization");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 9d7ac80..07c389c 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -303,7 +303,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
private boolean useSecureMode = false;
private boolean useLocalSasKeyMode = false;
-
+ private String delegationToken;
/**
* A test hook interface that can modify the operation context we use for
* Azure Storage operations, e.g. to inject errors.
@@ -478,7 +478,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
this.storageInteractionLayer = new StorageInterfaceImpl();
} else {
this.storageInteractionLayer = new SecureStorageInterfaceImpl(
- useLocalSasKeyMode, conf);
+ useLocalSasKeyMode, conf, delegationToken);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index b1956a7..4184a53 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -25,12 +25,9 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
-import java.net.URL;
import java.nio.charset.Charset;
-import java.security.PrivilegedExceptionAction;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -64,15 +61,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
-import org.apache.hadoop.fs.azure.security.Constants;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
-import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
@@ -1114,39 +1106,7 @@ public class NativeAzureFileSystem extends FileSystem {
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
private boolean appendSupportEnabled = false;
- private DelegationTokenAuthenticatedURL authURL;
- private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
- private String credServiceUrl;
-
- /**
- * Configuration key to enable authorization support in WASB.
- */
- public static final String KEY_AZURE_AUTHORIZATION =
- "fs.azure.authorization";
-
- /**
- * Default value for the authorization support in WASB.
- */
- private static final boolean DEFAULT_AZURE_AUTHORIZATION = false;
-
- /**
- * Flag controlling authorization support in WASB.
- */
- private boolean azureAuthorization = false;
-
- /**
- * Flag controlling Kerberos support in WASB.
- */
- private boolean kerberosSupportEnabled = false;
-
- /**
- * Authorizer to use when authorization support is enabled in
- * WASB.
- */
- private WasbAuthorizerInterface authorizer = null;
-
- private String delegationToken = null;
-
+
public NativeAzureFileSystem() {
// set store in initialize()
}
@@ -1277,31 +1237,6 @@ public class NativeAzureFileSystem extends FileSystem {
// Initialize thread counts from user configuration
deleteThreadCount = conf.getInt(AZURE_DELETE_THREADS, DEFAULT_AZURE_DELETE_THREADS);
renameThreadCount = conf.getInt(AZURE_RENAME_THREADS, DEFAULT_AZURE_RENAME_THREADS);
-
- this.azureAuthorization = conf.getBoolean(KEY_AZURE_AUTHORIZATION,
- DEFAULT_AZURE_AUTHORIZATION);
- this.kerberosSupportEnabled = conf.getBoolean(
- Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
-
- if (this.azureAuthorization) {
-
- this.authorizer =
- new RemoteWasbAuthorizerImpl();
- authorizer.init(conf);
- }
- if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
- DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator();
- authURL = new DelegationTokenAuthenticatedURL(authenticator);
- credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, String
- .format("http://%s:%s",
- InetAddress.getLocalHost().getCanonicalHostName(),
- Constants.DEFAULT_CRED_SERVICE_PORT));
- }
- }
-
- @VisibleForTesting
- public void updateWasbAuthorizer(WasbAuthorizerInterface authorizer) {
- this.authorizer = authorizer;
}
private NativeFileSystemStore createDefaultStore(Configuration conf) {
@@ -1415,15 +1350,6 @@ public class NativeAzureFileSystem extends FileSystem {
return store;
}
- private void performAuthCheck(String path, String accessType,
- String operation) throws WasbAuthorizationException, IOException {
-
- if (azureAuthorization && !this.authorizer.authorize(path, accessType)) {
- throw new WasbAuthorizationException(operation
- + " operation for Path : " + path + " not allowed");
- }
- }
-
/**
* Gets the metrics source for this file system.
* This is mainly here for unit testing purposes.
@@ -1446,10 +1372,6 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Opening file: {} for append", f);
Path absolutePath = makeAbsolute(f);
-
- performAuthCheck(absolutePath.toString(),
- WasbAuthorizationOperations.WRITE.toString(), "append");
-
String key = pathToKey(absolutePath);
FileMetadata meta = null;
try {
@@ -1650,10 +1572,6 @@ public class NativeAzureFileSystem extends FileSystem {
}
Path absolutePath = makeAbsolute(f);
-
- performAuthCheck(absolutePath.toString(),
- WasbAuthorizationOperations.WRITE.toString(), "create");
-
String key = pathToKey(absolutePath);
FileMetadata existingMetadata = store.retrieveMetadata(key);
@@ -1776,10 +1694,6 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Deleting file: {}", f.toString());
Path absolutePath = makeAbsolute(f);
-
- performAuthCheck(absolutePath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), "delete");
-
String key = pathToKey(absolutePath);
// Capture the metadata for the path.
@@ -2050,10 +1964,6 @@ public class NativeAzureFileSystem extends FileSystem {
// Capture the absolute path and the path to key.
Path absolutePath = makeAbsolute(f);
-
- performAuthCheck(absolutePath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), "getFileStatus");
-
String key = pathToKey(absolutePath);
if (key.length() == 0) { // root always exists
return newDirectory(null, absolutePath);
@@ -2152,10 +2062,6 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Listing status for {}", f.toString());
Path absolutePath = makeAbsolute(f);
-
- performAuthCheck(absolutePath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), "list");
-
String key = pathToKey(absolutePath);
Set<FileStatus> status = new TreeSet<FileStatus>();
FileMetadata meta = null;
@@ -2378,10 +2284,6 @@ public class NativeAzureFileSystem extends FileSystem {
}
Path absolutePath = makeAbsolute(f);
-
- performAuthCheck(absolutePath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), "mkdirs");
-
PermissionStatus permissionStatus = null;
if(noUmask) {
// ensure owner still has wx permissions at the minimum
@@ -2435,10 +2337,6 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Opening file: {}", f.toString());
Path absolutePath = makeAbsolute(f);
-
- performAuthCheck(absolutePath.toString(),
- WasbAuthorizationOperations.READ.toString(), "read");
-
String key = pathToKey(absolutePath);
FileMetadata meta = null;
try {
@@ -2495,12 +2393,7 @@ public class NativeAzureFileSystem extends FileSystem {
+ " through WASB that has colons in the name");
}
- Path absolutePath = makeAbsolute(src);
-
- performAuthCheck(absolutePath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), "rename");
-
- String srcKey = pathToKey(absolutePath);
+ String srcKey = pathToKey(makeAbsolute(src));
if (srcKey.length() == 0) {
// Cannot rename root of file system
@@ -2802,10 +2695,6 @@ public class NativeAzureFileSystem extends FileSystem {
@Override
public void setPermission(Path p, FsPermission permission) throws FileNotFoundException, IOException {
Path absolutePath = makeAbsolute(p);
-
- performAuthCheck(absolutePath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), "setPermission");
-
String key = pathToKey(absolutePath);
FileMetadata metadata = null;
try {
@@ -2844,10 +2733,6 @@ public class NativeAzureFileSystem extends FileSystem {
public void setOwner(Path p, String username, String groupname)
throws IOException {
Path absolutePath = makeAbsolute(p);
-
- performAuthCheck(absolutePath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), "setOwner");
-
String key = pathToKey(absolutePath);
FileMetadata metadata = null;
@@ -2910,42 +2795,6 @@ public class NativeAzureFileSystem extends FileSystem {
isClosed = true;
}
- @Override
- public Token<?> getDelegationToken(final String renewer) throws IOException {
- if(kerberosSupportEnabled) {
- try {
- final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- UserGroupInformation connectUgi = ugi.getRealUser();
- final UserGroupInformation proxyUser = connectUgi;
- if (connectUgi == null) {
- connectUgi = ugi;
- }
- if(!connectUgi.hasKerberosCredentials()){
- connectUgi = UserGroupInformation.getLoginUser();
- }
- connectUgi.checkTGTAndReloginFromKeytab();
- return connectUgi.doAs(new PrivilegedExceptionAction<Token<?>>() {
- @Override
- public Token<?> run() throws Exception {
- return authURL.getDelegationToken(new URL(credServiceUrl
- + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT),
- authToken, renewer, (proxyUser != null)? ugi.getShortUserName(): null);
- }
- });
- } catch (Exception ex) {
- LOG.error("Error in fetching the delegation token from remote service",
- ex);
- if (ex instanceof IOException) {
- throw (IOException) ex;
- } else {
- throw new IOException(ex);
- }
- }
- } else {
- return super.getDelegationToken(renewer);
- }
- }
-
/**
* A handler that defines what to do with blobs whose upload was
* interrupted.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
index 43672b2..404419d 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
@@ -19,23 +19,10 @@
package org.apache.hadoop.fs.azure;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
-import java.net.UnknownHostException;
-import java.security.PrivilegedExceptionAction;
-import java.util.Iterator;
-import org.apache.commons.lang.Validate;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.azure.security.Constants;
-import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import org.apache.hadoop.security.authentication.client.Authenticator;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
@@ -56,6 +43,12 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
public static final Logger LOG =
LoggerFactory.getLogger(AzureNativeFileSystemStore.class);
+ /**
+ * Configuration parameter name expected in the Configuration
+ * object to provide the url of the remote service {@value}
+ */
+ private static final String KEY_CRED_SERVICE_URL =
+ "fs.azure.cred.service.url";
/**
* Container SAS Key generation OP name. {@value}
@@ -89,7 +82,7 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
* Query parameter name for user info {@value}
*/
private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME =
- "delegation";
+ "delegation_token";
/**
* Query parameter name for the relative path inside the storage
@@ -101,40 +94,24 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
private String delegationToken = "";
private String credServiceUrl = "";
private WasbRemoteCallHelper remoteCallHelper = null;
- private boolean isSecurityEnabled;
- private boolean isKerberosSupportEnabled;
public RemoteSASKeyGeneratorImpl(Configuration conf) {
super(conf);
}
- public boolean initialize(Configuration conf) {
+ public boolean initialize(Configuration conf, String delegationToken) {
LOG.debug("Initializing RemoteSASKeyGeneratorImpl instance");
- Iterator<Token<? extends TokenIdentifier>> tokenIterator = null;
- try {
- tokenIterator = UserGroupInformation.getCurrentUser().getCredentials()
- .getAllTokens().iterator();
- while (tokenIterator.hasNext()) {
- Token<? extends TokenIdentifier> iteratedToken = tokenIterator.next();
- if (iteratedToken.getKind().equals(WasbDelegationTokenIdentifier.TOKEN_KIND)) {
- delegationToken = iteratedToken.encodeToUrlString();
- }
- }
- } catch (IOException e) {
- LOG.error("Error in fetching the WASB delegation token");
- }
+ credServiceUrl = conf.get(KEY_CRED_SERVICE_URL);
- try {
- credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, String
- .format("http://%s:%s",
- InetAddress.getLocalHost().getCanonicalHostName(),
- Constants.DEFAULT_CRED_SERVICE_PORT));
- } catch (UnknownHostException e) {
- LOG.error("Invalid CredService Url, configure it correctly.");
+ if (delegationToken == null || delegationToken.isEmpty()) {
+ LOG.error("Delegation Token not provided for initialization"
+ + " of RemoteSASKeyGenerator");
return false;
}
+ this.delegationToken = delegationToken;
+
if (credServiceUrl == null || credServiceUrl.isEmpty()) {
LOG.error("CredService Url not found in configuration to initialize"
+ " RemoteSASKeyGenerator");
@@ -142,17 +119,16 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
}
remoteCallHelper = new WasbRemoteCallHelper();
- this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
- this.isKerberosSupportEnabled = conf.getBoolean(
- Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
- LOG.debug("Initialization of RemoteSASKeyGenerator instance successful");
+ LOG.debug("Initialization of RemoteSASKeyGenerator instance successfull");
return true;
}
@Override
public URI getContainerSASUri(String storageAccount, String container)
throws SASKeyGenerationException {
+
try {
+
LOG.debug("Generating Container SAS Key for Container {} "
+ "inside Storage Account {} ", container, storageAccount);
URIBuilder uriBuilder = new URIBuilder(credServiceUrl);
@@ -163,131 +139,84 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
container);
uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME, ""
+ getSasKeyExpiryPeriod());
- if (isSecurityEnabled && (delegationToken != null && !delegationToken
- .isEmpty())) {
- uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
- this.delegationToken);
- }
-
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- UserGroupInformation connectUgi = ugi.getRealUser();
- if (connectUgi == null) {
- connectUgi = ugi;
+ uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
+ this.delegationToken);
+
+ RemoteSASKeyGenerationResponse sasKeyResponse =
+ makeRemoteRequest(uriBuilder.build());
+
+ if (sasKeyResponse == null) {
+ throw new SASKeyGenerationException("RemoteSASKeyGenerationResponse"
+ + " object null from remote call");
+ } else if (sasKeyResponse.getResponseCode()
+ == REMOTE_CALL_SUCCESS_CODE) {
+ return new URI(sasKeyResponse.getSasKey());
} else {
- uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName());
- }
-
- if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){
- connectUgi = UserGroupInformation.getLoginUser();
+ throw new SASKeyGenerationException("Remote Service encountered error"
+ + " in SAS Key generation : "
+ + sasKeyResponse.getResponseMessage());
}
- return getSASKey(uriBuilder.build(), connectUgi);
} catch (URISyntaxException uriSyntaxEx) {
throw new SASKeyGenerationException("Encountered URISyntaxException "
+ "while building the HttpGetRequest to remote cred service",
uriSyntaxEx);
- } catch (IOException e) {
- throw new SASKeyGenerationException("Encountered IOException"
- + " while building the HttpGetRequest to remote service", e);
}
}
@Override
public URI getRelativeBlobSASUri(String storageAccount, String container,
String relativePath) throws SASKeyGenerationException {
+
try {
+
LOG.debug("Generating RelativePath SAS Key for relativePath {} inside"
+ " Container {} inside Storage Account {} ",
relativePath, container, storageAccount);
URIBuilder uriBuilder = new URIBuilder(credServiceUrl);
uriBuilder.setPath("/" + BLOB_SAS_OP);
- uriBuilder.addParameter(STORAGE_ACCOUNT_QUERY_PARAM_NAME, storageAccount);
- uriBuilder.addParameter(CONTAINER_QUERY_PARAM_NAME, container);
+ uriBuilder.addParameter(STORAGE_ACCOUNT_QUERY_PARAM_NAME,
+ storageAccount);
+ uriBuilder.addParameter(CONTAINER_QUERY_PARAM_NAME,
+ container);
uriBuilder.addParameter(RELATIVE_PATH_QUERY_PARAM_NAME,
relativePath);
uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME, ""
+ getSasKeyExpiryPeriod());
-
- if (isSecurityEnabled && (delegationToken != null && !delegationToken
- .isEmpty())) {
- uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
- this.delegationToken);
- }
-
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- UserGroupInformation connectUgi = ugi.getRealUser();
- if (connectUgi == null) {
- connectUgi = ugi;
- } else{
- uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName());
- }
-
- if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){
- connectUgi = UserGroupInformation.getLoginUser();
+ uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
+ this.delegationToken);
+
+ RemoteSASKeyGenerationResponse sasKeyResponse =
+ makeRemoteRequest(uriBuilder.build());
+
+ if (sasKeyResponse == null) {
+ throw new SASKeyGenerationException("RemoteSASKeyGenerationResponse"
+ + " object null from remote call");
+ } else if (sasKeyResponse.getResponseCode()
+ == REMOTE_CALL_SUCCESS_CODE) {
+ return new URI(sasKeyResponse.getSasKey());
+ } else {
+ throw new SASKeyGenerationException("Remote Service encountered error"
+ + " in SAS Key generation : "
+ + sasKeyResponse.getResponseMessage());
}
- return getSASKey(uriBuilder.build(), connectUgi);
} catch (URISyntaxException uriSyntaxEx) {
throw new SASKeyGenerationException("Encountered URISyntaxException"
+ " while building the HttpGetRequest to " + " remote service",
uriSyntaxEx);
- } catch (IOException e) {
- throw new SASKeyGenerationException("Encountered IOException"
- + " while building the HttpGetRequest to remote service", e);
- }
- }
-
- private URI getSASKey(final URI uri, UserGroupInformation connectUgi)
- throws URISyntaxException, SASKeyGenerationException {
- RemoteSASKeyGenerationResponse sasKeyResponse = null;
- try {
- connectUgi.checkTGTAndReloginFromKeytab();
- sasKeyResponse = connectUgi.doAs(new PrivilegedExceptionAction<RemoteSASKeyGenerationResponse>() {
- @Override
- public RemoteSASKeyGenerationResponse run() throws Exception {
- AuthenticatedURL.Token token = null;
- if (isKerberosSupportEnabled && UserGroupInformation.isSecurityEnabled() && (
- delegationToken == null || delegationToken.isEmpty())) {
- token = new AuthenticatedURL.Token();
- final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator();
- kerberosAuthenticator.authenticate(uri.toURL(), token);
- Validate.isTrue(token.isSet(),
- "Authenticated Token is NOT present. The request cannot proceed.");
- }
- return makeRemoteRequest(uri, (token != null ? token.toString() : null));
- }
- });
- } catch (InterruptedException e) {
- LOG.error("Error fetching the SAS Key from Remote Service", e);
- } catch (IOException e) {
- LOG.error("Error fetching the SAS Key from Remote Service", e);
- }
-
- if (sasKeyResponse == null) {
- throw new SASKeyGenerationException(
- "RemoteSASKeyGenerationResponse" + " object null from remote call");
- } else if (sasKeyResponse.getResponseCode() == REMOTE_CALL_SUCCESS_CODE) {
- return new URI(sasKeyResponse.getSasKey());
- } else {
- throw new SASKeyGenerationException("Remote Service encountered error"
- + " in SAS Key generation : " + sasKeyResponse.getResponseMessage());
}
}
/**
* Helper method to make a remote request.
* @param uri - Uri to use for the remote request
- * @param token - hadoop.auth token for the remote request
* @return RemoteSASKeyGenerationResponse
*/
- private RemoteSASKeyGenerationResponse makeRemoteRequest(URI uri, String token)
+ private RemoteSASKeyGenerationResponse makeRemoteRequest(URI uri)
throws SASKeyGenerationException {
try {
- HttpGet httpGet = new HttpGet(uri);
- if(token != null){
- httpGet.setHeader("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + token);
- }
String responseBody =
- remoteCallHelper.makeRemoteGetRequest(httpGet);
+ remoteCallHelper.makeRemoteGetRequest(new HttpGet(uri));
ObjectMapper objectMapper = new ObjectMapper();
return objectMapper.readValue(responseBody,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
deleted file mode 100644
index bb1093d..0000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URISyntaxException;
-import java.security.PrivilegedExceptionAction;
-import java.util.Iterator;
-
-import org.apache.commons.lang.Validate;
-import org.apache.hadoop.conf.Configuration;
-
-import org.apache.hadoop.fs.azure.security.Constants;
-import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import org.apache.hadoop.security.authentication.client.Authenticator;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.URIBuilder;
-
-import com.fasterxml.jackson.core.JsonParseException;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.azure.WasbRemoteCallHelper.REMOTE_CALL_SUCCESS_CODE;
-
-/**
- * Class implementing WasbAuthorizerInterface using a remote
- * service that implements the authorization operation. This
- * class expects the url of the remote service to be passed
- * via config.
- */
-public class RemoteWasbAuthorizerImpl implements WasbAuthorizerInterface {
-
- public static final Logger LOG =
- LoggerFactory.getLogger(RemoteWasbAuthorizerImpl.class);
- private String remoteAuthorizerServiceUrl = "";
-
- /**
- * Configuration parameter name expected in the Configuration object to
- * provide the url of the remote service. {@value}
- */
- public static final String KEY_REMOTE_AUTH_SERVICE_URL =
- "fs.azure.authorization.remote.service.url";
-
- /**
- * Authorization operation OP name in the remote service {@value}
- */
- private static final String CHECK_AUTHORIZATION_OP =
- "CHECK_AUTHORIZATION";
-
- /**
- * Query parameter specifying the access operation type. {@value}
- */
- private static final String ACCESS_OPERATION_QUERY_PARAM_NAME =
- "operation_type";
-
- /**
- * Query parameter specifying the wasb absolute path. {@value}
- */
- private static final String WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME =
- "wasb_absolute_path";
-
- /**
- * Query parameter name for user info {@value}
- */
- private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME =
- "delegation";
-
- private WasbRemoteCallHelper remoteCallHelper = null;
- private String delegationToken;
- private boolean isSecurityEnabled;
- private boolean isKerberosSupportEnabled;
-
- @Override
- public void init(Configuration conf)
- throws WasbAuthorizationException, IOException {
- LOG.debug("Initializing RemoteWasbAuthorizerImpl instance");
- delegationToken = UserGroupInformation.getCurrentUser().getCredentials().getToken(WasbDelegationTokenIdentifier.TOKEN_KIND).encodeToUrlString();
-
- remoteAuthorizerServiceUrl = conf.get(KEY_REMOTE_AUTH_SERVICE_URL, String
- .format("http://%s:%s",
- InetAddress.getLocalHost().getCanonicalHostName(),
- Constants.DEFAULT_CRED_SERVICE_PORT));
-
- if (remoteAuthorizerServiceUrl == null
- || remoteAuthorizerServiceUrl.isEmpty()) {
- throw new WasbAuthorizationException(
- "fs.azure.authorization.remote.service.url config not set"
- + " in configuration.");
- }
-
- this.remoteCallHelper = new WasbRemoteCallHelper();
- this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
- this.isKerberosSupportEnabled = conf.getBoolean(
- Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
- }
-
- @Override
- public boolean authorize(String wasbAbsolutePath, String accessType)
- throws WasbAuthorizationException, IOException {
- try {
- final URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl);
- uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP);
- uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME,
- wasbAbsolutePath);
- uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME,
- accessType);
- if (isSecurityEnabled && (delegationToken != null && !delegationToken
- .isEmpty())) {
- uriBuilder
- .addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, delegationToken);
- }
- String responseBody = null;
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- UserGroupInformation connectUgi = ugi.getRealUser();
- if (connectUgi == null) {
- connectUgi = ugi;
- } else{
- uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName());
- }
- if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){
- connectUgi = UserGroupInformation.getLoginUser();
- }
- connectUgi.checkTGTAndReloginFromKeytab();
- try {
- responseBody = connectUgi.doAs(new PrivilegedExceptionAction<String>(){
- @Override
- public String run() throws Exception {
- AuthenticatedURL.Token token = null;
- HttpGet httpGet = new HttpGet(uriBuilder.build());
- if (isKerberosSupportEnabled && UserGroupInformation.isSecurityEnabled() && (
- delegationToken == null || delegationToken.isEmpty())) {
- token = new AuthenticatedURL.Token();
- final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator();
- kerberosAuthenticator
- .authenticate(uriBuilder.build().toURL(), token);
- Validate.isTrue(token.isSet(),
- "Authenticated Token is NOT present. The request cannot proceed.");
- if(token != null){
- httpGet.setHeader("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + token);
- }
- }
- return remoteCallHelper.makeRemoteGetRequest(httpGet);
- }});
- } catch (InterruptedException e) {
- LOG.error("Error in check authorization", e);
- }
-
- ObjectMapper objectMapper = new ObjectMapper();
- RemoteAuthorizerResponse authorizerResponse =
- objectMapper.readValue(responseBody, RemoteAuthorizerResponse.class);
-
- if (authorizerResponse == null) {
- throw new WasbAuthorizationException(
- "RemoteAuthorizerResponse object null from remote call");
- } else if (authorizerResponse.getResponseCode()
- == REMOTE_CALL_SUCCESS_CODE) {
- return authorizerResponse.getAuthorizationResult();
- } else {
- throw new WasbAuthorizationException("Remote authorization"
- + " serivce encountered an error "
- + authorizerResponse.getResponseMessage());
- }
- } catch (URISyntaxException | WasbRemoteCallException
- | JsonParseException | JsonMappingException ex) {
- throw new WasbAuthorizationException(ex);
- }
- }
-}
-
-/**
- * POJO representing the response expected from a remote
- * authorization service.
- * The remote service is expected to return the authorization
- * response in the following JSON format
- * {
- * "responseCode" : 0 or non-zero <int>,
- * "responseMessage" : relavant message of failure <String>
- * "authorizationResult" : authorization result <boolean>
- * true - if auhorization allowed
- * false - otherwise.
- *
- * }
- */
-class RemoteAuthorizerResponse {
-
- private int responseCode;
- private boolean authorizationResult;
- private String responseMessage;
-
- public RemoteAuthorizerResponse(){
- }
-
- public RemoteAuthorizerResponse(int responseCode,
- boolean authorizationResult, String message) {
- this.responseCode = responseCode;
- this.authorizationResult = authorizationResult;
- this.responseMessage = message;
- }
-
- public int getResponseCode() {
- return responseCode;
- }
-
- public void setResponseCode(int responseCode) {
- this.responseCode = responseCode;
- }
-
- public boolean getAuthorizationResult() {
- return authorizationResult;
- }
-
- public void setAuthorizationResult(boolean authorizationResult) {
- this.authorizationResult = authorizationResult;
- }
-
- public String getResponseMessage() {
- return responseMessage;
- }
-
- public void setResponseMessage(String message) {
- this.responseMessage = message;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
index 5ec9136..6749a76 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
@@ -69,17 +69,19 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
public static final String SAS_ERROR_CODE = "SAS Error";
private SASKeyGeneratorInterface sasKeyGenerator;
private String storageAccount;
+ private String delegationToken;
public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode,
- Configuration conf)
+ Configuration conf, String delegationToken)
throws SecureModeException {
+ this.delegationToken = delegationToken;
if (useLocalSASKeyMode) {
this.sasKeyGenerator = new LocalSASKeyGeneratorImpl(conf);
} else {
RemoteSASKeyGeneratorImpl remoteSasKeyGenerator =
new RemoteSASKeyGeneratorImpl(conf);
- if (!remoteSasKeyGenerator.initialize(conf)) {
+ if (!remoteSasKeyGenerator.initialize(conf, this.delegationToken)) {
throw new SecureModeException("Remote SAS Key mode could"
+ " not be initialized");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
deleted file mode 100644
index eff9248..0000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-/**
- * Exception that gets thrown during the authorization failures
- * in WASB.
- */
-public class WasbAuthorizationException extends AzureException {
-
- private static final long serialVersionUID = 1L;
-
- public WasbAuthorizationException(String message) {
- super(message);
- }
-
- public WasbAuthorizationException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public WasbAuthorizationException(Throwable t) {
- super(t);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
deleted file mode 100644
index bd76837..0000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-/**
- * Different authorization operations supported
- * in WASB.
- */
-
-public enum WasbAuthorizationOperations {
-
- READ, WRITE, EXECUTE;
-
- @Override
- public String toString() {
- switch(this) {
- case READ:
- return "read";
- case WRITE:
- return "write";
- case EXECUTE:
- return "execute";
- default:
- throw new IllegalArgumentException(
- "Invalid Auhtorization Operation");
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
deleted file mode 100644
index 0c61997..0000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Interface to implement authorization support in WASB.
- * API's of this interface will be implemented in the
- * StorageInterface Layer before making calls to Azure
- * Storage.
- */
-public interface WasbAuthorizerInterface {
- /**
- * Initializer method
- * @param conf - Configuration object
- */
- public void init(Configuration conf)
- throws WasbAuthorizationException, IOException;
-
- /**
- * Authorizer API to authorize access in WASB.
- * @param wasbAbsolutePath : Absolute WASB Path used for access.
- * @param accessType : Type of access
- * @return : true - If access allowed false - If access is not allowed.
- */
- public boolean authorize(String wasbAbsolutePath, String accessType)
- throws WasbAuthorizationException, IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
deleted file mode 100644
index 1050687..0000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-/**
- * Constants for used with WASB security implementation.
- */
-public final class Constants {
-
- private Constants() {
- }
-
- /**
- * Configuration parameter name expected in the Configuration
- * object to provide the url of the remote service {@value}
- */
- public static final String KEY_CRED_SERVICE_URL = "fs.azure.cred.service.url";
- /**
- * Default port of the remote service used as delegation token manager and Azure storage SAS key generator.
- */
- public static final int DEFAULT_CRED_SERVICE_PORT = 50911;
-
- /**
- * Default remote delegation token manager endpoint.
- */
- public static final String DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT = "/tokenmanager/v1";
-
- /**
- * The configuration property to enable Kerberos support.
- */
-
- public static final String AZURE_KERBEROS_SUPPORT_PROPERTY_NAME = "fs.azure.enable.kerberos.support";
-
- /**
- * Parameter to be used for impersonation.
- */
- public static final String DOAS_PARAM="doas";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java
deleted file mode 100644
index 530e045..0000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
-
-/**
- * Delegation token Identifier for WASB delegation tokens.
- */
-public class WasbDelegationTokenIdentifier extends DelegationTokenIdentifier {
- public static final Text TOKEN_KIND = new Text("WASB delegation");
-
- public WasbDelegationTokenIdentifier(){
- super(TOKEN_KIND);
- }
-
- public WasbDelegationTokenIdentifier(Text kind) {
- super(kind);
- }
-
- public WasbDelegationTokenIdentifier(Text kind, Text owner, Text renewer,
- Text realUser) {
- super(kind, owner, renewer, realUser);
- }
-
- @Override
- public Text getKind() {
- return TOKEN_KIND;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java
deleted file mode 100644
index 09b7349..0000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenRenewer;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
-import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URL;
-import java.security.PrivilegedExceptionAction;
-
-/**
- * Token Renewer for renewing WASB delegation tokens with remote service.
- */
-public class WasbTokenRenewer extends TokenRenewer {
- public static final Logger LOG = LoggerFactory
- .getLogger(WasbTokenRenewer.class);
-
- @Override
- public boolean handleKind(Text kind) {
- return WasbDelegationTokenIdentifier.TOKEN_KIND.equals(kind);
- }
-
- @Override
- public boolean isManaged(Token<?> token) throws IOException {
- return true;
- }
-
- @Override
- public long renew(final Token<?> token, Configuration conf)
- throws IOException, InterruptedException {
- LOG.debug("Renewing the delegation token");
- final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- UserGroupInformation connectUgi = ugi.getRealUser();
- final UserGroupInformation proxyUser = connectUgi;
- if (connectUgi == null) {
- connectUgi = ugi;
- }
- if(!connectUgi.hasKerberosCredentials()){
- connectUgi = UserGroupInformation.getLoginUser();
- }
- connectUgi.checkTGTAndReloginFromKeytab();
- final DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
- authToken
- .setDelegationToken((Token<AbstractDelegationTokenIdentifier>) token);
- final String credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL,
- String.format("http://%s:%s",
- InetAddress.getLocalHost().getCanonicalHostName(),
- Constants.DEFAULT_CRED_SERVICE_PORT));
- DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator();
- final DelegationTokenAuthenticatedURL authURL = new DelegationTokenAuthenticatedURL(
- authenticator);
-
- return connectUgi.doAs(new PrivilegedExceptionAction<Long>() {
- @Override
- public Long run() throws Exception {
- return authURL.renewDelegationToken(new URL(credServiceUrl
- + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT),
- authToken, (proxyUser != null) ? ugi.getShortUserName() : null);
- }
- });
- }
-
- @Override
- public void cancel(final Token<?> token, Configuration conf)
- throws IOException, InterruptedException {
- LOG.debug("Cancelling the delegation token");
- final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- UserGroupInformation connectUgi = ugi.getRealUser();
- final UserGroupInformation proxyUser = connectUgi;
- if (connectUgi == null) {
- connectUgi = ugi;
- }
- if(!connectUgi.hasKerberosCredentials()){
- connectUgi = UserGroupInformation.getLoginUser();
- }
- connectUgi.checkTGTAndReloginFromKeytab();
- final DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
- authToken
- .setDelegationToken((Token<AbstractDelegationTokenIdentifier>) token);
- final String credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL,
- String.format("http://%s:%s",
- InetAddress.getLocalHost().getCanonicalHostName(),
- Constants.DEFAULT_CRED_SERVICE_PORT));
- DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator();
- final DelegationTokenAuthenticatedURL authURL = new DelegationTokenAuthenticatedURL(
- authenticator);
- connectUgi.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- authURL.cancelDelegationToken(new URL(credServiceUrl
- + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT),
- authToken, (proxyUser != null) ? ugi.getShortUserName() : null);
- return null;
- }
- });
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html
deleted file mode 100644
index fe58c0a..0000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html
+++ /dev/null
@@ -1,28 +0,0 @@
-<html>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<body>
-
-<p>
- Infrastructure for WASB client Security to work with Kerberos and Delegation
- tokens.
-</p>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
deleted file mode 100644
index 7ec8216..0000000
--- a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
deleted file mode 100644
index f9c590a..0000000
--- a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.hadoop.fs.azure.security.WasbTokenRenewer
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index a06d297..2865223 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -333,40 +333,6 @@ The service is expected to return a response in JSON format:
"sasKey" : Requested SAS Key <String>
}
```
-
-## <a name="WASB Authorization" />Authorization Support in WASB.
-
-Authorization support can be enabled in WASB using the following configuration:
-
-```
- <property>
- <name>fs.azure.authorization</name>
- <value>true</value>
- </property>
-```
- The current implementation of authorization relies on the presence of an external service that can enforce
- the authorization. The service is expected to be running on a URL provided by the following config.
-
-```
- <property>
- <name>fs.azure.authorization.remote.service.url</name>
- <value>{URL}</value>
- </property>
-```
-
- The remote service is expected to provide support for the following REST call: ```{URL}/CHECK_AUTHORIZATION```
- An example request:
- ```{URL}/CHECK_AUTHORIZATION?wasb_absolute_path=<absolute_path>&operation_type=<operation type>&delegation_token=<delegation token>```
-
- The service is expected to return a response in JSON format:
- ```
- {
- "responseCode" : 0 or non-zero <int>,
- "responseMessage" : relavant message on failure <String>,
- "authorizationResult" : true/false <boolean>
- }
- ```
-
## <a name="Testing_the_hadoop-azure_Module" />Testing the hadoop-azure Module
The hadoop-azure module includes a full suite of unit tests. Most of the tests
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
deleted file mode 100644
index af5a537..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A mock wasb authorizer implementation.
- */
-
-public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface {
-
- private Map<AuthorizationComponent, Boolean> authRules;
-
- @Override
- public void init(Configuration conf) {
- authRules = new HashMap<AuthorizationComponent, Boolean>();
- }
-
- public void addAuthRule(String wasbAbsolutePath,
- String accessType, boolean access) {
- AuthorizationComponent component =
- new AuthorizationComponent(wasbAbsolutePath, accessType);
- this.authRules.put(component, access);
- }
-
- @Override
- public boolean authorize(String wasbAbsolutePath, String accessType)
- throws WasbAuthorizationException {
-
- AuthorizationComponent component =
- new AuthorizationComponent(wasbAbsolutePath, accessType);
-
- if (authRules.containsKey(component)) {
- return authRules.get(component);
- } else {
- return false;
- }
- }
-}
-
-class AuthorizationComponent {
-
- private String wasbAbsolutePath;
- private String accessType;
-
- public AuthorizationComponent(String wasbAbsolutePath,
- String accessType) {
- this.wasbAbsolutePath = wasbAbsolutePath;
- this.accessType = accessType;
- }
-
- @Override
- public int hashCode() {
- return this.wasbAbsolutePath.hashCode() ^ this.accessType.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
-
- if (obj == this) {
- return true;
- }
-
- if (obj == null
- || !(obj instanceof AuthorizationComponent)) {
- return false;
- }
-
- return ((AuthorizationComponent)obj).
- getWasbAbsolutePath().equals(this.wasbAbsolutePath)
- && ((AuthorizationComponent)obj).
- getAccessType().equals(this.accessType);
- }
-
- public String getWasbAbsolutePath() {
- return this.wasbAbsolutePath;
- }
-
- public String getAccessType() {
- return accessType;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d7d5aa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
deleted file mode 100644
index e765335..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-import com.sun.tools.javac.util.Assert;
-
-/**
- * Test class to hold all WASB authorization tests.
- */
-public class TestNativeAzureFileSystemAuthorization
- extends AbstractWasbTestBase {
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- Configuration conf = new Configuration();
- conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true");
- conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URL, "test_url");
- return AzureBlobStorageTestAccount.create(conf);
- }
-
- /**
- * Positive test to verify Create and delete access check
- * @throws Throwable
- */
- @Test
- public void testCreateAccessCheckPositive() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
- String testFile = "test.dat";
- Path testPath = new Path(fs.getWorkingDirectory(), testFile);
-
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
- authorizer.init(null);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), true);
- fs.updateWasbAuthorizer(authorizer);
- authorizer.addAuthRule(fs.getWorkingDirectory().toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), true);
-
- fs.create(testPath);
- Assert.check(fs.exists(testPath));
- fs.delete(testPath, false);
- }
-
- /**
- * Negative test to verify Create access check
- * @throws Throwable
- */
-
- @Test(expected=WasbAuthorizationException.class)
- public void testCreateAccessCheckNegative() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
- String testFile = "test.dat";
- Path testPath = new Path(fs.getWorkingDirectory(), testFile);
-
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
- authorizer.init(null);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.WRITE.toString(), false);
- fs.updateWasbAuthorizer(authorizer);
-
- fs.create(new Path(testFile));
- }
-
- /**
- * Positive test to verify Create and delete access check
- * @throws Throwable
- */
- @Test
- public void testListAccessCheckPositive() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
- String testFolder = "\\";
- Path testPath = new Path(fs.getWorkingDirectory(), testFolder);
-
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
- authorizer.init(null);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), true);
- fs.updateWasbAuthorizer(authorizer);
-
- fs.listStatus(testPath);
- }
-
- /**
- * Negative test to verify Create access check
- * @throws Throwable
- */
-
- @Test(expected=WasbAuthorizationException.class)
- public void testListAccessCheckNegative() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
- String testFolder = "\\";
- Path testPath = new Path(fs.getWorkingDirectory(), testFolder);
-
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
- authorizer.init(null);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), false);
- fs.updateWasbAuthorizer(authorizer);
-
- fs.listStatus(testPath);
- }
-
- /**
- * Positive test to verify rename access check.
- * @throws Throwable
- */
- @Test
- public void testRenameAccessCheckPositive() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
-
- String testFile = "test.dat";
- Path testPath = new Path(fs.getWorkingDirectory(), testFile);
- String renameFile = "test2.dat";
- Path renamePath = new Path(fs.getWorkingDirectory(), renameFile);
-
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
- authorizer.init(null);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), true);
- authorizer.addAuthRule(renamePath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), true);
- authorizer.addAuthRule(fs.getWorkingDirectory().toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), true);
- fs.updateWasbAuthorizer(authorizer);
- fs.create(testPath);
-
- Assert.check(fs.exists(testPath));
- fs.rename(testPath, renamePath);
- Assert.check(fs.exists(renamePath));
- fs.delete(renamePath, false);
- }
-
- /**
- * Negative test to verify rename access check.
- * @throws Throwable
- */
- @Test(expected=WasbAuthorizationException.class)
- public void testRenameAccessCheckNegative() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
- String testFile = "test.dat";
- Path testPath = new Path(fs.getWorkingDirectory(), testFile);
- Path renamePath = new Path("test2.dat");
-
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
- authorizer.init(null);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), false);
- fs.updateWasbAuthorizer(authorizer);
-
- try {
- fs.create(testPath);
-
- Assert.check(fs.exists(testPath));
- fs.rename(testPath, renamePath);
- Assert.check(fs.exists(renamePath));
- fs.delete(renamePath, false);
- } catch (WasbAuthorizationException ex) {
- throw ex;
- } finally {
- authorizer = new MockWasbAuthorizerImpl();
- authorizer.init(null);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), false);
- fs.updateWasbAuthorizer(authorizer);
- Assert.check(fs.exists(testPath));
- fs.delete(testPath, false);
- }
- }
-
- /**
- * Positive test for read access check.
- * @throws Throwable
- */
- @Test
- public void testReadAccessCheckPositive() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
- String testFile = "test.dat";
- Path testPath = new Path(fs.getWorkingDirectory(), testFile);
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
- authorizer.init(null);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), true);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.READ.toString(), true);
- authorizer.addAuthRule(fs.getWorkingDirectory().toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), true);
- fs.updateWasbAuthorizer(authorizer);
- fs.create(testPath);
- Assert.check(fs.exists(testPath));
- FSDataInputStream inputStream = fs.open(testPath);
- inputStream.close();
- fs.delete(testPath, false);
- }
-
- /**
- * Negative test to verify read access check.
- * @throws Throwable
- */
- @Test(expected=WasbAuthorizationException.class)
- public void testReadAccessCheckNegative() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
- String testFile = "test.dat";
- Path testPath = new Path(fs.getWorkingDirectory(), testFile);
- MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
- authorizer.init(null);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.EXECUTE.toString(), true);
- authorizer.addAuthRule(testPath.toString(),
- WasbAuthorizationOperations.READ.toString(), false);
- fs.updateWasbAuthorizer(authorizer);
-
- fs.create(new Path(testFile));
- Assert.check(fs.exists(testPath));
- FSDataInputStream inputStream = null;
- try {
- inputStream = fs.open(new Path(testFile));
- } catch (WasbAuthorizationException ex) {
- throw ex;
- } finally {
- fs.delete(new Path(testFile), false);
- if (inputStream != null) {
- inputStream.close();
- }
- }
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] hadoop git commit: HDFS-11479. Socket re-use address option
should be used in SimpleUdpServer. Contributed by Mukul Kumar Singh.
Posted by kk...@apache.org.
HDFS-11479. Socket re-use address option should be used in SimpleUdpServer. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/899d5c4d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/899d5c4d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/899d5c4d
Branch: refs/heads/YARN-5972
Commit: 899d5c4d49f00d90ddc3632efb2df92841867192
Parents: 82ef9ac
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Mar 1 12:59:02 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Mar 1 12:59:02 2017 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899d5c4d/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
index c9cb2b3..d691aba 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
@@ -63,6 +63,7 @@ public class SimpleUdpServer {
server.setOption("broadcast", "false");
server.setOption("sendBufferSize", SEND_BUFFER_SIZE);
server.setOption("receiveBufferSize", RECEIVE_BUFFER_SIZE);
+ server.setOption("reuseAddress", true);
// Listen to the UDP port
ch = server.bind(new InetSocketAddress(port));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] hadoop git commit: YARN-6275. Fail to show real-time tracking
charts in SLS (yufeigu via rkanter)
Posted by kk...@apache.org.
YARN-6275. Fail to show real-time tracking charts in SLS (yufeigu via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1598fd3b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1598fd3b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1598fd3b
Branch: refs/heads/YARN-5972
Commit: 1598fd3b7948b3592775e3be3227c4a336122bc9
Parents: 38d75df
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Mar 7 13:47:52 2017 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Mar 7 13:47:52 2017 -0800
----------------------------------------------------------------------
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 8 ++++++--
.../main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java | 2 ++
2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1598fd3b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
index 19b5c34..fb53045 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
@@ -103,12 +103,16 @@ function run_simulation() {
hadoop_java_exec sls org.apache.hadoop.yarn.sls.SLSRunner ${args}
}
+this="${BASH_SOURCE-$0}"
+bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+
+# copy 'html' directory to current directory to make sure web sever can access
+cp -r "${bin}/../html" "$(pwd)"
+
# let's locate libexec...
if [[ -n "${HADOOP_HOME}" ]]; then
HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
- this="${BASH_SOURCE-$0}"
- bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1598fd3b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
index abdf106..33d4846 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.scheduler.FairSchedulerMetrics;
import org.apache.hadoop.yarn.sls.scheduler.SchedulerMetrics;
import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper;
+import org.eclipse.jetty.http.MimeTypes;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
@@ -118,6 +119,7 @@ public class SLSWebApp extends HttpServlet {
public void start() throws Exception {
// static files
final ResourceHandler staticHandler = new ResourceHandler();
+ staticHandler.setMimeTypes(new MimeTypes());
staticHandler.setResourceBase("html");
Handler handler = new AbstractHandler() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] hadoop git commit: MAPREDUCE-6839. TestRecovery.testCrashed
failed (pairg via rkanter)
Posted by kk...@apache.org.
MAPREDUCE-6839. TestRecovery.testCrashed failed (pairg via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38d75dfd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38d75dfd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38d75dfd
Branch: refs/heads/YARN-5972
Commit: 38d75dfd3a643f8a1acd52e025a466d65065b60e
Parents: e0c239c
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Mar 7 13:34:46 2017 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Mar 7 13:34:46 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/mapreduce/v2/app/TestRecovery.java | 12 +++---------
1 file changed, 3 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38d75dfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
index 071575a..6332c5d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
@@ -159,9 +159,7 @@ public class TestRecovery {
app.waitForState(task1Attempt1, TaskAttemptState.RUNNING);
app.waitForState(task2Attempt, TaskAttemptState.RUNNING);
- // reduces must be in NEW state
- Assert.assertEquals("Reduce Task state not correct",
- TaskState.RUNNING, reduceTask.getReport().getTaskState());
+ app.waitForState(reduceTask, TaskState.RUNNING);
/////////// Play some games with the TaskAttempts of the first task //////
//send the fail signal to the 1st map task attempt
@@ -1301,9 +1299,7 @@ public class TestRecovery {
app.waitForState(task1Attempt2, TaskAttemptState.RUNNING);
app.waitForState(task2Attempt, TaskAttemptState.RUNNING);
- // reduces must be in NEW state
- Assert.assertEquals("Reduce Task state not correct",
- TaskState.RUNNING, reduceTask.getReport().getTaskState());
+ app.waitForState(reduceTask, TaskState.RUNNING);
//send the done signal to the map 1 attempt 1
app.getContext().getEventHandler().handle(
@@ -1431,9 +1427,7 @@ public class TestRecovery {
app.waitForState(task1Attempt, TaskAttemptState.RUNNING);
app.waitForState(task2Attempt, TaskAttemptState.RUNNING);
- // reduces must be in NEW state
- Assert.assertEquals("Reduce Task state not correct",
- TaskState.RUNNING, reduceTask.getReport().getTaskState());
+ app.waitForState(reduceTask, TaskState.RUNNING);
//send the done signal to the 1st map attempt
app.getContext().getEventHandler().handle(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] hadoop git commit: YARN-6218. Fix TestAMRMClient when using
FairScheduler. (Miklos Szegedi via rchiang)
Posted by kk...@apache.org.
YARN-6218. Fix TestAMRMClient when using FairScheduler. (Miklos Szegedi via rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2148b839
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2148b839
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2148b839
Branch: refs/heads/YARN-5972
Commit: 2148b83993fd8ce73bcbc7677c57ee5028a59cd4
Parents: 3085a60
Author: Ray Chiang <rc...@apache.org>
Authored: Fri Mar 3 12:55:45 2017 -0800
Committer: Ray Chiang <rc...@apache.org>
Committed: Fri Mar 3 13:04:51 2017 -0800
----------------------------------------------------------------------
.../yarn/client/api/impl/TestAMRMClient.java | 139 ++++++++++++-------
.../scheduler/fair/FairScheduler.java | 3 +-
2 files changed, 87 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2148b839/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 4f73bac..43c0271 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -64,47 +64,71 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.eclipse.jetty.util.log.Log;
import com.google.common.base.Supplier;
+/**
+ * Test application master client class to resource manager.
+ */
+@RunWith(value = Parameterized.class)
public class TestAMRMClient {
- static Configuration conf = null;
- static MiniYARNCluster yarnCluster = null;
- static YarnClient yarnClient = null;
- static List<NodeReport> nodeReports = null;
- static ApplicationAttemptId attemptId = null;
- static int nodeCount = 3;
+ private String schedulerName = null;
+ private Configuration conf = null;
+ private MiniYARNCluster yarnCluster = null;
+ private YarnClient yarnClient = null;
+ private List<NodeReport> nodeReports = null;
+ private ApplicationAttemptId attemptId = null;
+ private int nodeCount = 3;
static final int rolling_interval_sec = 13;
static final long am_expire_ms = 4000;
- static Resource capability;
- static Priority priority;
- static Priority priority2;
- static String node;
- static String rack;
- static String[] nodes;
- static String[] racks;
+ private Resource capability;
+ private Priority priority;
+ private Priority priority2;
+ private String node;
+ private String rack;
+ private String[] nodes;
+ private String[] racks;
private final static int DEFAULT_ITERATION = 3;
- @BeforeClass
- public static void setup() throws Exception {
+ public TestAMRMClient(String schedulerName) {
+ this.schedulerName = schedulerName;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ List<Object[]> list = new ArrayList<Object[]>(2);
+ list.add(new Object[] {CapacityScheduler.class.getName()});
+ list.add(new Object[] {FairScheduler.class.getName()});
+ return list;
+ }
+
+ @Before
+ public void setup() throws Exception {
// start minicluster
conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_SCHEDULER, schedulerName);
conf.setLong(
YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
rolling_interval_sec);
@@ -138,10 +162,7 @@ public class TestAMRMClient {
rack = nodeReports.get(0).getRackName();
nodes = new String[]{ node };
racks = new String[]{ rack };
- }
-
- @Before
- public void startApp() throws Exception {
+
// submit new app
ApplicationSubmissionContext appContext =
yarnClient.createApplication().getApplicationSubmissionContext();
@@ -199,13 +220,10 @@ public class TestAMRMClient {
}
@After
- public void cancelApp() throws YarnException, IOException {
+ public void teardown() throws YarnException, IOException {
yarnClient.killApplication(attemptId.getApplicationId());
attemptId = null;
- }
-
- @AfterClass
- public static void tearDown() {
+
if (yarnClient != null && yarnClient.getServiceState() == STATE.STARTED) {
yarnClient.stop();
}
@@ -656,8 +674,8 @@ public class TestAMRMClient {
amClient.releaseAssignedContainer(container.getId());
}
if(allocatedContainerCount < containersRequestedAny) {
- // sleep to let NM's heartbeat to RM and trigger allocations
- sleep(100);
+ // let NM heartbeat to RM and trigger allocations
+ triggerSchedulingWithNMHeartBeat();
}
}
@@ -678,7 +696,27 @@ public class TestAMRMClient {
}
}
}
-
+
+ /**
+ * Make sure we get allocations regardless of timing issues.
+ */
+ private void triggerSchedulingWithNMHeartBeat() {
+ // Simulate fair scheduler update thread
+ RMContext context = yarnCluster.getResourceManager().getRMContext();
+ if (context.getScheduler() instanceof FairScheduler) {
+ FairScheduler scheduler = (FairScheduler)context.getScheduler();
+ scheduler.update();
+ }
+ // Trigger NM's heartbeat to RM and trigger allocations
+ for (RMNode rmNode : context.getRMNodes().values()) {
+ context.getScheduler().handle(new NodeUpdateSchedulerEvent(rmNode));
+ }
+ if (context.getScheduler() instanceof FairScheduler) {
+ FairScheduler scheduler = (FairScheduler)context.getScheduler();
+ scheduler.update();
+ }
+ }
+
@Test (timeout=60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
AMRMClientImpl<ContainerRequest> amClient = null;
@@ -811,8 +849,8 @@ public class TestAMRMClient {
allocatedContainerCount += allocResponse.getAllocatedContainers().size();
if(allocatedContainerCount == 0) {
- // sleep to let NM's heartbeat to RM and trigger allocations
- sleep(100);
+ // let NM heartbeat to RM and trigger allocations
+ triggerSchedulingWithNMHeartBeat();
}
}
return allocatedContainerCount;
@@ -934,6 +972,8 @@ public class TestAMRMClient {
@Test(timeout=60000)
public void testAMRMClientWithContainerResourceChange()
throws YarnException, IOException {
+ // Fair scheduler does not support resource change
+ Assume.assumeTrue(schedulerName.equals(CapacityScheduler.class.getName()));
AMRMClient<ContainerRequest> amClient = null;
try {
// start am rm client
@@ -981,8 +1021,8 @@ public class TestAMRMClient {
}
// send allocation requests
amClient.allocate(0.1f);
- // sleep to let NM's heartbeat to RM and trigger allocations
- sleep(150);
+ // let NM heartbeat to RM and trigger allocations
+ triggerSchedulingWithNMHeartBeat();
// get allocations
AllocateResponse allocResponse = amClient.allocate(0.1f);
List<Container> containers = allocResponse.getAllocatedContainers();
@@ -1012,14 +1052,14 @@ public class TestAMRMClient {
if (status.getState() == ContainerState.RUNNING) {
break;
}
- sleep(100);
+ sleep(10);
}
}
} catch (YarnException e) {
throw new AssertionError("Exception is not expected: " + e);
}
- // sleep to let NM's heartbeat to RM to confirm container launch
- sleep(200);
+ // let NM's heartbeat to RM to confirm container launch
+ triggerSchedulingWithNMHeartBeat();
return containers;
}
@@ -1079,7 +1119,7 @@ public class TestAMRMClient {
allocResponse.getUpdatedContainers();
Assert.assertEquals(1, updatedContainers.size());
// we should get increase allocation after the next NM's heartbeat to RM
- sleep(150);
+ triggerSchedulingWithNMHeartBeat();
// get allocations
allocResponse = amClient.allocate(0.1f);
updatedContainers =
@@ -1142,8 +1182,8 @@ public class TestAMRMClient {
}
if(allocatedContainerCount < containersRequestedAny) {
- // sleep to let NM's heartbeat to RM and trigger allocations
- sleep(100);
+ // let NM heartbeat to RM and trigger allocations
+ triggerSchedulingWithNMHeartBeat();
}
}
@@ -1225,8 +1265,8 @@ public class TestAMRMClient {
}
}
if(numIterations > 0) {
- // sleep to make sure NM's heartbeat
- sleep(100);
+ // let NM heartbeat to RM and trigger allocations
+ triggerSchedulingWithNMHeartBeat();
}
}
assertEquals(0, amClient.ask.size());
@@ -1284,8 +1324,8 @@ public class TestAMRMClient {
}
if(allocatedContainers.size() < containersRequestedAny) {
- // sleep to let NM's heartbeat to RM and trigger allocations
- sleep(100);
+ // let NM heartbeat to RM and trigger allocations
+ triggerSchedulingWithNMHeartBeat();
}
}
@@ -1396,12 +1436,7 @@ public class TestAMRMClient {
while (System.currentTimeMillis() - startTime <
rolling_interval_sec * 1000) {
amClient.allocate(0.1f);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
+ sleep(1000);
}
amClient.allocate(0.1f);
@@ -1461,11 +1496,7 @@ public class TestAMRMClient {
}
}
amClient.allocate(0.1f);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // DO NOTHING
- }
+ sleep(1000);
}
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2148b839/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 0599414..4f3e4f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -349,7 +349,8 @@ public class FairScheduler extends
* fair shares, deficits, minimum slot allocations, and amount of used and
* required resources per job.
*/
- protected void update() {
+ @VisibleForTesting
+ public void update() {
try {
writeLock.lock();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] hadoop git commit: HDFS-11477. Simplify file IO profiling
configuration. Contributed by Hanisha Koneru.
Posted by kk...@apache.org.
HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/959940b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/959940b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/959940b0
Branch: refs/heads/YARN-5972
Commit: 959940b0ab563b4e42bace44f1dc9a8babcaa889
Parents: f597f4c
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Mar 7 10:12:35 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Mar 7 10:12:35 2017 -0800
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 7 ++++++-
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +-----
.../org/apache/hadoop/hdfs/server/common/Util.java | 17 +++++++++++++++++
.../apache/hadoop/hdfs/server/datanode/DNConf.java | 7 ++++---
.../hdfs/server/datanode/FileIoProvider.java | 2 +-
.../server/datanode/ProfilingFileIoEvents.java | 11 +++++------
.../server/datanode/TestDataNodeVolumeMetrics.java | 4 ++--
.../apache/hadoop/tools/TestHdfsConfigFields.java | 2 --
8 files changed, 36 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/959940b0/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 7900692..a8bdbeb 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -332,7 +332,12 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
FsVolume
--------
-Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics.
+Per-volume metrics contain Datanode Volume IO related statistics. Per-volume
+metrics are off by default. They can be enabled by setting `dfs.datanode
+.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0.
+Setting this value to 0.0 would mean profiling is not enabled. But enabling
+per-volume metrics may have a performance impact. Each metrics record
+contains tags such as Hostname as additional information along with metrics.
| Name | Description |
|:---- |:---- |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/959940b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index be20829..82d6073 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -718,10 +718,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT = 0; //no throttling
// Datanode File IO Stats
- public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
- "dfs.datanode.enable.fileio.profiling";
- public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
- false;
public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY =
"dfs.datanode.enable.fileio.fault.injection";
public static final boolean
@@ -730,7 +726,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
"dfs.datanode.fileio.profiling.sampling.fraction";
public static final double
- DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
+ DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0;
//Keys with no defaults
public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/959940b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
index 9c67f0a..fdb09df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
@@ -388,4 +388,21 @@ public final class Util {
addrsList.removeAll(addrsToExclude);
return addrsList;
}
+
+ public static boolean isDiskStatsEnabled(double fileIOSamplingFraction) {
+ final boolean isEnabled;
+ if (fileIOSamplingFraction < 0.000001) {
+ LOG.info(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
+ + fileIOSamplingFraction + ". Disabling file IO profiling");
+ isEnabled = false;
+ } else {
+ LOG.info(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
+ + fileIOSamplingFraction + ". Enabling file IO profiling");
+ isEnabled = true;
+ }
+
+ return isEnabled;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/959940b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 3275ba8..21ffccc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
+import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.security.SaslPropertiesResolver;
import java.util.concurrent.TimeUnit;
@@ -174,9 +175,9 @@ public class DNConf {
this.peerStatsEnabled = getConf().getBoolean(
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
- this.diskStatsEnabled = getConf().getBoolean(
- DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY,
- DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
+ this.diskStatsEnabled = Util.isDiskStatsEnabled(getConf().getDouble(
+ DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
+ DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT));
this.outliersReportIntervalMs = getConf().getTimeDuration(
DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/959940b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index 9def2e1..5508e0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*;
*
* Behavior can be injected into these events by enabling the
* profiling and/or fault injection event hooks through
- * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and
+ * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and
* {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}.
* These event hooks are disabled by default.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/959940b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 9af84fa..35118b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.util.Time;
@@ -43,13 +44,11 @@ class ProfilingFileIoEvents {
public ProfilingFileIoEvents(@Nullable Configuration conf) {
if (conf != null) {
- isEnabled = conf.getBoolean(DFSConfigKeys
- .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
- .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
- double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
+ double fileIOSamplingFraction = conf.getDouble(
+ DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT);
+ isEnabled = Util.isDiskStatsEnabled(fileIOSamplingFraction);
if (fileIOSamplingFraction > 1) {
LOG.warn(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/959940b0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
index 6a8ac9c..03e1dee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
@@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics {
private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException {
Configuration conf = new HdfsConfiguration();
- conf.setBoolean(DFSConfigKeys
- .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true);
+ conf.setDouble(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
SimulatedFSDataset.setFactory(conf);
return new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATANODES)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/959940b0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index cdce342..1fdf713 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -103,8 +103,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
.add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
configurationPropsToSkipCompare
.add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
- configurationPropsToSkipCompare
- .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
.DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] hadoop git commit: HADOOP-14026. start-build-env.sh: invalid docker image name (Contributed by Gergő Pásztor via Daniel Templeton)
Posted by kk...@apache.org.
HADOOP-14026. start-build-env.sh: invalid docker image name (Contributed by Gerg\u0151 P�sztor via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8bd8ac7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8bd8ac7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8bd8ac7
Branch: refs/heads/YARN-5972
Commit: c8bd8ac7f9642b8db5949a914f209d5de3d4ef9b
Parents: c1386a9
Author: Daniel Templeton <te...@apache.org>
Authored: Sat Mar 4 11:00:03 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Sat Mar 4 11:01:44 2017 -0800
----------------------------------------------------------------------
start-build-env.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bd8ac7/start-build-env.sh
----------------------------------------------------------------------
diff --git a/start-build-env.sh b/start-build-env.sh
index 1fa084f..18e3a8c 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -31,7 +31,7 @@ else # boot2docker uid and gid
GROUP_ID=50
fi
-docker build -t "hadoop-build-${USER_NAME}" - <<UserSpecificDocker
+docker build -t "hadoop-build-${USER_ID}" - <<UserSpecificDocker
FROM hadoop-build
RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER_NAME}
@@ -47,4 +47,4 @@ docker run --rm=true -t -i \
-w "/home/${USER_NAME}/hadoop" \
-v "${HOME}/.m2:/home/${USER_NAME}/.m2" \
-u "${USER_NAME}" \
- "hadoop-build-${USER_NAME}"
+ "hadoop-build-${USER_ID}"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] hadoop git commit: MAPREDUCE-6855. Specify charset when
create String in CredentialsTestJob. Contributed by Kai Sasaki.
Posted by kk...@apache.org.
MAPREDUCE-6855. Specify charset when create String in CredentialsTestJob. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14413989
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14413989
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14413989
Branch: refs/heads/YARN-5972
Commit: 14413989cac9acc1fa6f8d330fac32f772613325
Parents: 6868235
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Mar 7 13:10:59 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Mar 7 13:10:59 2017 +0900
----------------------------------------------------------------------
.../org/apache/hadoop/mapreduce/security/CredentialsTestJob.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14413989/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
index e66fb2f..755e2df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.mapreduce.security;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import org.apache.hadoop.conf.Configuration;
@@ -64,7 +65,7 @@ public class CredentialsTestJob extends Configured implements Tool {
// fail the test
}
- String secretValueStr = new String (secretValue);
+ String secretValueStr = new String (secretValue, StandardCharsets.UTF_8);
System.out.println(secretValueStr);
if ( !("password"+i).equals(secretValueStr)){
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] hadoop git commit: YARN-6270.
WebUtils.getRMWebAppURLWithScheme() needs to honor RM HA setting. Contributed
by Xuan Gong
Posted by kk...@apache.org.
YARN-6270. WebUtils.getRMWebAppURLWithScheme() needs to honor RM HA setting. Contributed by Xuan Gong
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/279d187f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/279d187f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/279d187f
Branch: refs/heads/YARN-5972
Commit: 279d187f723d01658ef8698a29263652e2a05618
Parents: 8db7a8c
Author: Jian He <ji...@apache.org>
Authored: Fri Mar 3 15:23:33 2017 -0800
Committer: Jian He <ji...@apache.org>
Committed: Fri Mar 3 15:24:04 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/webapp/util/WebAppUtils.java | 43 ++++++++++++++++----
.../hadoop/yarn/conf/TestYarnConfiguration.java | 16 ++++++++
2 files changed, 50 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/279d187f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index e412173..64a4b2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -89,21 +89,46 @@ public class WebAppUtils {
hostName + ":" + port);
}
}
-
- public static String getRMWebAppURLWithScheme(Configuration conf) {
- return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme(conf);
- }
-
- public static String getRMWebAppURLWithoutScheme(Configuration conf) {
- if (YarnConfiguration.useHttps(conf)) {
- return conf.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
+
+ public static String getRMWebAppURLWithoutScheme(Configuration conf,
+ boolean isHAEnabled) {
+ YarnConfiguration yarnConfig = new YarnConfiguration(conf);
+ // set RM_ID if we have not configure it.
+ if (isHAEnabled) {
+ String rmId = yarnConfig.get(YarnConfiguration.RM_HA_ID);
+ if (rmId == null || rmId.isEmpty()) {
+ List<String> rmIds = new ArrayList<>(HAUtil.getRMHAIds(conf));
+ if (rmIds != null && !rmIds.isEmpty()) {
+ yarnConfig.set(YarnConfiguration.RM_HA_ID, rmIds.get(0));
+ }
+ }
+ }
+ if (YarnConfiguration.useHttps(yarnConfig)) {
+ if (isHAEnabled) {
+ return HAUtil.getConfValueForRMInstance(
+ YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, yarnConfig);
+ }
+ return yarnConfig.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS);
}else {
- return conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ if (isHAEnabled) {
+ return HAUtil.getConfValueForRMInstance(
+ YarnConfiguration.RM_WEBAPP_ADDRESS, yarnConfig);
+ }
+ return yarnConfig.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS);
}
}
+ public static String getRMWebAppURLWithScheme(Configuration conf) {
+ return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme(
+ conf, HAUtil.isHAEnabled(conf));
+ }
+
+ public static String getRMWebAppURLWithoutScheme(Configuration conf) {
+ return getRMWebAppURLWithoutScheme(conf, false);
+ }
+
public static List<String> getProxyHostsAndPortsForAmFilter(
Configuration conf) {
List<String> addrs = new ArrayList<String>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/279d187f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
index 2e76865..7389423 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
@@ -41,6 +41,22 @@ public class TestYarnConfiguration {
// specifically add slashes and Jetty doesn't handle double slashes.
Assert.assertNotSame("RM Web Url is not correct", "http://0.0.0.0:8088",
rmWebUrl);
+
+ // test it in HA scenario
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1, rm2");
+ conf.set("yarn.resourcemanager.webapp.address.rm1", "10.10.10.10:18088");
+ conf.set("yarn.resourcemanager.webapp.address.rm2", "20.20.20.20:28088");
+ String rmWebUrlinHA = WebAppUtils.getRMWebAppURLWithScheme(conf);
+ Assert.assertEquals("http://10.10.10.10:18088", rmWebUrlinHA);
+
+ YarnConfiguration conf2 = new YarnConfiguration();
+ conf2.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf2.set(YarnConfiguration.RM_HA_IDS, "rm1, rm2");
+ conf2.set("yarn.resourcemanager.hostname.rm1", "30.30.30.30");
+ conf2.set("yarn.resourcemanager.hostname.rm2", "40.40.40.40");
+ String rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2);
+ Assert.assertEquals("http://30.30.30.30:8088", rmWebUrlinHA2);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] hadoop git commit: YARN-6263.
NMTokenSecretManagerInRM.createAndGetNMToken is not thread safe. Contributed
by Haibo Chen
Posted by kk...@apache.org.
YARN-6263. NMTokenSecretManagerInRM.createAndGetNMToken is not thread safe. Contributed by Haibo Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e58fc760
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e58fc760
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e58fc760
Branch: refs/heads/YARN-5972
Commit: e58fc7603053e3ac1bc2464f9622995017db5245
Parents: 0523763
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Fri Mar 3 08:23:08 2017 -0600
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Fri Mar 3 08:23:08 2017 -0600
----------------------------------------------------------------------
.../resourcemanager/security/NMTokenSecretManagerInRM.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e58fc760/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java
index 1471f78..956391e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java
@@ -192,7 +192,7 @@ public class NMTokenSecretManagerInRM extends BaseNMTokenSecretManager {
public NMToken createAndGetNMToken(String applicationSubmitter,
ApplicationAttemptId appAttemptId, Container container) {
try {
- this.readLock.lock();
+ this.writeLock.lock();
HashSet<NodeId> nodeSet = this.appAttemptToNodeKeyMap.get(appAttemptId);
NMToken nmToken = null;
if (nodeSet != null) {
@@ -208,7 +208,7 @@ public class NMTokenSecretManagerInRM extends BaseNMTokenSecretManager {
}
return nmToken;
} finally {
- this.readLock.unlock();
+ this.writeLock.unlock();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] hadoop git commit: HDFS-11477. Simplify file IO profiling
configuration. Contributed by Hanisha Koneru.
Posted by kk...@apache.org.
HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e61491d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e61491d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e61491d4
Branch: refs/heads/YARN-5972
Commit: e61491d4769536ce8b77b416005ce531278ce729
Parents: a97833e
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Mar 2 15:38:25 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Mar 2 15:38:25 2017 -0800
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 7 ++++-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +----
.../hdfs/server/datanode/FileIoProvider.java | 2 +-
.../server/datanode/ProfilingFileIoEvents.java | 27 +++++++++++++++-----
.../datanode/TestDataNodeVolumeMetrics.java | 4 +--
.../hadoop/tools/TestHdfsConfigFields.java | 2 --
6 files changed, 31 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 7900692..a8bdbeb 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -332,7 +332,12 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
FsVolume
--------
-Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics.
+Per-volume metrics contain Datanode Volume IO related statistics. Per-volume
+metrics are off by default. They can be enabled by setting `dfs.datanode
+.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0.
+Setting this value to 0.0 would mean profiling is not enabled. But enabling
+per-volume metrics may have a performance impact. Each metrics record
+contains tags such as Hostname as additional information along with metrics.
| Name | Description |
|:---- |:---- |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index be20829..82d6073 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -718,10 +718,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT = 0; //no throttling
// Datanode File IO Stats
- public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
- "dfs.datanode.enable.fileio.profiling";
- public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
- false;
public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY =
"dfs.datanode.enable.fileio.fault.injection";
public static final boolean
@@ -730,7 +726,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
"dfs.datanode.fileio.profiling.sampling.fraction";
public static final double
- DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
+ DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0;
//Keys with no defaults
public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index 9def2e1..5508e0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*;
*
* Behavior can be injected into these events by enabling the
* profiling and/or fault injection event hooks through
- * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and
+ * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and
* {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}.
* These event hooks are disabled by default.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 9af84fa..7eab969 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.datanode;
+import static java.lang.Math.abs;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -43,18 +44,32 @@ class ProfilingFileIoEvents {
public ProfilingFileIoEvents(@Nullable Configuration conf) {
if (conf != null) {
- isEnabled = conf.getBoolean(DFSConfigKeys
- .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
- .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
- if (fileIOSamplingFraction > 1) {
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT);
+ if (abs(fileIOSamplingFraction) < 0.000001) {
+ LOG.info(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
+ + fileIOSamplingFraction + ". Disabling file IO profiling");
+ isEnabled = false;
+ } else if (fileIOSamplingFraction < 0.000001) {
LOG.warn(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
- " value cannot be more than 1. Setting value to 1");
+ " value cannot be less than 0. Disabling file IO profiling.");
+ isEnabled = false;
+ } else if (fileIOSamplingFraction > 1) {
+ LOG.warn(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
+ " value cannot be more than 1. Setting value to 1 and enabling " +
+ "file IO profiling");
+ isEnabled = true;
fileIOSamplingFraction = 1;
+ } else {
+ LOG.info(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
+ + fileIOSamplingFraction + ". Enabling file IO profiling");
+ isEnabled = true;
}
sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
index 6a8ac9c..03e1dee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
@@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics {
private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException {
Configuration conf = new HdfsConfiguration();
- conf.setBoolean(DFSConfigKeys
- .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true);
+ conf.setDouble(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
SimulatedFSDataset.setFactory(conf);
return new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATANODES)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e61491d4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index cdce342..1fdf713 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -103,8 +103,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
.add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
configurationPropsToSkipCompare
.add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
- configurationPropsToSkipCompare
- .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
.DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] hadoop git commit: HDFS-11441. Add escaping to error message
in KMS web UI. Contributed by Aaron T. Myers.
Posted by kk...@apache.org.
HDFS-11441. Add escaping to error message in KMS web UI. Contributed by Aaron T. Myers.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec839b94
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec839b94
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec839b94
Branch: refs/heads/YARN-5972
Commit: ec839b94c0eb3f09e74f8a3b0bc9a08b3f5418b2
Parents: 209ecd1
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Mar 6 10:47:15 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Mar 6 10:47:15 2017 -0800
----------------------------------------------------------------------
.../hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec839b94/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
index 45e48e9..3e98a25 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.crypto.key.kms.server;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.kms.KMSDelegationToken;
+import org.apache.hadoop.http.HtmlQuoting;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter;
@@ -105,7 +106,7 @@ public class KMSAuthenticationFilter
public void sendError(int sc, String msg) throws IOException {
statusCode = sc;
this.msg = msg;
- super.sendError(sc, msg);
+ super.sendError(sc, HtmlQuoting.quoteHtmlChars(msg));
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] hadoop git commit: HDFS-11478. Update EC commands in
HDFSCommands.md. Contributed by Yiqun Lin.
Posted by kk...@apache.org.
HDFS-11478. Update EC commands in HDFSCommands.md. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/555d0c39
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/555d0c39
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/555d0c39
Branch: refs/heads/YARN-5972
Commit: 555d0c39950078e80a373f188c3b1529995d0af7
Parents: 6f6dfe0
Author: Yiqun Lin <yq...@apache.org>
Authored: Thu Mar 2 12:10:24 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Thu Mar 2 12:10:24 2017 +0800
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/555d0c39/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index b65cc78..d031dad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -436,8 +436,9 @@ Runs the diskbalancer CLI. See [HDFS Diskbalancer](./HDFSDiskbalancer.html) for
Usage:
hdfs ec [generic options]
- [-setPolicy [-p <policyName>] <path>]
- [-getPolicy <path>]
+ [-setPolicy -policy <policyName> -path <path>]
+ [-getPolicy -path <path>]
+ [-unsetPolicy -path <path>]
[-listPolicies]
[-usage [cmd ...]]
[-help [cmd ...]]
@@ -446,6 +447,7 @@ Usage:
|:---- |:---- |
|-setPolicy| Set a specified ErasureCoding policy to a directory|
|-getPolicy| Get ErasureCoding policy information about a specified path|
+|-unsetPolicy| Unset an ErasureCoding policy set by a previous call to "setPolicy" on a directory |
|-listPolicies| Lists all supported ErasureCoding policies|
Runs the ErasureCoding CLI. See [HDFS ErasureCoding](./HDFSErasureCoding.html#Administrative_commands) for more information on this command.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] hadoop git commit: YARN-6248. user is not removed from UsersManager’s when app is killed with pending container requests. Contributed by Eric Payne.
Posted by kk...@apache.org.
YARN-6248. user is not removed from UsersManager\u2019s when app is killed with pending container requests. Contributed by Eric Payne.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1386a90
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1386a90
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1386a90
Branch: refs/heads/YARN-5972
Commit: c1386a90d7ff65b57be00a7245de82c31abb4665
Parents: c61bc2d
Author: Sunil G <su...@apache.org>
Authored: Sat Mar 4 13:46:35 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Sat Mar 4 13:46:35 2017 +0530
----------------------------------------------------------------------
.../server/resourcemanager/scheduler/capacity/UsersManager.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1386a90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index 05503c6..c2134eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -872,7 +872,9 @@ public class UsersManager implements AbstractUsersManager {
this.writeLock.lock();
// For UT case: We might need to add the user to users list.
- User user = getUserAndAddIfAbsent(userName);
+ User user = getUser(userName);
+ if (user == null) return;
+
ResourceUsage resourceUsage = user.getResourceUsage();
// If User is moved to non-active list, moved resource usage from
// non-active to active list.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] hadoop git commit: HDFS-10838. Last full block report
received time for each DN should be easily discoverable. Contributed by
Surendra Singh Lilhore.
Posted by kk...@apache.org.
HDFS-10838. Last full block report received time for each DN should be easily discoverable. Contributed by Surendra Singh Lilhore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5adc5c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5adc5c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5adc5c3
Branch: refs/heads/YARN-5972
Commit: b5adc5c3011f111f86d232cb33ec522547f68a95
Parents: 5e74196
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Mar 6 16:39:53 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Mar 6 16:39:53 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/DatanodeInfo.java | 52 +++++++++++++++++++-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 8 ++-
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 2 +
.../src/main/proto/hdfs.proto | 2 +
.../server/blockmanagement/BlockManager.java | 3 ++
.../hdfs/server/namenode/FSNamesystem.java | 8 ++-
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 3 ++
.../src/main/webapps/hdfs/dfshealth.html | 2 +
.../src/main/webapps/hdfs/dfshealth.js | 1 +
.../server/namenode/TestNameNodeMXBean.java | 1 +
10 files changed, 78 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5adc5c3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index acbcffa..e1698c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -85,6 +85,8 @@ public class DatanodeInfo extends DatanodeID implements Node {
protected AdminStates adminState;
private long maintenanceExpireTimeInMS;
+ private long lastBlockReportTime;
+ private long lastBlockReportMonotonic;
protected DatanodeInfo(DatanodeInfo from) {
super(from);
@@ -101,6 +103,8 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.location = from.getNetworkLocation();
this.adminState = from.getAdminState();
this.upgradeDomain = from.getUpgradeDomain();
+ this.lastBlockReportTime = from.getLastBlockReportTime();
+ this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic();
}
protected DatanodeInfo(DatanodeID nodeID) {
@@ -116,6 +120,8 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.lastUpdateMonotonic = 0L;
this.xceiverCount = 0;
this.adminState = null;
+ this.lastBlockReportTime = 0L;
+ this.lastBlockReportMonotonic = 0L;
}
protected DatanodeInfo(DatanodeID nodeID, String location) {
@@ -131,7 +137,8 @@ public class DatanodeInfo extends DatanodeID implements Node {
final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
final long lastUpdate, final long lastUpdateMonotonic,
final int xceiverCount, final String networkLocation,
- final AdminStates adminState, final String upgradeDomain) {
+ final AdminStates adminState, final String upgradeDomain,
+ final long lastBlockReportTime, final long lastBlockReportMonotonic) {
super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
ipcPort);
this.capacity = capacity;
@@ -147,6 +154,8 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.location = networkLocation;
this.adminState = adminState;
this.upgradeDomain = upgradeDomain;
+ this.lastBlockReportTime = lastBlockReportTime;
+ this.lastBlockReportMonotonic = lastBlockReportMonotonic;
}
/** Network location name. */
@@ -391,6 +400,11 @@ public class DatanodeInfo extends DatanodeID implements Node {
.append(percent2String(cacheRemainingPercent)).append("\n");
buffer.append("Xceivers: ").append(getXceiverCount()).append("\n");
buffer.append("Last contact: ").append(new Date(lastUpdate)).append("\n");
+ buffer
+ .append("Last Block Report: ")
+ .append(
+ lastBlockReportTime != 0 ? new Date(lastBlockReportTime) : "Never")
+ .append("\n");
return buffer.toString();
}
@@ -503,6 +517,26 @@ public class DatanodeInfo extends DatanodeID implements Node {
return this.maintenanceExpireTimeInMS;
}
+ /** Sets the last block report time. */
+ public void setLastBlockReportTime(long lastBlockReportTime) {
+ this.lastBlockReportTime = lastBlockReportTime;
+ }
+
+ /** Sets the last block report monotonic time. */
+ public void setLastBlockReportMonotonic(long lastBlockReportMonotonic) {
+ this.lastBlockReportMonotonic = lastBlockReportMonotonic;
+ }
+
+ /** Last block report time. */
+ public long getLastBlockReportTime() {
+ return lastBlockReportTime;
+ }
+
+ /** Last block report monotonic time. */
+ public long getLastBlockReportMonotonic() {
+ return lastBlockReportMonotonic;
+ }
+
/**
* Take the node out of maintenance mode.
*/
@@ -643,6 +677,8 @@ public class DatanodeInfo extends DatanodeID implements Node {
private int infoSecurePort;
private int ipcPort;
private long nonDfsUsed = 0L;
+ private long lastBlockReportTime = 0L;
+ private long lastBlockReportMonotonic = 0L;
public DatanodeInfoBuilder setFrom(DatanodeInfo from) {
this.capacity = from.getCapacity();
@@ -658,6 +694,8 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.location = from.getNetworkLocation();
this.adminState = from.getAdminState();
this.upgradeDomain = from.getUpgradeDomain();
+ this.lastBlockReportTime = from.getLastBlockReportTime();
+ this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic();
setNodeID(from);
return this;
}
@@ -775,12 +813,22 @@ public class DatanodeInfo extends DatanodeID implements Node {
return this;
}
+ public DatanodeInfoBuilder setLastBlockReportTime(long time) {
+ this.lastBlockReportTime = time;
+ return this;
+ }
+
+ public DatanodeInfoBuilder setLastBlockReportMonotonic(long time) {
+ this.lastBlockReportMonotonic = time;
+ return this;
+ }
+
public DatanodeInfo build() {
return new DatanodeInfo(ipAddr, hostName, datanodeUuid, xferPort,
infoPort, infoSecurePort, ipcPort, capacity, dfsUsed, nonDfsUsed,
remaining, blockPoolUsed, cacheCapacity, cacheUsed, lastUpdate,
lastUpdateMonotonic, xceiverCount, location, adminState,
- upgradeDomain);
+ upgradeDomain, lastBlockReportTime, lastBlockReportMonotonic);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5adc5c3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index ad80bc2..b4fa926 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -303,6 +303,8 @@ public class PBHelperClient {
.setLastUpdateMonotonic(info.getLastUpdateMonotonic())
.setXceiverCount(info.getXceiverCount())
.setAdminState(convert(info.getAdminState()))
+ .setLastBlockReportTime(info.getLastBlockReportTime())
+ .setLastBlockReportMonotonic(info.getLastBlockReportMonotonic())
.build();
return builder.build();
}
@@ -650,7 +652,11 @@ public class PBHelperClient {
.setLastUpdateMonotonic(di.getLastUpdateMonotonic())
.setXceiverCount(di.getXceiverCount())
.setAdminState(convert(di.getAdminState())).setUpgradeDomain(
- di.hasUpgradeDomain() ? di.getUpgradeDomain() : null);
+ di.hasUpgradeDomain() ? di.getUpgradeDomain() : null)
+ .setLastBlockReportTime(di.hasLastBlockReportTime() ?
+ di.getLastBlockReportTime() : 0)
+ .setLastBlockReportMonotonic(di.hasLastBlockReportMonotonic() ?
+ di.getLastBlockReportMonotonic() : 0);
if (di.hasNonDfsUsed()) {
dinfo.setNonDfsUsed(di.getNonDfsUsed());
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5adc5c3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index 4204c54..cfcb4c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -295,6 +295,8 @@ class JsonUtilClient {
DatanodeInfo.AdminStates
.valueOf(getString(m, "adminState", "NORMAL")))
.setUpgradeDomain(getString(m, "upgradeDomain", ""))
+ .setLastBlockReportTime(getLong(m, "lastBlockReportTime", 0L))
+ .setLastBlockReportMonotonic(getLong(m, "lastBlockReportMonotonic", 0L))
.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5adc5c3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 8a039d4..7824fd1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -102,6 +102,8 @@ message DatanodeInfoProto {
optional uint64 cacheUsed = 12 [default = 0];
optional uint64 lastUpdateMonotonic = 13 [default = 0];
optional string upgradeDomain = 14;
+ optional uint64 lastBlockReportTime = 15 [default = 0];
+ optional uint64 lastBlockReportMonotonic = 16 [default = 0];
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5adc5c3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5ca0fa7..9ec28f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import static org.apache.hadoop.hdfs.protocol.BlockType.CONTIGUOUS;
import static org.apache.hadoop.hdfs.protocol.BlockType.STRIPED;
import static org.apache.hadoop.util.ExitUtil.terminate;
+import static org.apache.hadoop.util.Time.now;
import java.io.IOException;
import java.io.PrintWriter;
@@ -2377,6 +2378,8 @@ public class BlockManager implements BlockStatsMXBean {
long leaseId = this.getBlockReportLeaseManager().removeLease(node);
BlockManagerFaultInjector.getInstance().
removeBlockReportLease(node, leaseId);
+ node.setLastBlockReportTime(now());
+ node.setLastBlockReportMonotonic(Time.monotonicNow());
}
LOG.debug("Processing RPC with index {} out of total {} RPCs in "
+ "processReport 0x{}", context.getCurRpc(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5adc5c3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 81c5759..03277af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5472,7 +5472,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
.put("blockScheduled", node.getBlocksScheduled())
.put("blockPoolUsed", node.getBlockPoolUsed())
.put("blockPoolUsedPercent", node.getBlockPoolUsedPercent())
- .put("volfails", node.getVolumeFailures());
+ .put("volfails", node.getVolumeFailures())
+ // Block report time in minutes
+ .put("lastBlockReport", getLastBlockReport(node));
VolumeFailureSummary volumeFailureSummary = node.getVolumeFailureSummary();
if (volumeFailureSummary != null) {
innerinfo
@@ -5571,6 +5573,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return (monotonicNow() - alivenode.getLastUpdateMonotonic())/1000;
}
+ private Object getLastBlockReport(DatanodeDescriptor node) {
+ return (monotonicNow() - node.getLastBlockReportMonotonic()) / 60000;
+ }
+
private long getDfsUsed(DatanodeDescriptor alivenode) {
return alivenode.getDfsUsed();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5adc5c3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 0d89113..827e1a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -181,6 +181,9 @@ public class JsonUtil {
if (datanodeinfo.getUpgradeDomain() != null) {
m.put("upgradeDomain", datanodeinfo.getUpgradeDomain());
}
+ m.put("lastBlockReportTime", datanodeinfo.getLastBlockReportTime());
+ m.put("lastBlockReportMonotonic",
+ datanodeinfo.getLastBlockReportMonotonic());
return m;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5adc5c3/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index b33b9a2..c01f36c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -309,6 +309,7 @@
<th>Node</th>
<th>Http Address</th>
<th>Last contact</th>
+ <th>Last Block Report</th>
<th style="width:180px; text-align:center">Capacity</th>
<th>Blocks</th>
<th>Block pool used</th>
@@ -320,6 +321,7 @@
<td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{name} ({xferaddr})</td>
<td ng-value="{state}-{name}"><a href='//{dnWebAddress}'>{dnWebAddress}</a></td>
<td ng-value="{lastContact}">{lastContact}s</td>
+ <td ng-value="{lastBlockReport}">{lastBlockReport}m</td>
<td ng-value="{usedPercentage}">
<div>
<div style="display:inline-block; float: left; padding-right: 10px;">{capacity|fmt_bytes}</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5adc5c3/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
index 88096d3..59958a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
@@ -333,6 +333,7 @@
{ 'orderDataType': 'ng-value', 'searchable': true },
{ 'orderDataType': 'ng-value', 'type': 'numeric' },
{ 'orderDataType': 'ng-value', 'type': 'numeric' },
+ { 'orderDataType': 'ng-value', 'type': 'numeric' },
{ 'orderData': 3, 'type': 'numeric' },
{ 'orderDataType': 'ng-value', 'type': 'numeric'},
{ 'orderData': 5 }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5adc5c3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 037fd40..d4d4763 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -174,6 +174,7 @@ public class TestNameNodeMXBean {
assertTrue(((Long)liveNode.get("capacity")) > 0);
assertTrue(liveNode.containsKey("numBlocks"));
assertTrue(((Long)liveNode.get("numBlocks")) == 0);
+ assertTrue(liveNode.containsKey("lastBlockReport"));
// a. By default the upgrade domain isn't defined on any DN.
// b. If the upgrade domain is set on a DN, JMX should have the same
// value.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] hadoop git commit: HDFS-11494. Log message when DN is not
selected for block replication. Contributed by Yiqun Lin
Posted by kk...@apache.org.
HDFS-11494. Log message when DN is not selected for block replication. Contributed by Yiqun Lin
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f336bab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f336bab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f336bab
Branch: refs/heads/YARN-5972
Commit: 0f336bab9c070e0cdc935df4305084a036e2def7
Parents: 6b7cd62
Author: Mingliang Liu <li...@apache.org>
Authored: Fri Mar 3 18:47:13 2017 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Fri Mar 3 18:47:13 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f336bab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 15af6ab..7b3e4e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -687,6 +687,10 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
}
if (requiredSize > remaining - scheduledSize) {
+ LOG.debug(
+ "The node {} does not have enough {} space (required={},"
+ + " scheduled={}, remaining={}).",
+ this, t, requiredSize, scheduledSize, remaining);
return null;
}
return storage;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] hadoop git commit: HDFS-11432. Federation : Support fully
qualified path for Quota/Snapshot/cacheadmin/cryptoadmin commands.
Contributed by Brahma Reddy Battula.
Posted by kk...@apache.org.
HDFS-11432. Federation : Support fully qualified path for Quota/Snapshot/cacheadmin/cryptoadmin commands. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcd03df9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcd03df9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcd03df9
Branch: refs/heads/YARN-5972
Commit: dcd03df9f9e0080d7e179060ffc8148336c31b3e
Parents: 989bd56
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Wed Mar 1 10:45:56 2017 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Wed Mar 1 10:45:56 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/fs/shell/Command.java | 14 ++++-
.../apache/hadoop/hdfs/tools/CacheAdmin.java | 8 +--
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 18 +++----
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 57 +++++++++++---------
.../hdfs/tools/snapshot/SnapshotDiff.java | 19 ++++++-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 26 +++++++++
.../hadoop/hdfs/TestSnapshotCommands.java | 34 ++++++++++++
7 files changed, 136 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
index c573aa0..4c5cbad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
@@ -101,7 +101,17 @@ abstract public class Command extends Configured {
* @throws IOException if any error occurs
*/
abstract protected void run(Path path) throws IOException;
-
+
+ /**
+ * Execute the command on the input path data. Commands can override to make
+ * use of the resolved filesystem.
+ * @param pathData The input path with resolved filesystem
+ * @throws IOException
+ */
+ protected void run(PathData pathData) throws IOException {
+ run(pathData.path);
+ }
+
/**
* For each source path, execute the command
*
@@ -113,7 +123,7 @@ abstract public class Command extends Configured {
try {
PathData[] srcs = PathData.expandAsGlob(src, getConf());
for (PathData s : srcs) {
- run(s.path);
+ run(s);
}
} catch (IOException e) {
exitCode = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
index 522f701..d8cbfc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
@@ -189,8 +189,9 @@ public class CacheAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
-
- DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+
+ DistributedFileSystem dfs =
+ AdminHelper.getDFS(new Path(path).toUri(), conf);
CacheDirectiveInfo directive = builder.build();
EnumSet<CacheFlag> flags = EnumSet.noneOf(CacheFlag.class);
if (force) {
@@ -409,7 +410,8 @@ public class CacheAdmin extends Configured implements Tool {
}
int exitCode = 0;
try {
- DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+ DistributedFileSystem dfs =
+ AdminHelper.getDFS(new Path(path).toUri(), conf);
RemoteIterator<CacheDirectiveEntry> iter =
dfs.listCacheDirectives(
new CacheDirectiveInfo.Builder().
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
index 225f11a..4c7335f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
@@ -139,12 +139,12 @@ public class CryptoAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
-
- HdfsAdmin admin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ Path p = new Path(path);
+ HdfsAdmin admin = new HdfsAdmin(p.toUri(), conf);
EnumSet<CreateEncryptionZoneFlag> flags =
EnumSet.of(CreateEncryptionZoneFlag.PROVISION_TRASH);
try {
- admin.createEncryptionZone(new Path(path), keyName, flags);
+ admin.createEncryptionZone(p, keyName, flags);
System.out.println("Added encryption zone " + path);
} catch (IOException e) {
System.err.println(prettifyException(e));
@@ -226,12 +226,12 @@ public class CryptoAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
-
+ Path p = new Path(path);
final HdfsAdmin admin =
- new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ new HdfsAdmin(p.toUri(), conf);
try {
final FileEncryptionInfo fei =
- admin.getFileEncryptionInfo(new Path(path));
+ admin.getFileEncryptionInfo(p);
if (fei == null) {
System.out.println("No FileEncryptionInfo found for path " + path);
return 2;
@@ -273,10 +273,10 @@ public class CryptoAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
-
- HdfsAdmin admin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ Path p = new Path(path);
+ HdfsAdmin admin = new HdfsAdmin(p.toUri(), conf);
try {
- admin.provisionEncryptionZoneTrash(new Path(path));
+ admin.provisionEncryptionZoneTrash(p);
System.out.println("Created a trash directory for " + path);
} catch (IOException ioe) {
System.err.println(prettifyException(ioe));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index 4dd9d7e..c1f79e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFormat;
+import org.apache.hadoop.fs.shell.PathData;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HAUtilClient;
@@ -109,15 +110,21 @@ public class DFSAdmin extends FsShell {
* An abstract class for the execution of a file system command
*/
abstract private static class DFSAdminCommand extends Command {
- final DistributedFileSystem dfs;
+ protected DistributedFileSystem dfs;
/** Constructor */
- public DFSAdminCommand(FileSystem fs) {
- super(fs.getConf());
+ public DFSAdminCommand(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public void run(PathData pathData) throws IOException {
+ FileSystem fs = pathData.fs;
if (!(fs instanceof DistributedFileSystem)) {
- throw new IllegalArgumentException("FileSystem " + fs.getUri() +
- " is not an HDFS file system");
+ throw new IllegalArgumentException("FileSystem " + fs.getUri()
+ + " is not an HDFS file system");
}
- this.dfs = (DistributedFileSystem)fs;
+ this.dfs = (DistributedFileSystem) fs;
+ run(pathData.path);
}
}
@@ -133,8 +140,8 @@ public class DFSAdmin extends FsShell {
"\t\tIt does not fault if the directory has no quota.";
/** Constructor */
- ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
- super(fs);
+ ClearQuotaCommand(String[] args, int pos, Configuration conf) {
+ super(conf);
CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
List<String> parameters = c.parse(args, pos);
this.args = parameters.toArray(new String[parameters.size()]);
@@ -179,8 +186,8 @@ public class DFSAdmin extends FsShell {
private final long quota; // the quota to be set
/** Constructor */
- SetQuotaCommand(String[] args, int pos, FileSystem fs) {
- super(fs);
+ SetQuotaCommand(String[] args, int pos, Configuration conf) {
+ super(conf);
CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
List<String> parameters = c.parse(args, pos);
this.quota = Long.parseLong(parameters.remove(0));
@@ -230,8 +237,8 @@ public class DFSAdmin extends FsShell {
private StorageType type;
/** Constructor */
- ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
- super(fs);
+ ClearSpaceQuotaCommand(String[] args, int pos, Configuration conf) {
+ super(conf);
CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
c.addOptionWithValue("storageType");
List<String> parameters = c.parse(args, pos);
@@ -294,8 +301,8 @@ public class DFSAdmin extends FsShell {
private StorageType type;
/** Constructor */
- SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
- super(fs);
+ SetSpaceQuotaCommand(String[] args, int pos, Configuration conf) {
+ super(conf);
CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
List<String> parameters = c.parse(args, pos);
String str = parameters.remove(0).trim();
@@ -705,10 +712,11 @@ public class DFSAdmin extends FsShell {
* @param argv List of of command line parameters.
* @exception IOException
*/
- public void allowSnapshot(String[] argv) throws IOException {
- DistributedFileSystem dfs = getDFS();
+ public void allowSnapshot(String[] argv) throws IOException {
+ Path p = new Path(argv[1]);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), getConf());
try {
- dfs.allowSnapshot(new Path(argv[1]));
+ dfs.allowSnapshot(p);
} catch (SnapshotException e) {
throw new RemoteException(e.getClass().getName(), e.getMessage());
}
@@ -721,10 +729,11 @@ public class DFSAdmin extends FsShell {
* @param argv List of of command line parameters.
* @exception IOException
*/
- public void disallowSnapshot(String[] argv) throws IOException {
- DistributedFileSystem dfs = getDFS();
+ public void disallowSnapshot(String[] argv) throws IOException {
+ Path p = new Path(argv[1]);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), getConf());
try {
- dfs.disallowSnapshot(new Path(argv[1]));
+ dfs.disallowSnapshot(p);
} catch (SnapshotException e) {
throw new RemoteException(e.getClass().getName(), e.getMessage());
}
@@ -2042,13 +2051,13 @@ public class DFSAdmin extends FsShell {
} else if ("-metasave".equals(cmd)) {
exitCode = metaSave(argv, i);
} else if (ClearQuotaCommand.matches(cmd)) {
- exitCode = new ClearQuotaCommand(argv, i, getDFS()).runAll();
+ exitCode = new ClearQuotaCommand(argv, i, getConf()).runAll();
} else if (SetQuotaCommand.matches(cmd)) {
- exitCode = new SetQuotaCommand(argv, i, getDFS()).runAll();
+ exitCode = new SetQuotaCommand(argv, i, getConf()).runAll();
} else if (ClearSpaceQuotaCommand.matches(cmd)) {
- exitCode = new ClearSpaceQuotaCommand(argv, i, getDFS()).runAll();
+ exitCode = new ClearSpaceQuotaCommand(argv, i, getConf()).runAll();
} else if (SetSpaceQuotaCommand.matches(cmd)) {
- exitCode = new SetSpaceQuotaCommand(argv, i, getDFS()).runAll();
+ exitCode = new SetSpaceQuotaCommand(argv, i, getConf()).runAll();
} else if ("-refreshServiceAcl".equals(cmd)) {
exitCode = refreshServiceAcl();
} else if ("-refreshUserToGroupsMappings".equals(cmd)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
index b87d69f..3838ca1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
@@ -20,10 +20,12 @@ package org.apache.hadoop.hdfs.tools.snapshot;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.util.Tool;
@@ -41,6 +43,19 @@ import org.apache.hadoop.util.ToolRunner;
*/
@InterfaceAudience.Private
public class SnapshotDiff extends Configured implements Tool {
+ /**
+ * Construct a SnapshotDiff object.
+ */
+ public SnapshotDiff() {
+ this(new HdfsConfiguration());
+ }
+
+ /**
+ * Construct a SnapshotDiff object.
+ */
+ public SnapshotDiff(Configuration conf) {
+ super(conf);
+ }
private static String getSnapshotName(String name) {
if (Path.CUR_DIR.equals(name)) { // current directory
return "";
@@ -72,8 +87,8 @@ public class SnapshotDiff extends Configured implements Tool {
System.err.println("Usage: \n" + description);
return 1;
}
-
- FileSystem fs = FileSystem.get(getConf());
+
+ FileSystem fs = FileSystem.get(new Path(argv[0]).toUri(), getConf());
if (! (fs instanceof DistributedFileSystem)) {
System.err.println(
"SnapshotDiff can only be used in DistributedFileSystem");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index d58dd9e..4bfb62c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -1246,6 +1246,32 @@ public class TestQuota {
-1);
}
+ /**
+ * Test to all the commands by passing the fully qualified path.
+ */
+ @Test(timeout = 30000)
+ public void testQuotaCommandsWithURI() throws Exception {
+ DFSAdmin dfsAdmin = new DFSAdmin(conf);
+ final Path dir = new Path("/" + this.getClass().getSimpleName(),
+ GenericTestUtils.getMethodName());
+ assertTrue(dfs.mkdirs(dir));
+
+ /* set space quota */
+ testSetAndClearSpaceQuotaRegularInternal(
+ new String[] { "-setSpaceQuota", "1024",
+ dfs.getUri() + "/" + dir.toString() }, dir, 0, 1024);
+
+ /* clear space quota */
+ testSetAndClearSpaceQuotaRegularInternal(
+ new String[] { "-clrSpaceQuota", dfs.getUri() + "/" + dir.toString() },
+ dir, 0, -1);
+ runCommand(dfsAdmin, false, "-setQuota", "1000",
+ dfs.getUri() + "/" + dir.toString());
+
+ runCommand(dfsAdmin, false, "-clrQuota",
+ dfs.getUri() + "/" + dir.toString());
+ }
+
private void testSetAndClearSpaceQuotaRegularInternal(
final String[] args,
final Path dir,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd03df9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
index eec4e99..1d5e071 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -60,6 +62,7 @@ public class TestSnapshotCommands {
@Before
public void setUp() throws IOException {
fs.mkdirs(new Path("/sub1"));
+ fs.mkdirs(new Path("/Fully/QPath"));
fs.allowSnapshot(new Path("/sub1"));
fs.mkdirs(new Path("/sub1/sub1sub1"));
fs.mkdirs(new Path("/sub1/sub1sub2"));
@@ -161,4 +164,35 @@ public class TestSnapshotCommands {
// now it can be deleted
DFSTestUtil.FsShellRun("-rmr /sub1", conf);
}
+
+ @Test (timeout=60000)
+ public void testSnapshotCommandsWithURI()throws Exception {
+ Configuration config = new HdfsConfiguration();
+ //fs.defaultFS should not be used, when path is fully qualified.
+ config.set("fs.defaultFS", "hdfs://127.0.0.1:1024");
+ String path = fs.getUri() + "/Fully/QPath";
+ DFSTestUtil.DFSAdminRun("-allowSnapshot " + path, 0,
+ "Allowing snaphot on " + path + " succeeded", config);
+ DFSTestUtil.FsShellRun("-createSnapshot " + path + " sn1", config);
+ // create file1
+ DFSTestUtil
+ .createFile(fs, new Path(fs.getUri() + "/Fully/QPath/File1"), 1024,
+ (short) 1, 100);
+ // create file2
+ DFSTestUtil
+ .createFile(fs, new Path(fs.getUri() + "/Fully/QPath/File2"), 1024,
+ (short) 1, 100);
+ DFSTestUtil.FsShellRun("-createSnapshot " + path + " sn2", config);
+ // verify the snapshotdiff using api and command line
+ SnapshotDiffReport report =
+ fs.getSnapshotDiffReport(new Path(path), "sn1", "sn2");
+ DFSTestUtil.toolRun(new SnapshotDiff(config), path + " sn1 sn2", 0,
+ report.toString());
+ DFSTestUtil.FsShellRun("-renameSnapshot " + path + " sn2 sn3", config);
+ DFSTestUtil.FsShellRun("-deleteSnapshot " + path + " sn1", config);
+ DFSTestUtil.FsShellRun("-deleteSnapshot " + path + " sn3", config);
+ DFSTestUtil.DFSAdminRun("-disallowSnapshot " + path, 0,
+ "Disallowing snaphot on " + path + " succeeded", config);
+ fs.delete(new Path("/Fully/QPath"), true);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] hadoop git commit: HDFS-11476. Fix NPE in
FsDatasetImpl#checkAndUpdate. Contributed by Xiaobing Zhou.
Posted by kk...@apache.org.
HDFS-11476. Fix NPE in FsDatasetImpl#checkAndUpdate. Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac5ae006
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac5ae006
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac5ae006
Branch: refs/heads/YARN-5972
Commit: ac5ae0065a127ac150a887fa6c6f3cffd86ef733
Parents: 2148b83
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Mar 3 13:31:20 2017 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Fri Mar 3 13:31:20 2017 -0800
----------------------------------------------------------------------
.../server/datanode/fsdataset/impl/FsDatasetImpl.java | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac5ae006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 6d00d75..aff19ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -2282,12 +2282,14 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if (memBlockInfo.getGenerationStamp() != diskGS) {
File memMetaFile = FsDatasetUtil.getMetaFile(diskFile,
memBlockInfo.getGenerationStamp());
- if (memMetaFile.exists()) {
- if (memMetaFile.compareTo(diskMetaFile) != 0) {
- LOG.warn("Metadata file in memory "
- + memMetaFile.getAbsolutePath()
- + " does not match file found by scan "
- + (diskMetaFile == null? null: diskMetaFile.getAbsolutePath()));
+ if (fileIoProvider.exists(vol, memMetaFile)) {
+ String warningPrefix = "Metadata file in memory "
+ + memMetaFile.getAbsolutePath()
+ + " does not match file found by scan ";
+ if (!diskMetaFileExists) {
+ LOG.warn(warningPrefix + "null");
+ } else if (memMetaFile.compareTo(diskMetaFile) != 0) {
+ LOG.warn(warningPrefix + diskMetaFile.getAbsolutePath());
}
} else {
// Metadata file corresponding to block in memory is missing
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] hadoop git commit: HADOOP-14057. Fix package.html to compile
with Java 9.
Posted by kk...@apache.org.
HADOOP-14057. Fix package.html to compile with Java 9.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/490abfb1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/490abfb1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/490abfb1
Branch: refs/heads/YARN-5972
Commit: 490abfb10f8c75ec75edc2359705601c4d4cb258
Parents: e58fc76
Author: Akira Ajisaka <aa...@apache.org>
Authored: Sat Mar 4 00:25:22 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Sat Mar 4 00:25:22 2017 +0900
----------------------------------------------------------------------
.../org/apache/hadoop/examples/pi/package.html | 46 ++++++------
.../java/org/apache/hadoop/record/package.html | 74 ++++++++++----------
.../org/apache/hadoop/typedbytes/package.html | 13 ++--
3 files changed, 65 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/490abfb1/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
index ad06b0b..a2b279d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
@@ -39,24 +39,24 @@
<li>Combine the job outputs and print the π bits.</li>
</ol>
-<table><tr valign=top><td width=420>
+<table summary="The Bits of Pi"><tr valign=top><td width=420>
<h3>The Bits of π</h3>
<p>
The table on the right are the results computed by distbbp.
</p>
<ul>
-<p><li>Row 0 to Row 7
+<li>Row 0 to Row 7
<ul><li>They were computed by a single machine.</li>
<li>A single run of Row 7 took several seconds.</li>
-</ul></li></p>
-<p><li>Row 8 to Row 14
+</ul></li>
+<li>Row 8 to Row 14
<ul><li>They were computed by a 7600-task-capacity cluster.</li>
<li>A single run of Row 14 took 27 hours.</li>
<li>The computations in Row 13 and Row 14 were completed on May 20, 2009.
It seems that the corresponding bits were never computed before.</li>
-</ul></li></p>
-<p><li>The first part of Row 15 (<tt>6216B06</tt>)
+</ul></li>
+<li>The first part of Row 15 (<tt>6216B06</tt>)
<ul><li>The first 30% of the computation was done in idle cycles of some
clusters spread over 20 days.</li>
@@ -68,8 +68,8 @@ The table on the right are the results computed by distbbp.
<li>The result was posted in
<a href="http://yahoohadoop.tumblr.com/post/98338598026/hadoop-computes-the-10-15-1st-bit-of-\u03c0">this YDN blog</a>.</li>
-</ul></li></p>
-<p><li>The second part of Row 15 (<tt>D3611</tt>)
+</ul></li>
+<li>The second part of Row 15 (<tt>D3611</tt>)
<ul><li>The starting position is 1,000,000,000,000,053, totally 20 bits.</li>
<li>Two computations, at positions <i>n</i> and <i>n</i>+4, were performed.
<li>A single computation was divided into 14,000 jobs
@@ -83,10 +83,10 @@ The table on the right are the results computed by distbbp.
The last bit, the 1,000,000,000,000,072<sup>nd</sup> bit,
probably is the highest bit (or the least significant bit) of π
computed ever in the history.</li>
-</ul></li></p>
-
+</ul></li>
+</ul>
</td><td width=20></td><td>
-<table border=1 width=400 cellpadding=5>
+<table border=1 width=400 cellpadding=5 summary="Pi in hex">
<tr><th width=30></th><th>Position <i>n</i></th><th>π bits (in hex) starting at <i>n</i></th></tr>
<tr><td align=right>0</td><td align=right>1</td><td><tt>243F6A8885A3</tt><sup>*</sup></td></tr>
@@ -110,29 +110,27 @@ The table on the right are the results computed by distbbp.
<tr><td align=right>15</td><td align=right>1,000,000,000,000,001</td><td><tt>6216B06</tt> ... <tt>D3611</tt></td></tr>
</table>
-<p><sup>*</sup>
+<sup>*</sup>
By representing π in decimal, hexadecimal and binary, we have
-<ul><table><tr>
+<table summary="Pi in various formats"><tr>
<td>π</td><td>=</td><td><tt>3.1415926535 8979323846 2643383279</tt> ...</td>
</tr><tr>
<td></td><td>=</td><td><tt>3.243F6A8885 A308D31319 8A2E037073</tt> ...</td>
</tr><tr>
<td></td><td>=</td><td><tt>11.0010010000 1111110110 1010100010</tt> ...</td>
-
-</td></tr></table></ul>
+</tr></table>
The first ten bits of π are <tt>0010010000</tt>.
-</p>
</td></tr></table>
<h3>Command Line Usages</h3>
The command line format is:
- <ul><pre>
+ <pre>
$ hadoop org.apache.hadoop.examples.pi.DistBbp \
- <b> <nThreads> <nJobs> <type> <nPart> <remoteDir> <localDir></pre></ul>
+ <b> <nThreads> <nJobs> <type> <nPart> <remoteDir> <localDir></pre>
And the parameters are:
- <ul><table>
+ <table summary="command line option">
<tr>
<td><b></td>
<td>The number of bits to skip, i.e. compute the (b+1)th position.</td>
@@ -159,14 +157,14 @@ $ hadoop org.apache.hadoop.examples.pi.DistBbp \
<td><localDir></td>
<td>Local directory for storing output files.</td>
</tr>
- </table></ul>
+ </table>
Note that it may take a long time to finish all the jobs when <b> is large.
If the program is killed in the middle of the execution, the same command with
a different <remoteDir> can be used to resume the execution. For example, suppose
we use the following command to compute the (10^15+57)th bit of π.
- <ul><pre>
+ <pre>
$ hadoop org.apache.hadoop.examples.pi.DistBbp \
- 1,000,000,000,000,056 20 1000 x 500 remote/a local/output</pre></ul>
+ 1,000,000,000,000,056 20 1000 x 500 remote/a local/output</pre>
It uses 20 threads to summit jobs so that there are at most 20 concurrent jobs.
Each sum (there are totally 14 sums) is partitioned into 1000 jobs.
The jobs will be executed in map-side or reduce-side. Each job has 500 parts.
@@ -174,8 +172,8 @@ $ hadoop org.apache.hadoop.examples.pi.DistBbp \
for storing output is local/output. Depends on the cluster configuration,
it may take many days to finish the entire execution. If the execution is killed,
we may resume it by
- <ul><pre>
+ <pre>
$ hadoop org.apache.hadoop.examples.pi.DistBbp \
- 1,000,000,000,000,056 20 1000 x 500 remote/b local/output</pre></ul>
+ 1,000,000,000,000,056 20 1000 x 500 remote/b local/output</pre>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/490abfb1/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/package.html
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/package.html b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/package.html
index d25e12c..2c1e945 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/package.html
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/package.html
@@ -204,8 +204,8 @@ type := (ptype / ctype)
ptype := ("byte" / "boolean" / "int" |
"long" / "float" / "double"
"ustring" / "buffer")
-ctype := (("vector" "<" type ">") /
- ("map" "<" type "," type ">" ) ) / name)
+ctype := (("vector" "<" type ">") /
+ ("map" "<" type "," type ">" ) ) / name)
</code></pre>
A DDL file describes one or more record types. It begins with zero or
@@ -255,7 +255,7 @@ include "links.jr"
module outlinks {
class OutLinks {
ustring baseURL;
- vector<links.Link> outLinks;
+ vector<links.Link> outLinks;
};
}
</code></pre>
@@ -269,7 +269,7 @@ record description files as a mandatory argument and an
optional language argument (the default is Java) --language or
-l. Thus a typical invocation would look like:
<pre><code>
-$ rcc -l C++ <filename> ...
+$ rcc -l C++ <filename> ...
</code></pre>
@@ -306,7 +306,7 @@ namespace hadoop {
class IOError : public runtime_error {
public:
- explicit IOError(const std::string& msg);
+ explicit IOError(const std::string& msg);
};
class IArchive;
@@ -314,18 +314,18 @@ namespace hadoop {
class RecordReader {
public:
- RecordReader(InStream& in, RecFormat fmt);
+ RecordReader(InStream& in, RecFormat fmt);
virtual ~RecordReader(void);
- virtual void read(Record& rec);
+ virtual void read(Record& rec);
};
class RecordWriter {
public:
- RecordWriter(OutStream& out, RecFormat fmt);
+ RecordWriter(OutStream& out, RecFormat fmt);
virtual ~RecordWriter(void);
- virtual void write(Record& rec);
+ virtual void write(Record& rec);
};
@@ -337,10 +337,10 @@ namespace hadoop {
virtual bool validate(void) const = 0;
virtual void
- serialize(OArchive& oa, const std::string& tag) const = 0;
+ serialize(OArchive& oa, const std::string& tag) const = 0;
virtual void
- deserialize(IArchive& ia, const std::string& tag) = 0;
+ deserialize(IArchive& ia, const std::string& tag) = 0;
};
}
</code></pre>
@@ -445,11 +445,11 @@ private:
...
public:
- std::string& getMyBuf() {
+ std::string& getMyBuf() {
return mMyBuf;
};
- const std::string& getMyBuf() const {
+ const std::string& getMyBuf() const {
return mMyBuf;
};
...
@@ -474,7 +474,7 @@ and the testrec.jr file contains:
include "inclrec.jr"
module testrec {
class R {
- vector<float> VF;
+ vector<float> VF;
RI Rec;
buffer Buf;
};
@@ -511,8 +511,8 @@ namespace inclrec {
RI(void);
virtual ~RI(void);
- virtual bool operator==(const RI& peer) const;
- virtual bool operator<(const RI& peer) const;
+ virtual bool operator==(const RI& peer) const;
+ virtual bool operator<(const RI& peer) const;
virtual int32_t getI32(void) const { return I32; }
virtual void setI32(int32_t v) { I32 = v; }
@@ -520,16 +520,16 @@ namespace inclrec {
virtual double getD(void) const { return D; }
virtual void setD(double v) { D = v; }
- virtual std::string& getS(void) const { return S; }
- virtual const std::string& getS(void) const { return S; }
+ virtual std::string& getS(void) const { return S; }
+ virtual const std::string& getS(void) const { return S; }
virtual std::string type(void) const;
virtual std::string signature(void) const;
protected:
- virtual void serialize(hadoop::OArchive& a) const;
- virtual void deserialize(hadoop::IArchive& a);
+ virtual void serialize(hadoop::OArchive& a) const;
+ virtual void deserialize(hadoop::IArchive& a);
};
} // end namespace inclrec
@@ -552,7 +552,7 @@ namespace testrec {
private:
- std::vector<float> VF;
+ std::vector<float> VF;
inclrec::RI Rec;
std::string Buf;
@@ -561,20 +561,20 @@ namespace testrec {
R(void);
virtual ~R(void);
- virtual bool operator==(const R& peer) const;
- virtual bool operator<(const R& peer) const;
+ virtual bool operator==(const R& peer) const;
+ virtual bool operator<(const R& peer) const;
- virtual std::vector<float>& getVF(void) const;
- virtual const std::vector<float>& getVF(void) const;
+ virtual std::vector<float>& getVF(void) const;
+ virtual const std::vector<float>& getVF(void) const;
- virtual std::string& getBuf(void) const ;
- virtual const std::string& getBuf(void) const;
+ virtual std::string& getBuf(void) const ;
+ virtual const std::string& getBuf(void) const;
- virtual inclrec::RI& getRec(void) const;
- virtual const inclrec::RI& getRec(void) const;
+ virtual inclrec::RI& getRec(void) const;
+ virtual const inclrec::RI& getRec(void) const;
- virtual bool serialize(hadoop::OutArchive& a) const;
- virtual bool deserialize(hadoop::InArchive& a);
+ virtual bool serialize(hadoop::OutArchive& a) const;
+ virtual bool deserialize(hadoop::InArchive& a);
virtual std::string type(void) const;
virtual std::string signature(void) const;
@@ -619,8 +619,8 @@ double double double
ustring std::string java.lang.String
buffer std::string org.apache.hadoop.record.Buffer
class type class type class type
-vector<type> std::vector<type> java.util.ArrayList<type>
-map<type,type> std::map<type,type> java.util.TreeMap<type,type>
+vector<type> std::vector<type> java.util.ArrayList<type>
+map<type,type> std::map<type,type> java.util.TreeMap<type,type>
</code></pre>
<h2>Data encodings</h2>
@@ -651,7 +651,7 @@ Primitive types are serialized as follows:
<li> byte: Represented by 1 byte, as is.
<li> boolean: Represented by 1-byte (0 or 1)
<li> int/long: Integers and longs are serialized zero compressed.
-Represented as 1-byte if -120 <= value < 128. Otherwise, serialized as a
+Represented as 1-byte if -120 <= value < 128. Otherwise, serialized as a
sequence of 2-5 bytes for ints, 2-9 bytes for longs. The first byte represents
the number of trailing bytes, N, as the negative number (-120-N). For example,
the number 1024 (0x400) is represented by the byte sequence 'x86 x04 x00'.
@@ -741,7 +741,7 @@ replace CRLF sequences with line feeds. Programming languages that we work
with do not impose these restrictions on string types. To work around these
restrictions, disallowed characters and CRs are percent escaped in strings.
The '%' character is also percent escaped.
-<li> buffer: XML tag <string&>. Values: Arbitrary binary
+<li> buffer: XML tag <string>. Values: Arbitrary binary
data. Represented as hexBinary, each byte is replaced by its 2-byte
hexadecimal representation.
</ul>
@@ -755,7 +755,7 @@ element and a <value> element. The <name> is a string that must
match /[a-zA-Z][a-zA-Z0-9_]*/. The value of the member is represented
by a <value> element.
-<li> vector: XML tag <array<. An <array> contains a
+<li> vector: XML tag <array>. An <array> contains a
single <data> element. The <data> element is a sequence of
<value> elements each of which represents an element of the vector.
@@ -768,7 +768,7 @@ For example:
<pre><code>
class {
int MY_INT; // value 5
- vector<float> MY_VEC; // values 0.1, -0.89, 2.45e4
+ vector<float> MY_VEC; // values 0.1, -0.89, 2.45e4
buffer MY_BUF; // value '\00\n\tabc%'
}
</code></pre>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/490abfb1/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/package.html
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/package.html b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/package.html
index 38c09f1..3494fbd 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/package.html
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/package.html
@@ -25,8 +25,8 @@ Typed bytes are sequences of bytes in which the first byte is a type code. They
<h3>Type Codes</h3>
Each typed bytes sequence starts with an unsigned byte that contains the type code. Possible values are:
-<p>
-<table border="1" cellpadding="2">
+
+<table border="1" cellpadding="2" summary="Type Codes">
<tr><th>Code</th><th>Type</th></tr>
<tr><td><i>0</i></td><td>A sequence of bytes.</td></tr>
<tr><td><i>1</i></td><td>A byte.</td></tr>
@@ -40,19 +40,19 @@ Each typed bytes sequence starts with an unsigned byte that contains the type co
<tr><td><i>9</i></td><td>A list.</td></tr>
<tr><td><i>10</i></td><td>A map.</td></tr>
</table>
-</p>
+
The type codes <i>50</i> to <i>200</i> are treated as aliases for <i>0</i>, and can thus be used for
application-specific serialization.
<h3>Subsequent Bytes</h3>
These are the subsequent bytes for the different type codes (everything is big-endian and unpadded):
-<p>
-<table border="1" cellpadding="2">
+
+<table border="1" cellpadding="2" summary="Subsequent Bytes">
<tr><th>Code</th><th>Subsequent Bytes</th></tr>
<tr><td><i>0</i></td><td><32-bit signed integer> <as many bytes as indicated by the integer></td></tr>
<tr><td><i>1</i></td><td><signed byte></td></tr>
-<tr><td><i>2</i></td><td><signed byte (<i>0 = <i>false</i> and <i>1</i> = <i>true</i>)></td></tr>
+<tr><td><i>2</i></td><td><signed byte (<i>0</i> = <i>false</i> and <i>1</i> = <i>true</i>)></td></tr>
<tr><td><i>3</i></td><td><32-bit signed integer></td></tr>
<tr><td><i>4</i></td><td><64-bit signed integer></td></tr>
<tr><td><i>5</i></td><td><32-bit IEEE floating point number></td></tr>
@@ -62,7 +62,6 @@ These are the subsequent bytes for the different type codes (everything is big-e
<tr><td><i>9</i></td><td><variable number of typed bytes sequences> <<i>255</i> written as an unsigned byte></td></tr>
<tr><td><i>10</i></td><td><32-bit signed integer> <as many (key-value) pairs of typed bytes sequences as indicated by the integer></td></tr>
</table>
-</p>
</body>
</html>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] hadoop git commit: YARN-5216. Expose configurable preemption
policy for OPPORTUNISTIC containers running on the NM. (Hitesh Sharma via
asuresh)
Posted by kk...@apache.org.
YARN-5216. Expose configurable preemption policy for OPPORTUNISTIC containers running on the NM. (Hitesh Sharma via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6999ad2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6999ad2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6999ad2e
Branch: refs/heads/YARN-5972
Commit: 6999ad2e0b74b3e0a97ee50d09ce20c422e05c1b
Parents: a135270
Author: Arun Suresh <as...@apache.org>
Authored: Sat Dec 24 17:16:52 2016 -0800
Committer: Konstantinos Karanasos <kk...@apache.org>
Committed: Tue Mar 7 14:53:41 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 9 ++
.../src/main/resources/yarn-default.xml | 9 ++
.../containermanager/container/Container.java | 2 +
.../container/ContainerImpl.java | 32 ++++--
.../scheduler/ContainerScheduler.java | 84 ++++++++++++---
.../TestContainerSchedulerQueuing.java | 103 +++++++++++++++++++
.../nodemanager/webapp/MockContainer.java | 5 +
7 files changed, 219 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6999ad2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b366855..9b12a55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -975,6 +975,15 @@ public class YarnConfiguration extends Configuration {
NM_PREFIX + "container-retry-minimum-interval-ms";
public static final int DEFAULT_NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS = 1000;
+ /**
+ * Use container pause as the preemption policy over kill in the container
+ * queue at a NodeManager.
+ **/
+ public static final String NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION =
+ NM_PREFIX + "opportunistic-containers-use-pause-for-preemption";
+ public static final boolean
+ DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION = false;
+
/** Interval at which the delayed token removal thread runs */
public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS =
RM_PREFIX + "delayed.delegation-token.removal-interval-ms";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6999ad2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 645a342..1b530da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2885,6 +2885,15 @@
<property>
<description>
+ Use container pause as the preemption policy over kill in the container
+ queue at a NodeManager.
+ </description>
+ <name>yarn.nodemanager.opportunistic-containers-use-pause-for-preemption</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
Error filename pattern, to identify the file in the container's
Log directory which contain the container's error log. As error file
redirection is done by client/AM and yarn will not be aware of the error
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6999ad2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 8004f33..b182151 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -92,4 +92,6 @@ public interface Container extends EventHandler<ContainerEvent> {
void sendLaunchEvent();
void sendKillEvent(int exitStatus, String description);
+
+ void sendPauseEvent(String description);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6999ad2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 86dced2..3f700a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -796,15 +796,22 @@ public class ContainerImpl implements Container {
@SuppressWarnings("unchecked") // dispatcher not typed
@Override
public void sendLaunchEvent() {
- ContainersLauncherEventType launcherEvent =
- ContainersLauncherEventType.LAUNCH_CONTAINER;
- if (recoveredStatus == RecoveredContainerStatus.LAUNCHED) {
- // try to recover a container that was previously launched
- launcherEvent = ContainersLauncherEventType.RECOVER_CONTAINER;
+ if (ContainerState.PAUSED == getContainerState()) {
+ dispatcher.getEventHandler().handle(
+ new ContainerResumeEvent(containerId,
+ "Container Resumed as some resources freed up"));
+ } else {
+ ContainersLauncherEventType launcherEvent =
+ ContainersLauncherEventType.LAUNCH_CONTAINER;
+ if (recoveredStatus == RecoveredContainerStatus.LAUNCHED) {
+ // try to recover a container that was previously launched
+ launcherEvent = ContainersLauncherEventType.RECOVER_CONTAINER;
+ }
+ containerLaunchStartTime = clock.getTime();
+ dispatcher.getEventHandler().handle(
+ new ContainersLauncherEvent(this, launcherEvent));
}
- containerLaunchStartTime = clock.getTime();
- dispatcher.getEventHandler().handle(
- new ContainersLauncherEvent(this, launcherEvent));
+
}
@SuppressWarnings("unchecked") // dispatcher not typed
@@ -824,6 +831,13 @@ public class ContainerImpl implements Container {
}
@SuppressWarnings("unchecked") // dispatcher not typed
+ @Override
+ public void sendPauseEvent(String description) {
+ dispatcher.getEventHandler().handle(
+ new ContainerPauseEvent(containerId, description));
+ }
+
+ @SuppressWarnings("unchecked") // dispatcher not typed
private void sendRelaunchEvent() {
ContainersLauncherEventType launcherEvent =
ContainersLauncherEventType.RELAUNCH_CONTAINER;
@@ -1770,7 +1784,7 @@ public class ContainerImpl implements Container {
/**
* Transitions upon receiving PAUSE_CONTAINER.
- * - RUNNING -> PAUSED
+ * - RUNNING -> PAUSING
*/
@SuppressWarnings("unchecked") // dispatcher not typed
static class PauseContainerTransition implements
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6999ad2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 24530b3..3c1eb6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -31,6 +32,7 @@ import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
@@ -71,7 +73,7 @@ public class ContainerScheduler extends AbstractService implements
queuedOpportunisticContainers = new LinkedHashMap<>();
// Used to keep track of containers that have been marked to be killed
- // to make room for a guaranteed container.
+ // or paused to make room for a guaranteed container.
private final Map<ContainerId, Container> oppContainersToKill =
new HashMap<>();
@@ -95,6 +97,8 @@ public class ContainerScheduler extends AbstractService implements
private final AsyncDispatcher dispatcher;
private final NodeManagerMetrics metrics;
+ private Boolean usePauseEventForPreemption = false;
+
/**
* Instantiate a Container Scheduler.
* @param context NodeManager Context.
@@ -109,6 +113,17 @@ public class ContainerScheduler extends AbstractService implements
DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH));
}
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ this.usePauseEventForPreemption =
+ conf.getBoolean(
+ YarnConfiguration.NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION,
+ YarnConfiguration.
+ DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION);
+ }
+
@VisibleForTesting
public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
NodeManagerMetrics metrics, int qLength) {
@@ -133,8 +148,9 @@ public class ContainerScheduler extends AbstractService implements
case SCHEDULE_CONTAINER:
scheduleContainer(event.getContainer());
break;
+ case CONTAINER_PAUSED:
case CONTAINER_COMPLETED:
- onContainerCompleted(event.getContainer());
+ onResourcesReclaimed(event.getContainer());
break;
case SHED_QUEUED_CONTAINERS:
shedQueuedOpportunisticContainers();
@@ -164,6 +180,12 @@ public class ContainerScheduler extends AbstractService implements
return this.queuedOpportunisticContainers.size();
}
+ @VisibleForTesting
+ public void setUsePauseEventForPreemption(
+ boolean usePauseEventForPreemption) {
+ this.usePauseEventForPreemption = usePauseEventForPreemption;
+ }
+
public OpportunisticContainersStatus getOpportunisticContainersStatus() {
this.opportunisticContainersStatus.setQueuedOpportContainers(
getNumQueuedOpportunisticContainers());
@@ -178,7 +200,7 @@ public class ContainerScheduler extends AbstractService implements
return this.opportunisticContainersStatus;
}
- private void onContainerCompleted(Container container) {
+ private void onResourcesReclaimed(Container container) {
oppContainersToKill.remove(container.getContainerId());
// This could be killed externally for eg. by the ContainerManager,
@@ -206,6 +228,24 @@ public class ContainerScheduler extends AbstractService implements
// Start pending guaranteed containers, if resources available.
boolean resourcesAvailable =
startContainersFromQueue(queuedGuaranteedContainers.values());
+ // Resume opportunistic containers, if resource available.
+ if (resourcesAvailable) {
+ List<Container> pausedContainers = new ArrayList<Container>();
+ Map<ContainerId, Container> containers =
+ context.getContainers();
+ for (Map.Entry<ContainerId, Container>entry : containers.entrySet()) {
+ ContainerId contId = entry.getKey();
+ // Find containers that were not already started and are in paused state
+ if(false == runningContainers.containsKey(contId)) {
+ if(containers.get(contId).getContainerState()
+ == ContainerState.PAUSED) {
+ pausedContainers.add(containers.get(contId));
+ }
+ }
+ }
+ resourcesAvailable =
+ startContainersFromQueue(pausedContainers);
+ }
// Start opportunistic containers, if resources available.
if (resourcesAvailable) {
startContainersFromQueue(queuedOpportunisticContainers.values());
@@ -245,9 +285,9 @@ public class ContainerScheduler extends AbstractService implements
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.GUARANTEED) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
- // Kill running opportunistic containers to make space for
+ // Kill / Pause running opportunistic containers to make space for
// guaranteed container.
- killOpportunisticContainers(container);
+ reclaimOpportunisticContainerResources(container);
} else {
if (queuedOpportunisticContainers.size() <= maxOppQueueLength) {
LOG.info("Opportunistic container {} will be queued at the NM.",
@@ -276,22 +316,34 @@ public class ContainerScheduler extends AbstractService implements
}
}
- private void killOpportunisticContainers(Container container) {
- List<Container> extraOpportContainersToKill =
- pickOpportunisticContainersToKill(container.getContainerId());
+ @SuppressWarnings("unchecked")
+ private void reclaimOpportunisticContainerResources(Container container) {
+ List<Container> extraOppContainersToReclaim =
+ pickOpportunisticContainersToReclaimResources(
+ container.getContainerId());
// Kill the opportunistic containers that were chosen.
- for (Container contToKill : extraOpportContainersToKill) {
- contToKill.sendKillEvent(
- ContainerExitStatus.KILLED_BY_CONTAINER_SCHEDULER,
- "Container Killed to make room for Guaranteed Container.");
- oppContainersToKill.put(contToKill.getContainerId(), contToKill);
+ for (Container contToReclaim : extraOppContainersToReclaim) {
+ String preemptionAction = usePauseEventForPreemption == true ? "paused" :
+ "resumed";
LOG.info(
- "Opportunistic container {} will be killed in order to start the "
+ "Container {} will be {} to start the "
+ "execution of guaranteed container {}.",
- contToKill.getContainerId(), container.getContainerId());
+ contToReclaim.getContainerId(), preemptionAction,
+ container.getContainerId());
+
+ if (usePauseEventForPreemption) {
+ contToReclaim.sendPauseEvent(
+ "Container Paused to make room for Guaranteed Container");
+ } else {
+ contToReclaim.sendKillEvent(
+ ContainerExitStatus.KILLED_BY_CONTAINER_SCHEDULER,
+ "Container Killed to make room for Guaranteed Container.");
+ }
+ oppContainersToKill.put(contToReclaim.getContainerId(), contToReclaim);
}
}
+ @SuppressWarnings("unchecked")
private void startAllocatedContainer(Container container) {
LOG.info("Starting container [" + container.getContainerId()+ "]");
runningContainers.put(container.getContainerId(), container);
@@ -303,7 +355,7 @@ public class ContainerScheduler extends AbstractService implements
container.sendLaunchEvent();
}
- private List<Container> pickOpportunisticContainersToKill(
+ private List<Container> pickOpportunisticContainersToReclaimResources(
ContainerId containerToStartId) {
// The opportunistic containers that need to be killed for the
// given container to start.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6999ad2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index 24e388f..1da4804 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -23,6 +23,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@@ -46,6 +48,7 @@ import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
@@ -125,17 +128,37 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
@Override
protected ContainerExecutor createContainerExecutor() {
DefaultContainerExecutor exec = new DefaultContainerExecutor() {
+ ConcurrentMap<String, Boolean> oversleepMap =
+ new ConcurrentHashMap<String, Boolean>();
@Override
public int launchContainer(ContainerStartContext ctx) throws IOException {
+ oversleepMap.put(ctx.getContainer().getContainerId().toString(), false);
if (delayContainers) {
try {
Thread.sleep(10000);
+ if(oversleepMap.get(ctx.getContainer().getContainerId().toString())
+ == true) {
+ Thread.sleep(10000);
+ }
} catch (InterruptedException e) {
// Nothing..
}
}
return super.launchContainer(ctx);
}
+
+ @Override
+ public void pauseContainer(Container container) {
+ // To mimic pausing we force the container to be in the PAUSED state
+ // a little longer by oversleeping.
+ oversleepMap.put(container.getContainerId().toString(), true);
+ LOG.info("Container was paused");
+ }
+
+ @Override
+ public void resumeContainer(Container container) {
+ LOG.info("Container was resumed");
+ }
};
exec.setConf(conf);
return spy(exec);
@@ -421,6 +444,86 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
}
/**
+ * Submit two OPPORTUNISTIC and one GUARANTEED containers. The resources
+ * requests by each container as such that only one can run in parallel.
+ * Thus, the OPPORTUNISTIC container that started running, will be
+ * paused for the GUARANTEED container to start.
+ * Once the GUARANTEED container finishes its execution, the remaining
+ * OPPORTUNISTIC container will be executed.
+ * @throws Exception
+ */
+ @Test
+ public void testPauseOpportunisticForGuaranteedContainer() throws Exception {
+ containerManager.start();
+ containerManager.getContainerScheduler().
+ setUsePauseEventForPreemption(true);
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+
+ List<StartContainerRequest> list = new ArrayList<>();
+ list.add(StartContainerRequest.newInstance(
+ containerLaunchContext,
+ createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
+ context.getNodeId(),
+ user, BuilderUtils.newResource(2048, 1),
+ context.getContainerTokenSecretManager(), null,
+ ExecutionType.OPPORTUNISTIC)));
+
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
+
+ BaseContainerManagerTest.waitForNMContainerState(containerManager,
+ createContainerId(0), ContainerState.RUNNING, 40);
+
+ list = new ArrayList<>();
+ list.add(StartContainerRequest.newInstance(
+ containerLaunchContext,
+ createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
+ context.getNodeId(),
+ user, BuilderUtils.newResource(2048, 1),
+ context.getContainerTokenSecretManager(), null,
+ ExecutionType.GUARANTEED)));
+ allRequests =
+ StartContainersRequest.newInstance(list);
+
+ containerManager.startContainers(allRequests);
+
+ BaseContainerManagerTest.waitForNMContainerState(containerManager,
+ createContainerId(1), ContainerState.RUNNING, 40);
+
+ // Get container statuses. Container 0 should be paused, container 1
+ // should be running.
+ List<ContainerId> statList = new ArrayList<ContainerId>();
+ for (int i = 0; i < 2; i++) {
+ statList.add(createContainerId(i));
+ }
+ GetContainerStatusesRequest statRequest =
+ GetContainerStatusesRequest.newInstance(statList);
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(createContainerId(0))) {
+ Assert.assertTrue(status.getDiagnostics().contains(
+ "Container Paused to make room for Guaranteed Container"));
+ } else if (status.getContainerId().equals(createContainerId(1))) {
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
+ status.getState());
+ }
+ System.out.println("\nStatus : [" + status + "]\n");
+ }
+
+ // Make sure that the GUARANTEED container completes
+ BaseContainerManagerTest.waitForNMContainerState(containerManager,
+ createContainerId(1), ContainerState.DONE, 40);
+ // Make sure that the PAUSED opportunistic container resumes and
+ // starts running
+ BaseContainerManagerTest.waitForNMContainerState(containerManager,
+ createContainerId(0), ContainerState.DONE, 40);
+ }
+
+ /**
* 1. Submit a long running GUARANTEED container to hog all NM resources.
* 2. Submit 6 OPPORTUNISTIC containers, all of which will be queued.
* 3. Update the Queue Limit to 2.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6999ad2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 686a0d9..487d332 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -230,4 +230,9 @@ public class MockContainer implements Container {
public void sendKillEvent(int exitStatus, String description) {
}
+
+ @Override
+ public void sendPauseEvent(String description) {
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] hadoop git commit: HADOOP-6801. io.sort.mb and io.sort.factor
were renamed and moved to mapreduce but are still in
CommonConfigurationKeysPublic.java and used in SequenceFile.java.
Posted by kk...@apache.org.
HADOOP-6801. io.sort.mb and io.sort.factor were renamed and moved to mapreduce but are still in CommonConfigurationKeysPublic.java and used in SequenceFile.java.
This closes #146
Signed-off-by: Akira Ajisaka <aa...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb5a1795
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb5a1795
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb5a1795
Branch: refs/heads/YARN-5972
Commit: eb5a17954a758fdb1f3f29ef34e129d5f37d3a26
Parents: 19a1fc6
Author: Harsh J <ha...@cloudera.com>
Authored: Wed Oct 26 13:21:51 2016 +0530
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Mar 3 18:38:04 2017 +0900
----------------------------------------------------------------------
.../fs/CommonConfigurationKeysPublic.java | 29 ++++++++-
.../java/org/apache/hadoop/io/SequenceFile.java | 20 +++++-
.../src/main/resources/core-default.xml | 19 ++++++
.../org/apache/hadoop/io/TestSequenceFile.java | 68 +++++++++++++++++++-
4 files changed, 131 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb5a1795/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index e1feda1..6b267dc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -250,18 +250,43 @@ public class CommonConfigurationKeysPublic {
* @deprecated Moved to mapreduce, see mapreduce.task.io.sort.mb
* in mapred-default.xml
* See https://issues.apache.org/jira/browse/HADOOP-6801
+ *
+ * For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ * instead, see {@link #SEQ_IO_SORT_MB_KEY}.
*/
public static final String IO_SORT_MB_KEY = "io.sort.mb";
- /** Default value for IO_SORT_MB_DEFAULT */
+ /** Default value for {@link #IO_SORT_MB_KEY}. */
public static final int IO_SORT_MB_DEFAULT = 100;
/**
* @deprecated Moved to mapreduce, see mapreduce.task.io.sort.factor
* in mapred-default.xml
* See https://issues.apache.org/jira/browse/HADOOP-6801
+ *
+ * For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ * instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.
*/
public static final String IO_SORT_FACTOR_KEY = "io.sort.factor";
- /** Default value for IO_SORT_FACTOR_DEFAULT */
+ /** Default value for {@link #IO_SORT_FACTOR_KEY}. */
public static final int IO_SORT_FACTOR_DEFAULT = 100;
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final String SEQ_IO_SORT_MB_KEY = "seq.io.sort.mb";
+ /** Default value for {@link #SEQ_IO_SORT_MB_KEY}. */
+ public static final int SEQ_IO_SORT_MB_DEFAULT = 100;
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final String SEQ_IO_SORT_FACTOR_KEY = "seq.io.sort.factor";
+ /** Default value for {@link #SEQ_IO_SORT_FACTOR_KEY}. */
+ public static final int SEQ_IO_SORT_FACTOR_DEFAULT = 100;
+
/**
* @see
* <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb5a1795/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index c510ff7..de0bf4f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -2816,14 +2816,30 @@ public class SequenceFile {
}
/** Sort and merge using an arbitrary {@link RawComparator}. */
+ @SuppressWarnings("deprecation")
public Sorter(FileSystem fs, RawComparator comparator, Class keyClass,
Class valClass, Configuration conf, Metadata metadata) {
this.fs = fs;
this.comparator = comparator;
this.keyClass = keyClass;
this.valClass = valClass;
- this.memory = conf.getInt("io.sort.mb", 100) * 1024 * 1024;
- this.factor = conf.getInt("io.sort.factor", 100);
+ // Remember to fall-back on the deprecated MB and Factor keys
+ // until they are removed away permanently.
+ if (conf.get(CommonConfigurationKeys.IO_SORT_MB_KEY) != null) {
+ this.memory = conf.getInt(CommonConfigurationKeys.IO_SORT_MB_KEY,
+ CommonConfigurationKeys.SEQ_IO_SORT_MB_DEFAULT) * 1024 * 1024;
+ } else {
+ this.memory = conf.getInt(CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY,
+ CommonConfigurationKeys.SEQ_IO_SORT_MB_DEFAULT) * 1024 * 1024;
+ }
+ if (conf.get(CommonConfigurationKeys.IO_SORT_FACTOR_KEY) != null) {
+ this.factor = conf.getInt(CommonConfigurationKeys.IO_SORT_FACTOR_KEY,
+ CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_DEFAULT);
+ } else {
+ this.factor = conf.getInt(
+ CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY,
+ CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_DEFAULT);
+ }
this.conf = conf;
this.metadata = metadata;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb5a1795/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 08ca05b..35be56b 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2476,4 +2476,23 @@
in audit logs.
</description>
</property>
+<!-- SequenceFile's Sorter properties -->
+ <property>
+ <name>seq.io.sort.mb</name>
+ <value>100</value>
+ <description>
+ The total amount of buffer memory to use while sorting files,
+ while using SequenceFile.Sorter, in megabytes. By default,
+ gives each merge stream 1MB, which should minimize seeks.
+ </description>
+ </property>
+ <property>
+ <name>seq.io.sort.factor</name>
+ <value>100</value>
+ <description>
+ The number of streams to merge at once while sorting
+ files using SequenceFile.Sorter.
+ This determines the number of open file handles.
+ </description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb5a1795/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
index b76cff6..e97ab6a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
@@ -38,6 +38,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import org.mockito.Mockito;
@@ -54,7 +55,72 @@ public class TestSequenceFile {
compressedSeqFileTest(new DefaultCodec());
LOG.info("Successfully tested SequenceFile with DefaultCodec");
}
-
+
+ @SuppressWarnings("deprecation")
+ public void testSorterProperties() throws IOException {
+ // Test to ensure that deprecated properties have no default
+ // references anymore.
+ Configuration config = new Configuration();
+ assertNull("The deprecated sort memory property "
+ + CommonConfigurationKeys.IO_SORT_MB_KEY
+ + " must not exist in any core-*.xml files.",
+ config.get(CommonConfigurationKeys.IO_SORT_MB_KEY));
+ assertNull("The deprecated sort factor property "
+ + CommonConfigurationKeys.IO_SORT_FACTOR_KEY
+ + " must not exist in any core-*.xml files.",
+ config.get(CommonConfigurationKeys.IO_SORT_FACTOR_KEY));
+
+ // Test deprecated property honoring
+ // Set different values for old and new property names
+ // and compare which one gets loaded
+ config = new Configuration();
+ FileSystem fs = FileSystem.get(config);
+ config.setInt(CommonConfigurationKeys.IO_SORT_MB_KEY, 10);
+ config.setInt(CommonConfigurationKeys.IO_SORT_FACTOR_KEY, 10);
+ config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY, 20);
+ config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY, 20);
+ SequenceFile.Sorter sorter = new SequenceFile.Sorter(
+ fs, Text.class, Text.class, config);
+ assertEquals("Deprecated memory conf must be honored over newer property",
+ 10*1024*1024, sorter.getMemory());
+ assertEquals("Deprecated factor conf must be honored over newer property",
+ 10, sorter.getFactor());
+
+ // Test deprecated properties (graceful deprecation)
+ config = new Configuration();
+ fs = FileSystem.get(config);
+ config.setInt(CommonConfigurationKeys.IO_SORT_MB_KEY, 10);
+ config.setInt(CommonConfigurationKeys.IO_SORT_FACTOR_KEY, 10);
+ sorter = new SequenceFile.Sorter(
+ fs, Text.class, Text.class, config);
+ assertEquals("Deprecated memory property "
+ + CommonConfigurationKeys.IO_SORT_MB_KEY
+ + " must get properly applied.",
+ 10*1024*1024, // In bytes
+ sorter.getMemory());
+ assertEquals("Deprecated sort factor property "
+ + CommonConfigurationKeys.IO_SORT_FACTOR_KEY
+ + " must get properly applied.",
+ 10, sorter.getFactor());
+
+ // Test regular properties (graceful deprecation)
+ config = new Configuration();
+ fs = FileSystem.get(config);
+ config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY, 20);
+ config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY, 20);
+ sorter = new SequenceFile.Sorter(
+ fs, Text.class, Text.class, config);
+ assertEquals("Memory property "
+ + CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY
+ + " must get properly applied if present.",
+ 20*1024*1024, // In bytes
+ sorter.getMemory());
+ assertEquals("Merge factor property "
+ + CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY
+ + " must get properly applied if present.",
+ 20, sorter.getFactor());
+ }
+
public void compressedSeqFileTest(CompressionCodec codec) throws Exception {
int count = 1024 * 10;
int megabytes = 1;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] hadoop git commit: HADOOP-14056. Update maven-javadoc-plugin
to 2.10.4.
Posted by kk...@apache.org.
HADOOP-14056. Update maven-javadoc-plugin to 2.10.4.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fad766e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fad766e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fad766e1
Branch: refs/heads/YARN-5972
Commit: fad766e1d255648ee5dd9bc1f58cef42b930bc83
Parents: c8bd8ac
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Mar 6 10:30:01 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Mar 6 10:30:01 2017 +0900
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad766e1/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index c8aa857..4c8b900 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1322,7 +1322,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
- <version>2.8.1</version>
+ <version>${maven-javadoc-plugin.version}</version>
<configuration>
<additionalparam>-Xmaxwarns 10000</additionalparam>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad766e1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3eeba1e..f07256d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,7 +98,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
<maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
<maven-enforcer-plugin.version>1.4.1</maven-enforcer-plugin.version>
- <maven-javadoc-plugin.version>2.9.1</maven-javadoc-plugin.version>
+ <maven-javadoc-plugin.version>2.10.4</maven-javadoc-plugin.version>
<maven-gpg-plugin.version>1.5</maven-gpg-plugin.version>
<maven-remote-resources-plugin.version>1.5</maven-remote-resources-plugin.version>
<maven-resources-plugin.version>3.0.1</maven-resources-plugin.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] hadoop git commit: YARN-5292. NM Container lifecycle and
state transitions to support for PAUSED container state. (Hitesh Sharma via
asuresh)
Posted by kk...@apache.org.
YARN-5292. NM Container lifecycle and state transitions to support for PAUSED container state. (Hitesh Sharma via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1352700
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1352700
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1352700
Branch: refs/heads/YARN-5972
Commit: a1352700544acf723156cb42bf68c25962ca04eb
Parents: 1598fd3
Author: Arun Suresh <as...@apache.org>
Authored: Fri Dec 9 07:51:03 2016 -0800
Committer: Konstantinos Karanasos <kk...@apache.org>
Committed: Tue Mar 7 14:53:41 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/api/records/ContainerState.java | 7 +-
.../src/main/proto/yarn_protos.proto | 1 +
.../server/nodemanager/ContainerExecutor.java | 22 +++
.../container/ContainerEventType.java | 6 +-
.../container/ContainerImpl.java | 170 ++++++++++++++++++-
.../container/ContainerPauseEvent.java | 40 +++++
.../container/ContainerResumeEvent.java | 39 +++++
.../container/ContainerState.java | 3 +-
.../launcher/ContainerLaunch.java | 90 +++++++++-
.../launcher/ContainersLauncher.java | 32 ++++
.../launcher/ContainersLauncherEventType.java | 3 +
.../scheduler/ContainerSchedulerEventType.java | 1 +
.../container/TestContainer.java | 51 ++++++
13 files changed, 454 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
index 696fe06..45e5bd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
@@ -33,11 +33,14 @@ public enum ContainerState {
/** Running container */
RUNNING,
-
+
/** Completed container */
COMPLETE,
/** Scheduled (awaiting resources) at the NM. */
@InterfaceStability.Unstable
- SCHEDULED
+ SCHEDULED,
+
+ /** Paused at the NM. */
+ PAUSED
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 3b26a5c..2ae82a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -83,6 +83,7 @@ enum ContainerStateProto {
C_RUNNING = 2;
C_COMPLETE = 3;
C_SCHEDULED = 4;
+ C_PAUSED = 5;
}
message ContainerProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 64d2af9..7cb44fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -692,6 +692,28 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
+ * Pause the container. The default implementation is to raise a kill event.
+ * Specific executor implementations can override this behavior.
+ * @param container
+ * the Container
+ */
+ public void pauseContainer(Container container) {
+ LOG.warn(container.getContainerId() + " doesn't support pausing.");
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Resume the container from pause state. The default implementation ignores
+ * this event. Specific implementations can override this behavior.
+ * @param container
+ * the Container
+ */
+ public void resumeContainer(Container container) {
+ LOG.warn(container.getContainerId() + " doesn't support resume.");
+ throw new UnsupportedOperationException();
+ }
+
+ /**
* Get the process-identifier for the container.
*
* @param containerID the container ID
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
index afea0e6..1475435 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
@@ -27,6 +27,8 @@ public enum ContainerEventType {
CONTAINER_DONE,
REINITIALIZE_CONTAINER,
ROLLBACK_REINIT,
+ PAUSE_CONTAINER,
+ RESUME_CONTAINER,
// DownloadManager
CONTAINER_INITED,
@@ -38,5 +40,7 @@ public enum ContainerEventType {
CONTAINER_LAUNCHED,
CONTAINER_EXITED_WITH_SUCCESS,
CONTAINER_EXITED_WITH_FAILURE,
- CONTAINER_KILLED_ON_REQUEST
+ CONTAINER_KILLED_ON_REQUEST,
+ CONTAINER_PAUSED,
+ CONTAINER_RESUMED
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index cae30cd..86dced2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -298,6 +298,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.NEW, ContainerState.DONE,
ContainerEventType.KILL_CONTAINER, new KillOnNewTransition())
+ .addTransition(ContainerState.NEW, ContainerState.DONE,
+ ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
// From LOCALIZING State
.addTransition(ContainerState.LOCALIZING,
@@ -313,6 +315,8 @@ public class ContainerImpl implements Container {
.addTransition(ContainerState.LOCALIZING, ContainerState.KILLING,
ContainerEventType.KILL_CONTAINER,
new KillBeforeRunningTransition())
+ .addTransition(ContainerState.LOCALIZING, ContainerState.KILLING,
+ ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
// From LOCALIZATION_FAILED State
.addTransition(ContainerState.LOCALIZATION_FAILED,
@@ -326,7 +330,8 @@ public class ContainerImpl implements Container {
// container not launched so kill is a no-op
.addTransition(ContainerState.LOCALIZATION_FAILED,
ContainerState.LOCALIZATION_FAILED,
- ContainerEventType.KILL_CONTAINER)
+ EnumSet.of(ContainerEventType.KILL_CONTAINER,
+ ContainerEventType.PAUSE_CONTAINER))
// container cleanup triggers a release of all resources
// regardless of whether they were localized or not
// LocalizedResource handles release event in all states
@@ -382,6 +387,76 @@ public class ContainerImpl implements Container {
ContainerState.EXITED_WITH_FAILURE,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
new KilledExternallyTransition())
+ .addTransition(ContainerState.RUNNING, ContainerState.PAUSING,
+ ContainerEventType.PAUSE_CONTAINER, new PauseContainerTransition())
+
+ // From PAUSING State
+ .addTransition(ContainerState.PAUSING, ContainerState.KILLING,
+ ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.PAUSING, ContainerState.PAUSING,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ UPDATE_DIAGNOSTICS_TRANSITION)
+ .addTransition(ContainerState.PAUSING, ContainerState.PAUSED,
+ ContainerEventType.CONTAINER_PAUSED, new PausedContainerTransition())
+ // In case something goes wrong then container will exit from the
+ // PAUSING state
+ .addTransition(ContainerState.PAUSING,
+ ContainerState.EXITED_WITH_SUCCESS,
+ ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS)
+ .addTransition(ContainerState.PAUSING,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ new ExitedWithFailureTransition(true))
+ .addTransition(ContainerState.PAUSING, ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ new KilledExternallyTransition())
+
+ // From PAUSED State
+ .addTransition(ContainerState.PAUSED, ContainerState.KILLING,
+ ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.PAUSED, ContainerState.PAUSED,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ UPDATE_DIAGNOSTICS_TRANSITION)
+ .addTransition(ContainerState.PAUSED, ContainerState.PAUSED,
+ ContainerEventType.PAUSE_CONTAINER)
+ .addTransition(ContainerState.PAUSED, ContainerState.RESUMING,
+ ContainerEventType.RESUME_CONTAINER, new ResumeContainerTransition())
+ // In case something goes wrong then container will exit from the
+ // PAUSED state
+ .addTransition(ContainerState.PAUSED,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ new ExitedWithFailureTransition(true))
+ .addTransition(ContainerState.PAUSED, ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ new KilledExternallyTransition())
+ .addTransition(ContainerState.PAUSED,
+ ContainerState.EXITED_WITH_SUCCESS,
+ ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
+ new ExitedWithSuccessTransition(true))
+
+ // From RESUMING State
+ .addTransition(ContainerState.RESUMING, ContainerState.KILLING,
+ ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.RESUMING, ContainerState.RUNNING,
+ ContainerEventType.CONTAINER_RESUMED)
+ .addTransition(ContainerState.RESUMING, ContainerState.RESUMING,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ UPDATE_DIAGNOSTICS_TRANSITION)
+ // In case something goes wrong then container will exit from the
+ // RESUMING state
+ .addTransition(ContainerState.RESUMING,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ new ExitedWithFailureTransition(true))
+ .addTransition(ContainerState.RESUMING,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ new KilledExternallyTransition())
+ .addTransition(ContainerState.RESUMING,
+ ContainerState.EXITED_WITH_SUCCESS,
+ ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
+ new ExitedWithSuccessTransition(true))
// From REINITIALIZING State
.addTransition(ContainerState.REINITIALIZING,
@@ -405,6 +480,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.REINITIALIZING, ContainerState.KILLING,
ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.REINITIALIZING, ContainerState.KILLING,
+ ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
.addTransition(ContainerState.REINITIALIZING,
ContainerState.SCHEDULED,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
@@ -422,6 +499,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.RELAUNCHING, ContainerState.KILLING,
ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.RELAUNCHING, ContainerState.KILLING,
+ ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
// From CONTAINER_EXITED_WITH_SUCCESS State
.addTransition(ContainerState.EXITED_WITH_SUCCESS, ContainerState.DONE,
@@ -433,7 +512,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.EXITED_WITH_SUCCESS,
ContainerState.EXITED_WITH_SUCCESS,
- ContainerEventType.KILL_CONTAINER)
+ EnumSet.of(ContainerEventType.KILL_CONTAINER,
+ ContainerEventType.PAUSE_CONTAINER))
// From EXITED_WITH_FAILURE State
.addTransition(ContainerState.EXITED_WITH_FAILURE, ContainerState.DONE,
@@ -445,7 +525,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.EXITED_WITH_FAILURE,
ContainerState.EXITED_WITH_FAILURE,
- ContainerEventType.KILL_CONTAINER)
+ EnumSet.of(ContainerEventType.KILL_CONTAINER,
+ ContainerEventType.PAUSE_CONTAINER))
// From KILLING State.
.addTransition(ContainerState.KILLING,
@@ -479,7 +560,8 @@ public class ContainerImpl implements Container {
// in the container launcher
.addTransition(ContainerState.KILLING,
ContainerState.KILLING,
- ContainerEventType.CONTAINER_LAUNCHED)
+ EnumSet.of(ContainerEventType.CONTAINER_LAUNCHED,
+ ContainerEventType.PAUSE_CONTAINER))
// From CONTAINER_CLEANEDUP_AFTER_KILL State.
.addTransition(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
@@ -495,11 +577,13 @@ public class ContainerImpl implements Container {
EnumSet.of(ContainerEventType.KILL_CONTAINER,
ContainerEventType.RESOURCE_FAILED,
ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
- ContainerEventType.CONTAINER_EXITED_WITH_FAILURE))
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ ContainerEventType.PAUSE_CONTAINER))
// From DONE
.addTransition(ContainerState.DONE, ContainerState.DONE,
- ContainerEventType.KILL_CONTAINER)
+ EnumSet.of(ContainerEventType.KILL_CONTAINER,
+ ContainerEventType.PAUSE_CONTAINER))
.addTransition(ContainerState.DONE, ContainerState.DONE,
ContainerEventType.INIT_CONTAINER)
.addTransition(ContainerState.DONE, ContainerState.DONE,
@@ -525,6 +609,8 @@ public class ContainerImpl implements Container {
case LOCALIZING:
case LOCALIZATION_FAILED:
case SCHEDULED:
+ case PAUSED:
+ case RESUMING:
return org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED;
case RUNNING:
case RELAUNCHING:
@@ -534,6 +620,7 @@ public class ContainerImpl implements Container {
case KILLING:
case CONTAINER_CLEANEDUP_AFTER_KILL:
case CONTAINER_RESOURCES_CLEANINGUP:
+ case PAUSING:
return org.apache.hadoop.yarn.api.records.ContainerState.RUNNING;
case DONE:
default:
@@ -1473,6 +1560,26 @@ public class ContainerImpl implements Container {
}
/**
+ * Transitions upon receiving PAUSE_CONTAINER.
+ * - LOCALIZED -> KILLING.
+ * - REINITIALIZING -> KILLING.
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ static class KillOnPauseTransition implements
+ SingleArcTransition<ContainerImpl, ContainerEvent> {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ // Kill the process/process-grp
+ container.setIsReInitializing(false);
+ container.dispatcher.getEventHandler().handle(
+ new ContainersLauncherEvent(container,
+ ContainersLauncherEventType.CLEANUP_CONTAINER));
+ }
+ }
+
+ /**
* Transition from KILLING to CONTAINER_CLEANEDUP_AFTER_KILL
* upon receiving CONTAINER_KILLED_ON_REQUEST.
*/
@@ -1661,6 +1768,57 @@ public class ContainerImpl implements Container {
}
}
+ /**
+ * Transitions upon receiving PAUSE_CONTAINER.
+ * - RUNNING -> PAUSED
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ static class PauseContainerTransition implements
+ SingleArcTransition<ContainerImpl, ContainerEvent> {
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ // Pause the process/process-grp if it is supported by the container
+ container.dispatcher.getEventHandler().handle(
+ new ContainersLauncherEvent(container,
+ ContainersLauncherEventType.PAUSE_CONTAINER));
+ ContainerPauseEvent pauseEvent = (ContainerPauseEvent) event;
+ container.addDiagnostics(pauseEvent.getDiagnostic(), "\n");
+ }
+ }
+
+ /**
+ * Transitions upon receiving PAUSED_CONTAINER.
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ static class PausedContainerTransition implements
+ SingleArcTransition<ContainerImpl, ContainerEvent> {
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ // Container was PAUSED so tell the scheduler
+ container.dispatcher.getEventHandler().handle(
+ new ContainerSchedulerEvent(container,
+ ContainerSchedulerEventType.CONTAINER_PAUSED));
+ }
+ }
+
+ /**
+ * Transitions upon receiving RESUME_CONTAINER.
+ * - PAUSED -> RUNNING
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ static class ResumeContainerTransition implements
+ SingleArcTransition<ContainerImpl, ContainerEvent> {
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ // Pause the process/process-grp if it is supported by the container
+ container.dispatcher.getEventHandler().handle(
+ new ContainersLauncherEvent(container,
+ ContainersLauncherEventType.RESUME_CONTAINER));
+ ContainerResumeEvent resumeEvent = (ContainerResumeEvent) event;
+ container.addDiagnostics(resumeEvent.getDiagnostic(), "\n");
+ }
+ }
+
@Override
public void handle(ContainerEvent event) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerPauseEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerPauseEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerPauseEvent.java
new file mode 100644
index 0000000..898304e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerPauseEvent.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+/**
+ * ContainerEvent for ContainerEventType.PAUSE_CONTAINER.
+ */
+public class ContainerPauseEvent extends ContainerEvent {
+
+ private final String diagnostic;
+
+ public ContainerPauseEvent(ContainerId cId,
+ String diagnostic) {
+ super(cId, ContainerEventType.PAUSE_CONTAINER);
+ this.diagnostic = diagnostic;
+ }
+
+ public String getDiagnostic() {
+ return this.diagnostic;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResumeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResumeEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResumeEvent.java
new file mode 100644
index 0000000..d7c9e9a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResumeEvent.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+/**
+ * ContainerEvent for ContainerEventType.RESUME_CONTAINER.
+ */
+public class ContainerResumeEvent extends ContainerEvent {
+
+ private final String diagnostic;
+
+ public ContainerResumeEvent(ContainerId cId,
+ String diagnostic) {
+ super(cId, ContainerEventType.RESUME_CONTAINER);
+ this.diagnostic = diagnostic;
+ }
+
+ public String getDiagnostic() {
+ return this.diagnostic;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
index 91d1356..7c3fea8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
@@ -21,5 +21,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
public enum ContainerState {
NEW, LOCALIZING, LOCALIZATION_FAILED, SCHEDULED, RUNNING, RELAUNCHING,
REINITIALIZING, EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE, KILLING,
- CONTAINER_CLEANEDUP_AFTER_KILL, CONTAINER_RESOURCES_CLEANINGUP, DONE
+ CONTAINER_CLEANEDUP_AFTER_KILL, CONTAINER_RESOURCES_CLEANINGUP, DONE,
+ PAUSING, PAUSED, RESUMING
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index a1c407f..5385874 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
@@ -85,6 +86,7 @@ import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable<Integer> {
@@ -104,8 +106,10 @@ public class ContainerLaunch implements Callable<Integer> {
private final Configuration conf;
private final Context context;
private final ContainerManagerImpl containerManager;
-
+
protected AtomicBoolean containerAlreadyLaunched = new AtomicBoolean(false);
+ protected AtomicBoolean shouldPauseContainer = new AtomicBoolean(false);
+
protected AtomicBoolean completed = new AtomicBoolean(false);
private volatile boolean killedBeforeStart = false;
@@ -742,6 +746,90 @@ public class ContainerLaunch implements Callable<Integer> {
}
/**
+ * Pause the container.
+ * Cancels the launch if the container isn't launched yet. Otherwise asks the
+ * executor to pause the container.
+ * @throws IOException in case of errors.
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ public void pauseContainer() throws IOException {
+ ContainerId containerId = container.getContainerId();
+ String containerIdStr = containerId.toString();
+ LOG.info("Pausing the container " + containerIdStr);
+
+ // The pause event is only handled if the container is in the running state
+ // (the container state machine), so we don't check for
+ // shouldLaunchContainer over here
+
+ if (!shouldPauseContainer.compareAndSet(false, true)) {
+ LOG.info("Container " + containerId + " not paused as "
+ + "resume already called");
+ return;
+ }
+
+ try {
+ // Pause the container
+ exec.pauseContainer(container);
+
+ // PauseContainer is a blocking call. We are here almost means the
+ // container is paused, so send out the event.
+ dispatcher.getEventHandler().handle(new ContainerEvent(
+ containerId,
+ ContainerEventType.CONTAINER_PAUSED));
+ } catch (Exception e) {
+ String message =
+ "Exception when trying to pause container " + containerIdStr
+ + ": " + StringUtils.stringifyException(e);
+ LOG.info(message);
+ container.handle(new ContainerKillEvent(container.getContainerId(),
+ ContainerExitStatus.PREEMPTED, "Container preempted as there was "
+ + " an exception in pausing it."));
+ }
+ }
+
+ /**
+ * Resume the container.
+ * Cancels the launch if the container isn't launched yet. Otherwise asks the
+ * executor to pause the container.
+ * @throws IOException in case of error.
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ public void resumeContainer() throws IOException {
+ ContainerId containerId = container.getContainerId();
+ String containerIdStr = containerId.toString();
+ LOG.info("Resuming the container " + containerIdStr);
+
+ // The resume event is only handled if the container is in a paused state
+ // so we don't check for the launched flag here.
+
+ // paused flag will be set to true if process already paused
+ boolean alreadyPaused = !shouldPauseContainer.compareAndSet(false, true);
+ if (!alreadyPaused) {
+ LOG.info("Container " + containerIdStr + " not paused."
+ + " No resume necessary");
+ return;
+ }
+
+ // If the container has already started
+ try {
+ exec.resumeContainer(container);
+ // ResumeContainer is a blocking call. We are here almost means the
+ // container is resumed, so send out the event.
+ dispatcher.getEventHandler().handle(new ContainerEvent(
+ containerId,
+ ContainerEventType.CONTAINER_RESUMED));
+ } catch (Exception e) {
+ String message =
+ "Exception when trying to resume container " + containerIdStr
+ + ": " + StringUtils.stringifyException(e);
+ LOG.info(message);
+ container.handle(new ContainerKillEvent(container.getContainerId(),
+ ContainerExitStatus.PREEMPTED, "Container preempted as there was "
+ + " an exception in pausing it."));
+ }
+ }
+
+ /**
* Loop through for a time-bounded interval waiting to
* read the process id from a file generated by a running process.
* @param pidFilePath File from which to read the process id
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index d4a7bfd..eb6eaf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import com.google.common.annotations.VisibleForTesting;
@@ -170,6 +172,36 @@ public class ContainersLauncher extends AbstractService
+ " with command " + signalEvent.getCommand());
}
break;
+ case PAUSE_CONTAINER:
+ ContainerLaunch launchedContainer = running.get(containerId);
+ if (launchedContainer == null) {
+ // Container not launched. So nothing needs to be done.
+ return;
+ }
+
+ // Pause the container
+ try {
+ launchedContainer.pauseContainer();
+ } catch (Exception e) {
+ LOG.info("Got exception while pausing container: " +
+ StringUtils.stringifyException(e));
+ }
+ break;
+ case RESUME_CONTAINER:
+ ContainerLaunch launchCont = running.get(containerId);
+ if (launchCont == null) {
+ // Container not launched. So nothing needs to be done.
+ return;
+ }
+
+ // Resume the container.
+ try {
+ launchCont.resumeContainer();
+ } catch (Exception e) {
+ LOG.info("Got exception while resuming container: " +
+ StringUtils.stringifyException(e));
+ }
+ break;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEventType.java
index 380a032..1054e06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEventType.java
@@ -25,4 +25,7 @@ public enum ContainersLauncherEventType {
CLEANUP_CONTAINER, // The process(grp) itself.
CLEANUP_CONTAINER_FOR_REINIT, // The process(grp) itself.
SIGNAL_CONTAINER,
+ PAUSE_CONTAINER,
+ RESUME_CONTAINER
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index 086cb9b..9ff731f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -26,4 +26,5 @@ public enum ContainerSchedulerEventType {
CONTAINER_COMPLETED,
// Producer: Node HB response - RM has asked to shed the queue
SHED_QUEUED_CONTAINERS,
+ CONTAINER_PAUSED
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1352700/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index 33f4609..8909088 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -103,6 +103,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentMatcher;
+import org.mockito.Mockito;
public class TestContainer {
@@ -207,6 +208,42 @@ public class TestContainer {
@Test
@SuppressWarnings("unchecked") // mocked generic
+ public void testContainerPauseAndResume() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(13, 314159265358979L, 4344, "yak");
+ wc.initContainer();
+ wc.localizeResources();
+ int running = metrics.getRunningContainers();
+ wc.launchContainer();
+ assertEquals(running + 1, metrics.getRunningContainers());
+ reset(wc.localizerBus);
+ wc.pauseContainer();
+ assertEquals(ContainerState.PAUSED,
+ wc.c.getContainerState());
+ wc.resumeContainer();
+ assertEquals(ContainerState.RUNNING,
+ wc.c.getContainerState());
+ wc.containerKilledOnRequest();
+ assertEquals(ContainerState.EXITED_WITH_FAILURE,
+ wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyCleanupCall(wc);
+ int failed = metrics.getFailedContainers();
+ wc.containerResourcesCleanup();
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ assertEquals(failed + 1, metrics.getFailedContainers());
+ assertEquals(running, metrics.getRunningContainers());
+ }
+ finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked") // mocked generic
public void testCleanupOnFailure() throws Exception {
WrappedContainer wc = null;
try {
@@ -955,6 +992,8 @@ public class TestContainer {
NodeStatusUpdater nodeStatusUpdater = mock(NodeStatusUpdater.class);
when(context.getNodeStatusUpdater()).thenReturn(nodeStatusUpdater);
ContainerExecutor executor = mock(ContainerExecutor.class);
+ Mockito.doNothing().when(executor).pauseContainer(any(Container.class));
+ Mockito.doNothing().when(executor).resumeContainer(any(Container.class));
launcher =
new ContainersLauncher(context, dispatcher, executor, null, null);
// create a mock ExecutorService, which will not really launch
@@ -1143,6 +1182,18 @@ public class TestContainer {
drainDispatcherEvents();
}
+ public void pauseContainer() {
+ c.handle(new ContainerPauseEvent(cId,
+ "PauseRequest"));
+ drainDispatcherEvents();
+ }
+
+ public void resumeContainer() {
+ c.handle(new ContainerResumeEvent(cId,
+ "ResumeRequest"));
+ drainDispatcherEvents();
+ }
+
public void containerKilledOnRequest() {
int exitCode = ContainerExitStatus.KILLED_BY_RESOURCEMANAGER;
String diagnosticMsg = "Container completed with exit code " + exitCode;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] hadoop git commit: MAPREDUCE-6753. Variable in byte printed
directly in mapreduce client. Contributed by Kai Sasaki.
Posted by kk...@apache.org.
MAPREDUCE-6753. Variable in byte printed directly in mapreduce client. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19a1fc63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19a1fc63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19a1fc63
Branch: refs/heads/YARN-5972
Commit: 19a1fc6373f0dea56e5845f19990d469428489bc
Parents: 3749152
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Mar 3 16:11:10 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Mar 3 16:11:10 2017 +0900
----------------------------------------------------------------------
.../org/apache/hadoop/mapreduce/security/CredentialsTestJob.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19a1fc63/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
index 2141080..e66fb2f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
@@ -58,7 +58,6 @@ public class CredentialsTestJob extends Configured implements Tool {
String secretName = "alias"+i;
// get token storage and a key
byte[] secretValue = ts.getSecretKey(new Text(secretName));
- System.out.println(secretValue);
if (secretValue == null){
throw new RuntimeException("The key "+ secretName + " is not available. ");
@@ -66,6 +65,7 @@ public class CredentialsTestJob extends Configured implements Tool {
}
String secretValueStr = new String (secretValue);
+ System.out.println(secretValueStr);
if ( !("password"+i).equals(secretValueStr)){
throw new RuntimeException("The key "+ secretName +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] hadoop git commit: HDFS-11428. Change setErasureCodingPolicy
to take a required string EC policy name. Contributed by Andrew Wang.
Posted by kk...@apache.org.
HDFS-11428. Change setErasureCodingPolicy to take a required string EC policy name. Contributed by Andrew Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82ef9acc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82ef9acc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82ef9acc
Branch: refs/heads/YARN-5972
Commit: 82ef9accafe7318278efb169678e17065e082c8e
Parents: dcd03df
Author: Rakesh Radhakrishnan <ra...@apache.org>
Authored: Wed Mar 1 16:06:02 2017 +0530
Committer: Rakesh Radhakrishnan <ra...@apache.org>
Committed: Wed Mar 1 16:06:02 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +--
.../hadoop/hdfs/DistributedFileSystem.java | 11 +++---
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 10 +++---
.../hadoop/hdfs/protocol/ClientProtocol.java | 5 ++-
.../ClientNamenodeProtocolTranslatorPB.java | 6 ++--
.../src/main/proto/erasurecoding.proto | 2 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 4 +--
.../server/namenode/FSDirErasureCodingOp.java | 13 +++++--
.../hdfs/server/namenode/FSNamesystem.java | 9 ++---
.../hdfs/server/namenode/NameNodeRpcServer.java | 4 +--
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 26 +-------------
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 6 ++--
.../hdfs/ErasureCodeBenchmarkThroughput.java | 3 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 3 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 3 +-
.../TestDFSStripedOutputStreamWithFailure.java | 2 +-
.../hdfs/TestDecommissionWithStriped.java | 3 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 37 +++++++++++---------
.../TestErasureCodingPolicyWithSnapshot.java | 10 +++---
.../apache/hadoop/hdfs/TestFileChecksum.java | 3 +-
.../hadoop/hdfs/TestFileStatusWithECPolicy.java | 2 +-
.../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +-
.../hdfs/TestReadStripedFileWithDecoding.java | 3 +-
.../TestReadStripedFileWithMissingBlocks.java | 2 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 3 +-
.../hdfs/TestSafeModeWithStripedFile.java | 3 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 18 +++++-----
.../hadoop/hdfs/TestWriteReadStripedFile.java | 3 +-
.../hdfs/TestWriteStripedFileWithFailure.java | 3 +-
.../hdfs/server/balancer/TestBalancer.java | 3 +-
.../TestBlockTokenWithDFSStriped.java | 4 +--
...constructStripedBlocksWithRackAwareness.java | 9 +++--
.../TestSequentialBlockGroupId.java | 4 +--
.../TestDataNodeErasureCodingMetrics.java | 3 +-
.../hadoop/hdfs/server/mover/TestMover.java | 3 +-
.../TestAddOverReplicatedStripedBlocks.java | 3 +-
.../namenode/TestAddStripedBlockInFBR.java | 3 +-
.../server/namenode/TestAddStripedBlocks.java | 3 +-
.../server/namenode/TestFSEditLogLoader.java | 4 +--
.../hdfs/server/namenode/TestFSImage.java | 6 ++--
.../hadoop/hdfs/server/namenode/TestFsck.java | 8 +++--
.../server/namenode/TestNameNodeMXBean.java | 3 +-
.../namenode/TestQuotaWithStripedBlocks.java | 3 +-
.../namenode/TestReconstructStripedBlocks.java | 6 ++--
.../server/namenode/TestStripedINodeFile.java | 6 ++--
.../TestOfflineImageViewer.java | 3 +-
...TestOfflineImageViewerWithStripedBlocks.java | 3 +-
.../test/resources/testErasureCodingConf.xml | 2 +-
.../java/org/apache/hadoop/fs/TestDFSIO.java | 2 +-
49 files changed, 152 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index e0ccd62..ae1d821 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2611,12 +2611,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
- public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
+ public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
checkOpen();
try (TraceScope ignored =
newPathTraceScope("setErasureCodingPolicy", src)) {
- namenode.setErasureCodingPolicy(src, ecPolicy);
+ namenode.setErasureCodingPolicy(src, ecPolicyName);
} catch (RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
SafeModeException.class,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 30dcfa4..1eef560 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2397,17 +2397,18 @@ public class DistributedFileSystem extends FileSystem {
* Set the source path to the specified erasure coding policy.
*
* @param path The directory to set the policy
- * @param ecPolicy The erasure coding policy. If not specified default will
- * be used.
+ * @param ecPolicyName The erasure coding policy name.
* @throws IOException
*/
public void setErasureCodingPolicy(final Path path,
- final ErasureCodingPolicy ecPolicy) throws IOException {
+ final String ecPolicyName) throws IOException {
Path absF = fixRelativePart(path);
+ Preconditions.checkNotNull(ecPolicyName, "Erasure coding policy cannot be" +
+ " null.");
new FileSystemLinkResolver<Void>() {
@Override
public Void doCall(final Path p) throws IOException {
- dfs.setErasureCodingPolicy(getPathName(p), ecPolicy);
+ dfs.setErasureCodingPolicy(getPathName(p), ecPolicyName);
return null;
}
@@ -2415,7 +2416,7 @@ public class DistributedFileSystem extends FileSystem {
public Void next(final FileSystem fs, final Path p) throws IOException {
if (fs instanceof DistributedFileSystem) {
DistributedFileSystem myDfs = (DistributedFileSystem) fs;
- myDfs.setErasureCodingPolicy(p, ecPolicy);
+ myDfs.setErasureCodingPolicy(p, ecPolicyName);
return null;
}
throw new UnsupportedOperationException(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index a192fa8..456f280 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -465,13 +465,15 @@ public class HdfsAdmin {
* Set the source path to the specified erasure coding policy.
*
* @param path The source path referring to a directory.
- * @param ecPolicy The erasure coding policy for the directory.
- * If null, the default will be used.
+ * @param ecPolicyName The erasure coding policy name for the directory.
+ *
* @throws IOException
+ * @throws HadoopIllegalArgumentException if the specified EC policy is not
+ * enabled on the cluster
*/
public void setErasureCodingPolicy(final Path path,
- final ErasureCodingPolicy ecPolicy) throws IOException {
- dfs.setErasureCodingPolicy(path, ecPolicy);
+ final String ecPolicyName) throws IOException {
+ dfs.setErasureCodingPolicy(path, ecPolicyName);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index eaebd6f..041d226 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1510,11 +1510,10 @@ public interface ClientProtocol {
/**
* Set an erasure coding policy on a specified path.
* @param src The path to set policy on.
- * @param ecPolicy The erasure coding policy. If null, default policy will
- * be used
+ * @param ecPolicyName The erasure coding policy name.
*/
@AtMostOnce
- void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
+ void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index de474b5..da4a17f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -1459,14 +1459,12 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
- public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
+ public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
final SetErasureCodingPolicyRequestProto.Builder builder =
SetErasureCodingPolicyRequestProto.newBuilder();
builder.setSrc(src);
- if (ecPolicy != null) {
- builder.setEcPolicy(PBHelperClient.convertErasureCodingPolicy(ecPolicy));
- }
+ builder.setEcPolicyName(ecPolicyName);
SetErasureCodingPolicyRequestProto req = builder.build();
try {
rpcProxy.setErasureCodingPolicy(null, req);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
index 7ea8d32..4eab4d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
@@ -25,7 +25,7 @@ import "hdfs.proto";
message SetErasureCodingPolicyRequestProto {
required string src = 1;
- optional ErasureCodingPolicyProto ecPolicy = 2;
+ required string ecPolicyName = 2;
}
message SetErasureCodingPolicyResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 1aa15c8..1944fe7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -1442,9 +1442,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RpcController controller, SetErasureCodingPolicyRequestProto req)
throws ServiceException {
try {
- ErasureCodingPolicy ecPolicy = req.hasEcPolicy() ?
- PBHelperClient.convertErasureCodingPolicy(req.getEcPolicy()) : null;
- server.setErasureCodingPolicy(req.getSrc(), ecPolicy);
+ server.setErasureCodingPolicy(req.getSrc(), req.getEcPolicyName());
return SetErasureCodingPolicyResponseProto.newBuilder().build();
} catch (IOException e) {
throw new ServiceException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 0ab8c89..f4c4ad7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -59,14 +59,16 @@ final class FSDirErasureCodingOp {
*
* @param fsn The namespace
* @param srcArg The path of the target directory.
- * @param ecPolicy The erasure coding policy to set on the target directory.
+ * @param ecPolicyName The erasure coding policy name to set on the target
+ * directory.
* @param logRetryCache whether to record RPC ids in editlog for retry
* cache rebuilding
* @return {@link HdfsFileStatus}
* @throws IOException
+ * @throws HadoopIllegalArgumentException if the policy is not enabled
*/
static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn,
- final String srcArg, final ErasureCodingPolicy ecPolicy,
+ final String srcArg, final String ecPolicyName,
final boolean logRetryCache) throws IOException {
assert fsn.hasWriteLock();
@@ -78,6 +80,13 @@ final class FSDirErasureCodingOp {
List<XAttr> xAttrs;
fsd.writeLock();
try {
+ ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicyManager()
+ .getPolicyByName(ecPolicyName);
+ if (ecPolicy == null) {
+ throw new HadoopIllegalArgumentException("Policy '" +
+ ecPolicyName + "' does not match any supported erasure coding " +
+ "policies.");
+ }
iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
src = iip.getPath();
xAttrs = setErasureCodingPolicyXAttr(fsn, iip, ecPolicy);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 38a326c..254ccfa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -6772,13 +6772,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Set an erasure coding policy on the given path.
* @param srcArg The path of the target directory.
- * @param ecPolicy The erasure coding policy to set on the target directory.
+ * @param ecPolicyName The erasure coding policy to set on the target
+ * directory.
* @throws AccessControlException if the caller is not the superuser.
* @throws UnresolvedLinkException if the path can't be resolved.
* @throws SafeModeException if the Namenode is in safe mode.
*/
- void setErasureCodingPolicy(final String srcArg, final ErasureCodingPolicy
- ecPolicy, final boolean logRetryCache) throws IOException,
+ void setErasureCodingPolicy(final String srcArg, final String ecPolicyName,
+ final boolean logRetryCache) throws IOException,
UnresolvedLinkException, SafeModeException, AccessControlException {
final String operationName = "setErasureCodingPolicy";
checkSuperuserPrivilege();
@@ -6790,7 +6791,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg);
resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this,
- srcArg, ecPolicy, logRetryCache);
+ srcArg, ecPolicyName, logRetryCache);
success = true;
} catch (AccessControlException ace) {
logAuditEvent(success, operationName, srcArg, null,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index f9cfa42..cf14e8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -2018,7 +2018,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
}
@Override // ClientProtocol
- public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
+ public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
checkNNStartup();
final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
@@ -2027,7 +2027,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
}
boolean success = false;
try {
- namesystem.setErasureCodingPolicy(src, ecPolicy, cacheEntry != null);
+ namesystem.setErasureCodingPolicy(src, ecPolicyName, cacheEntry != null);
success = true;
} finally {
RetryCache.setState(cacheEntry, success);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 29c65b1..548f754 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
@@ -227,30 +226,7 @@ public class ECAdmin extends Configured implements Tool {
final Path p = new Path(path);
final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
try {
- ErasureCodingPolicy ecPolicy = null;
- ErasureCodingPolicy[] ecPolicies =
- dfs.getClient().getErasureCodingPolicies();
- for (ErasureCodingPolicy policy : ecPolicies) {
- if (ecPolicyName.equals(policy.getName())) {
- ecPolicy = policy;
- break;
- }
- }
- if (ecPolicy == null) {
- StringBuilder sb = new StringBuilder();
- sb.append("Policy '");
- sb.append(ecPolicyName);
- sb.append("' does not match any of the supported policies.");
- sb.append(" Please select any one of ");
- List<String> ecPolicyNames = new ArrayList<String>();
- for (ErasureCodingPolicy policy : ecPolicies) {
- ecPolicyNames.add(policy.getName());
- }
- sb.append(ecPolicyNames);
- System.err.println(sb.toString());
- return 3;
- }
- dfs.setErasureCodingPolicy(p, ecPolicy);
+ dfs.setErasureCodingPolicy(p, ecPolicyName);
System.out.println("Set erasure coding policy " + ecPolicyName +
" on " + path);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 1fbc1d9..60a1a4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -136,6 +136,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -1898,7 +1899,7 @@ public class DFSTestUtil {
Path dir, int numBlocks, int numStripesPerBlk, boolean toMkdir)
throws Exception {
createStripedFile(cluster, file, dir, numBlocks, numStripesPerBlk,
- toMkdir, null);
+ toMkdir, ErasureCodingPolicyManager.getSystemDefaultPolicy());
}
/**
@@ -1922,7 +1923,8 @@ public class DFSTestUtil {
assert dir != null;
dfs.mkdirs(dir);
try {
- dfs.getClient().setErasureCodingPolicy(dir.toString(), ecPolicy);
+ dfs.getClient()
+ .setErasureCodingPolicy(dir.toString(), ecPolicy.getName());
} catch (IOException e) {
if (!e.getMessage().contains("non-empty directory")) {
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
index 13dc997..d1a7569 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
@@ -192,7 +192,8 @@ public class ErasureCodeBenchmarkThroughput
}
if (!dfs.exists(ecPath)) {
dfs.mkdirs(ecPath);
- dfs.getClient().setErasureCodingPolicy(ecPath.toString(), ecPolicy);
+ dfs.getClient()
+ .setErasureCodingPolicy(ecPath.toString(), ecPolicy.getName());
} else {
Preconditions.checkArgument(
dfs.getClient().
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 9b69904..50b366b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -108,7 +108,8 @@ public class TestDFSStripedInputStream {
}
fs = cluster.getFileSystem();
fs.mkdirs(dirPath);
- fs.getClient().setErasureCodingPolicy(dirPath.toString(), ecPolicy);
+ fs.getClient()
+ .setErasureCodingPolicy(dirPath.toString(), ecPolicy.getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index 8834e14..5956cc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -89,7 +89,8 @@ public class TestDFSStripedOutputStream {
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy
+ .getName());
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index fabd417..01bd3c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -221,7 +221,7 @@ public class TestDFSStripedOutputStreamWithFailure {
cluster.waitActive();
dfs = cluster.getFileSystem();
dfs.mkdirs(dir);
- dfs.setErasureCodingPolicy(dir, ecPolicy);
+ dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
}
private void tearDown() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
index 2eab309..f8ed0c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
@@ -142,7 +142,8 @@ public class TestDecommissionWithStriped {
client = getDfsClient(cluster.getNameNode(0), conf);
dfs.mkdirs(ecDir);
- dfs.setErasureCodingPolicy(ecDir, null);
+ dfs.setErasureCodingPolicy(ecDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 0e431fa..8608626 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -47,6 +47,8 @@ public class TestErasureCodingPolicies {
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private static final int BLOCK_SIZE = 1024;
+ private static final ErasureCodingPolicy EC_POLICY =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
private FSNamesystem namesystem;
@Before
@@ -80,7 +82,8 @@ public class TestErasureCodingPolicies {
DFSTestUtil.createFile(fs, replicatedFile, 0, (short) 3, 0L);
// set ec policy on dir
- fs.setErasureCodingPolicy(dir, null);
+ fs.setErasureCodingPolicy(dir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// create a file which should be using ec
final Path ecSubDir = new Path(dir, "ecSubDir");
final Path ecFile = new Path(ecSubDir, "ecFile");
@@ -132,7 +135,7 @@ public class TestErasureCodingPolicies {
fs.mkdir(testDir, FsPermission.getDirDefault());
/* Normal creation of an erasure coding directory */
- fs.getClient().setErasureCodingPolicy(testDir.toString(), null);
+ fs.setErasureCodingPolicy(testDir, EC_POLICY.getName());
/* Verify files under the directory are striped */
final Path ECFilePath = new Path(testDir, "foo");
@@ -148,7 +151,7 @@ public class TestErasureCodingPolicies {
fs.mkdir(notEmpty, FsPermission.getDirDefault());
final Path oldFile = new Path(notEmpty, "old");
fs.create(oldFile);
- fs.getClient().setErasureCodingPolicy(notEmpty.toString(), null);
+ fs.setErasureCodingPolicy(notEmpty, EC_POLICY.getName());
final Path newFile = new Path(notEmpty, "new");
fs.create(newFile);
INode oldInode = namesystem.getFSDirectory().getINode(oldFile.toString());
@@ -160,10 +163,10 @@ public class TestErasureCodingPolicies {
final Path dir1 = new Path("/dir1");
final Path dir2 = new Path(dir1, "dir2");
fs.mkdir(dir1, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(dir1.toString(), null);
+ fs.setErasureCodingPolicy(dir1, EC_POLICY.getName());
fs.mkdir(dir2, FsPermission.getDirDefault());
try {
- fs.getClient().setErasureCodingPolicy(dir2.toString(), null);
+ fs.setErasureCodingPolicy(dir2, EC_POLICY.getName());
} catch (IOException e) {
fail("Nested erasure coding policies are supported");
}
@@ -172,7 +175,7 @@ public class TestErasureCodingPolicies {
final Path fPath = new Path("/file");
fs.create(fPath);
try {
- fs.getClient().setErasureCodingPolicy(fPath.toString(), null);
+ fs.setErasureCodingPolicy(fPath, EC_POLICY.getName());
fail("Erasure coding policy on file");
} catch (IOException e) {
assertExceptionContains("erasure coding policy for a file", e);
@@ -185,8 +188,8 @@ public class TestErasureCodingPolicies {
final Path dstECDir = new Path("/dstEC");
fs.mkdir(srcECDir, FsPermission.getDirDefault());
fs.mkdir(dstECDir, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(srcECDir.toString(), null);
- fs.getClient().setErasureCodingPolicy(dstECDir.toString(), null);
+ fs.setErasureCodingPolicy(srcECDir, EC_POLICY.getName());
+ fs.setErasureCodingPolicy(dstECDir, EC_POLICY.getName());
final Path srcFile = new Path(srcECDir, "foo");
fs.create(srcFile);
@@ -220,7 +223,8 @@ public class TestErasureCodingPolicies {
public void testReplication() throws IOException {
final Path testDir = new Path("/ec");
fs.mkdir(testDir, FsPermission.getDirDefault());
- fs.setErasureCodingPolicy(testDir, null);
+ fs.setErasureCodingPolicy(testDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
final Path fooFile = new Path(testDir, "foo");
// create ec file with replication=0
fs.create(fooFile, FsPermission.getFileDefault(), true,
@@ -241,8 +245,9 @@ public class TestErasureCodingPolicies {
// dir EC policy should be null
assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
// dir EC policy after setting
- fs.getClient().setErasureCodingPolicy(src, null); //Default one will be used.
- ErasureCodingPolicy sysDefaultECPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ ErasureCodingPolicy sysDefaultECPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ fs.getClient().setErasureCodingPolicy(src, sysDefaultECPolicy.getName());
verifyErasureCodingInfo(src, sysDefaultECPolicy);
fs.create(new Path(ecDir, "child1")).close();
// verify for the files in ec dir
@@ -263,7 +268,7 @@ public class TestErasureCodingPolicies {
// dir ECInfo before being set
assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
// dir ECInfo after set
- fs.getClient().setErasureCodingPolicy(src, usingECPolicy);
+ fs.getClient().setErasureCodingPolicy(src, usingECPolicy.getName());
verifyErasureCodingInfo(src, usingECPolicy);
fs.create(new Path(ecDir, "child1")).close();
// verify for the files in ec dir
@@ -291,12 +296,12 @@ public class TestErasureCodingPolicies {
final Path ecDir = new Path(src);
try {
fs.mkdir(ecDir, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(src, ecPolicy);
+ fs.getClient().setErasureCodingPolicy(src, ecPolicy.getName());
fail("HadoopIllegalArgumentException should be thrown for"
+ "setting an invalid erasure coding policy");
} catch (Exception e) {
- assertExceptionContains("Policy [ RS-4-2-128k ] does not match " +
- "any of the supported policies",e);
+ assertExceptionContains("Policy 'RS-4-2-128k' does not match " +
+ "any supported erasure coding policies",e);
}
}
@@ -338,7 +343,7 @@ public class TestErasureCodingPolicies {
for (ErasureCodingPolicy policy : sysPolicies) {
Path dir = new Path("/policy_" + policy.getId());
fs.mkdir(dir, FsPermission.getDefault());
- fs.setErasureCodingPolicy(dir, policy);
+ fs.setErasureCodingPolicy(dir, policy.getName());
Path file = new Path(dir, "child");
fs.create(file).close();
assertEquals(policy, fs.getErasureCodingPolicy(file));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
index e02c0bf..6cf4ef4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
@@ -75,7 +75,7 @@ public class TestErasureCodingPolicyWithSnapshot {
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDirParent);
// set erasure coding policy
- fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
String contents = DFSTestUtil.readFile(fs, ecFile);
final Path snap1 = fs.createSnapshot(ecDirParent, "snap1");
@@ -93,7 +93,7 @@ public class TestErasureCodingPolicyWithSnapshot {
fs.getErasureCodingPolicy(snap2ECDir));
// Make dir again with system default ec policy
- fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
final Path snap3 = fs.createSnapshot(ecDirParent, "snap3");
final Path snap3ECDir = new Path(snap3, ecDir.getName());
// Check that snap3's ECPolicy has the correct settings
@@ -134,7 +134,7 @@ public class TestErasureCodingPolicyWithSnapshot {
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDir);
- fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap1));
@@ -150,7 +150,7 @@ public class TestErasureCodingPolicyWithSnapshot {
fs.allowSnapshot(ecDir);
// set erasure coding policy
- fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1);
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
@@ -182,7 +182,7 @@ public class TestErasureCodingPolicyWithSnapshot {
fs.allowSnapshot(ecDir);
// set erasure coding policy
- fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
index 9d60798..8ae176f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
@@ -81,7 +81,8 @@ public class TestFileChecksum {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
Path ecPath = new Path(ecDir);
cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault());
- cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir, null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
client = fs.getClient();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
index 2210fbe..d7b7327 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
@@ -73,7 +73,7 @@ public class TestFileStatusWithECPolicy {
final ErasureCodingPolicy ecPolicy1 = ErasureCodingPolicyManager.getSystemDefaultPolicy();
// set EC policy on dir
- fs.setErasureCodingPolicy(dir, ecPolicy1);
+ fs.setErasureCodingPolicy(dir, ecPolicy1.getName());
final ErasureCodingPolicy ecPolicy2 = client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy();
assertNotNull(ecPolicy2);
assertTrue(ecPolicy1.equals(ecPolicy2));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index 246818b..710ff62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -94,7 +94,7 @@ public class TestLeaseRecoveryStriped {
cluster.waitActive();
dfs = cluster.getFileSystem();
dfs.mkdirs(dir);
- dfs.setErasureCodingPolicy(dir, ecPolicy);
+ dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
index 506a005..d7d9cc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
@@ -102,7 +102,8 @@ public class TestReadStripedFileWithDecoding {
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
index 2ee8663..b65626e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
@@ -61,7 +61,7 @@ public class TestReadStripedFileWithMissingBlocks {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy(
- "/", ecPolicy);
+ "/", ecPolicy.getName());
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index 08b53c2..38939f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -107,7 +107,8 @@ public class TestReconstructStripedFile {
cluster.waitActive();
fs = cluster.getFileSystem();
- fs.getClient().setErasureCodingPolicy("/", null);
+ fs.getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
List<DataNode> datanodes = cluster.getDataNodes();
for (int i = 0; i < dnNum; i++) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
index 2d37c06..0731779 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
@@ -63,7 +63,8 @@ public class TestSafeModeWithStripedFile {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
cluster.waitActive();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
index f26aa06..86624cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -101,7 +101,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
// Test unset a directory which has no EC policy
fs.unsetErasureCodingPolicy(dirPath);
// Set EC policy on directory
- fs.setErasureCodingPolicy(dirPath, ecPolicy);
+ fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
fs.unsetErasureCodingPolicy(dirPath);
@@ -142,12 +142,12 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
.getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID);
fs.mkdirs(parentDir);
- fs.setErasureCodingPolicy(parentDir, ecPolicy);
+ fs.setErasureCodingPolicy(parentDir, ecPolicy.getName());
fs.mkdirs(childDir);
// Create RS(6,3) EC policy file
DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L);
// Set RS(3,2) EC policy on child directory
- fs.setErasureCodingPolicy(childDir, ec32Policy);
+ fs.setErasureCodingPolicy(childDir, ec32Policy.getName());
// Create RS(3,2) EC policy file
DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L);
@@ -202,7 +202,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
// Test unset root path which has no EC policy
fs.unsetErasureCodingPolicy(rootPath);
// Set EC policy on root path
- fs.setErasureCodingPolicy(rootPath, ecPolicy);
+ fs.setErasureCodingPolicy(rootPath, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
fs.unsetErasureCodingPolicy(rootPath);
DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 1, 0L);
@@ -240,11 +240,11 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
.getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID);
fs.unsetErasureCodingPolicy(rootPath);
- fs.setErasureCodingPolicy(rootPath, ecPolicy);
+ fs.setErasureCodingPolicy(rootPath, ecPolicy.getName());
// Create RS(6,3) EC policy file
DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L);
// Change EC policy from RS(6,3) to RS(3,2)
- fs.setErasureCodingPolicy(rootPath, ec32Policy);
+ fs.setErasureCodingPolicy(rootPath, ec32Policy.getName());
DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L);
// start to check
@@ -281,7 +281,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
final Path replicateFilePath2 = new Path(ecDirPath, "rep_file2");
fs.mkdirs(ecDirPath);
- fs.setErasureCodingPolicy(ecDirPath, ecPolicy);
+ fs.setErasureCodingPolicy(ecDirPath, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
fs.unsetErasureCodingPolicy(ecDirPath);
DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 3, 0L);
@@ -328,7 +328,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
// Set EC policy on non-existent directory
try {
- fs.setErasureCodingPolicy(dirPath, ecPolicy);
+ fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
fail("FileNotFoundException should be thrown for a non-existent"
+ " file path");
} catch (FileNotFoundException e) {
@@ -347,7 +347,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
// Set EC policy on file
try {
- fs.setErasureCodingPolicy(ecFilePath, ecPolicy);
+ fs.setErasureCodingPolicy(ecFilePath, ecPolicy.getName());
fail("IOException should be thrown for setting EC policy on file");
} catch (IOException e) {
assertExceptionContains("Attempt to set an erasure coding policy " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
index 712f326..76ca704 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
@@ -80,7 +80,8 @@ public class TestWriteReadStripedFile {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
fs = cluster.getFileSystem();
fs.mkdirs(new Path("/ec"));
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
index b89e169..23a9821 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
@@ -59,7 +59,8 @@ public class TestWriteStripedFileWithFailure {
public void setup() throws IOException {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 1906d74..70aa4e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -1940,7 +1940,8 @@ public class TestBalancer {
cluster.waitActive();
client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
ClientProtocol.class).getProxy();
- client.setErasureCodingPolicy("/", null);
+ client.setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
long totalCapacity = sum(capacities);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
index 834df6f..20c3acc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
@@ -83,8 +83,8 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS {
.nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
.numDataNodes(numDNs)
.build();
- cluster.getFileSystem().getClient()
- .setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
try {
cluster.waitActive();
doTestRead(conf, cluster, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
index ace8732..832876e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
@@ -150,7 +150,8 @@ public class TestReconstructStripedBlocksWithRackAwareness {
.numDataNodes(hosts.length).build();
cluster.waitActive();
fs = cluster.getFileSystem();
- fs.setErasureCodingPolicy(new Path("/"), null);
+ fs.setErasureCodingPolicy(new Path("/"),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
FSNamesystem fsn = cluster.getNamesystem();
BlockManager bm = fsn.getBlockManager();
@@ -220,7 +221,8 @@ public class TestReconstructStripedBlocksWithRackAwareness {
.numDataNodes(hosts.length).build();
cluster.waitActive();
fs = cluster.getFileSystem();
- fs.setErasureCodingPolicy(new Path("/"), null);
+ fs.setErasureCodingPolicy(new Path("/"),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
MiniDFSCluster.DataNodeProperties lastHost = stopDataNode(
hosts[hosts.length - 1]);
@@ -273,7 +275,8 @@ public class TestReconstructStripedBlocksWithRackAwareness {
.numDataNodes(hostNames.length).build();
cluster.waitActive();
fs = cluster.getFileSystem();
- fs.setErasureCodingPolicy(new Path("/"), null);
+ fs.setErasureCodingPolicy(new Path("/"),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
final BlockManager bm = cluster.getNamesystem().getBlockManager();
final DatanodeManager dm = bm.getDatanodeManager();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
index 10ea01f..7920c59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
@@ -88,8 +88,8 @@ public class TestSequentialBlockGroupId {
blockGrpIdGenerator = cluster.getNamesystem().getBlockManager()
.getBlockIdManager().getBlockGroupIdGenerator();
fs.mkdirs(ecDir);
- cluster.getFileSystem().getClient()
- .setErasureCodingPolicy("/ecDir", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/ecDir",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
index 7e64214..7036c7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
@@ -75,7 +75,8 @@ public class TestDataNodeErasureCodingMetrics {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index d565548..a403ff4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -537,7 +537,8 @@ public class TestMover {
client.setStoragePolicy(barDir,
HdfsConstants.HOT_STORAGE_POLICY_NAME);
// set an EC policy on "/bar" directory
- client.setErasureCodingPolicy(barDir, null);
+ client.setErasureCodingPolicy(barDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// write file to barDir
final String fooFile = "/bar/foo";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
index c8b25564..670efd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
@@ -81,7 +81,8 @@ public class TestAddOverReplicatedStripedBlocks {
cluster.waitActive();
fs = cluster.getFileSystem();
fs.mkdirs(dirPath);
- fs.getClient().setErasureCodingPolicy(dirPath.toString(), null);
+ fs.getClient().setErasureCodingPolicy(dirPath.toString(),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
index 6e02372..87fbcc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
@@ -87,7 +87,8 @@ public class TestAddStripedBlockInFBR {
final Path repDir = new Path("/rep");
dfs.mkdirs(ecDir);
dfs.mkdirs(repDir);
- dfs.getClient().setErasureCodingPolicy(ecDir.toString(), null);
+ dfs.getClient().setErasureCodingPolicy(ecDir.toString(),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// create several non-EC files and one EC file
final Path[] repFiles = new Path[groupSize];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
index ffd3fa7..2df1aa4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
@@ -86,7 +86,8 @@ public class TestAddStripedBlocks {
.numDataNodes(groupSize).build();
cluster.waitActive();
dfs = cluster.getFileSystem();
- dfs.getClient().setErasureCodingPolicy("/", null);
+ dfs.getClient().setErasureCodingPolicy("/", ErasureCodingPolicyManager
+ .getSystemDefaultPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
index a57a936..72d76b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
@@ -479,7 +479,7 @@ public class TestFSEditLogLoader {
//set the storage policy of the directory
fs.mkdir(new Path(testDir), new FsPermission("755"));
fs.getClient().getNamenode().setErasureCodingPolicy(
- testDir, testECPolicy);
+ testDir, testECPolicy.getName());
// Create a file with striped block
Path p = new Path(testFilePath);
@@ -552,7 +552,7 @@ public class TestFSEditLogLoader {
//set the storage policy of the directory
fs.mkdir(new Path(testDir), new FsPermission("755"));
fs.getClient().getNamenode().setErasureCodingPolicy(
- testDir, testECPolicy);
+ testDir, testECPolicy.getName());
//create a file with striped blocks
Path p = new Path(testFilePath);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 179e297..ae15491 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -156,7 +156,7 @@ public class TestFSImage {
private void testSaveAndLoadStripedINodeFile(FSNamesystem fsn, Configuration conf,
boolean isUC) throws IOException{
// Construct an INode with StripedBlock for saving and loading
- fsn.setErasureCodingPolicy("/", testECPolicy, false);
+ fsn.setErasureCodingPolicy("/", testECPolicy.getName(), false);
long id = 123456789;
byte[] name = "testSaveAndLoadInodeFile_testfile".getBytes();
PermissionStatus permissionStatus = new PermissionStatus("testuser_a",
@@ -472,8 +472,8 @@ public class TestFSImage {
// Create directories and files
fs.mkdirs(parentDir);
fs.mkdirs(childDir);
- fs.setErasureCodingPolicy(parentDir, testECPolicy);
- fs.setErasureCodingPolicy(childDir, ec32Policy);
+ fs.setErasureCodingPolicy(parentDir, testECPolicy.getName());
+ fs.setErasureCodingPolicy(childDir, ec32Policy.getName());
Path file_10_4 = new Path(parentDir, "striped_file_10_4");
Path file_3_2 = new Path(childDir, "striped_file_3_2");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index c02e880..76c5378 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -690,7 +690,7 @@ public class TestFsck {
util.createFiles(fs, topDir);
// set topDir to EC when it has replicated files
cluster.getFileSystem().getClient().setErasureCodingPolicy(
- topDir, ecPolicy);
+ topDir, ecPolicy.getName());
// create a new file under topDir
DFSTestUtil.createFile(fs, new Path(topDir, "ecFile"), 1024, (short) 1, 0L);
@@ -2307,7 +2307,8 @@ public class TestFsck {
// create file
Path ecDirPath = new Path("/striped");
fs.mkdir(ecDirPath, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null);
+ fs.getClient().setErasureCodingPolicy(ecDirPath.toString(),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
Path file = new Path(ecDirPath, "corrupted");
final int length = cellSize * dataBlocks;
final byte[] bytes = StripedFileTestUtil.generateBytes(length);
@@ -2372,7 +2373,8 @@ public class TestFsck {
// create file
Path ecDirPath = new Path("/striped");
fs.mkdir(ecDirPath, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null);
+ fs.getClient().setErasureCodingPolicy(ecDirPath.toString(),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
Path file = new Path(ecDirPath, "missing");
final int length = cellSize * dataBlocks;
final byte[] bytes = StripedFileTestUtil.generateBytes(length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index ef1e6e4..037fd40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -738,7 +738,8 @@ public class TestNameNodeMXBean {
// create file
Path ecDirPath = new Path("/striped");
fs.mkdir(ecDirPath, FsPermission.getDirDefault());
- fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null);
+ fs.getClient().setErasureCodingPolicy(ecDirPath.toString(),
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
Path file = new Path(ecDirPath, "corrupted");
final int length = cellSize * dataBlocks;
final byte[] bytes = StripedFileTestUtil.generateBytes(length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
index 9ec34d2..326ddc8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
@@ -71,7 +71,8 @@ public class TestQuotaWithStripedBlocks {
dfs = cluster.getFileSystem();
dfs.mkdirs(ecDir);
- dfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy);
+ dfs.getClient()
+ .setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName());
dfs.setQuota(ecDir, Long.MAX_VALUE - 1, DISK_QUOTA);
dfs.setQuotaByStorageType(ecDir, StorageType.DISK, DISK_QUOTA);
dfs.setStoragePolicy(ecDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
index cf1585c..5e4a9db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
@@ -201,7 +201,8 @@ public class TestReconstructStripedBlocks {
cluster.waitActive();
DistributedFileSystem fs = cluster.getFileSystem();
BlockManager bm = cluster.getNamesystem().getBlockManager();
- fs.getClient().setErasureCodingPolicy("/", null);
+ fs.getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
int fileLen = dataBlocks * blockSize;
Path p = new Path("/test2RecoveryTasksForSameBlockGroup");
final byte[] data = new byte[fileLen];
@@ -266,7 +267,8 @@ public class TestReconstructStripedBlocks {
try {
fs.mkdirs(dirPath);
- fs.setErasureCodingPolicy(dirPath, null);
+ fs.setErasureCodingPolicy(dirPath,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
DFSTestUtil.createFile(fs, filePath,
cellSize * dataBlocks * 2, (short) 1, 0L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
index 03bd124..a712498 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -307,7 +307,8 @@ public class TestStripedINodeFile {
dfs.mkdirs(ecDir);
// set erasure coding policy
- dfs.setErasureCodingPolicy(ecDir, null);
+ dfs.setErasureCodingPolicy(ecDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
DFSTestUtil.createFile(dfs, ecFile, len, (short) 1, 0xFEED);
DFSTestUtil.createFile(dfs, contiguousFile, len, (short) 1, 0xFEED);
final FSDirectory fsd = fsn.getFSDirectory();
@@ -408,7 +409,8 @@ public class TestStripedINodeFile {
client.mkdirs(fooDir, new FsPermission((short) 777), true);
client.setStoragePolicy(fooDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
// set an EC policy on "/foo" directory
- client.setErasureCodingPolicy(fooDir, null);
+ client.setErasureCodingPolicy(fooDir,
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// write file to fooDir
final String barFile = "/foo/bar";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index 053b856..b587017 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -236,7 +236,8 @@ public class TestOfflineImageViewer {
ErasureCodingPolicy ecPolicy =
ErasureCodingPolicyManager.getPolicyByPolicyID(
HdfsConstants.XOR_2_1_POLICY_ID);
- hdfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy);
+ hdfs.getClient().setErasureCodingPolicy(ecDir.toString(),
+ ecPolicy.getName());
writtenFiles.put(ecDir.toString(), hdfs.getFileStatus(ecDir));
// Create an empty Erasure Coded file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
index db21217..e7794d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
@@ -63,7 +63,8 @@ public class TestOfflineImageViewerWithStripedBlocks {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
- cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
Path eczone = new Path("/eczone");
fs.mkdirs(eczone);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index 9b9003a..278963a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -359,7 +359,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Policy 'invalidpolicy' does not match any of the supported policies. Please select any one of [</expected-output>
+ <expected-output>Policy 'invalidpolicy' does not match any supported erasure coding policies.</expected-output>
</comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
index d218169..2eb4b29 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
@@ -989,7 +989,7 @@ public class TestDFSIO implements Tool {
((DistributedFileSystem) fs).getAllErasureCodingPolicies();
for (ErasureCodingPolicy ec : list) {
if (erasureCodePolicyName.equals(ec.getName())) {
- ((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec);
+ ((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec.getName());
LOG.info("enable erasureCodePolicy = " + erasureCodePolicyName +
" on " + path.toString());
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] hadoop git commit: YARN-5280. Allow YARN containers to run
with Java Security Manager (gphillips via rkanter)
Posted by kk...@apache.org.
YARN-5280. Allow YARN containers to run with Java Security Manager (gphillips via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f6dfe02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f6dfe02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f6dfe02
Branch: refs/heads/YARN-5972
Commit: 6f6dfe0202249c129b36edfd145a2224140139cc
Parents: 899d5c4
Author: Robert Kanter <rk...@apache.org>
Authored: Wed Mar 1 14:53:47 2017 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Mar 1 14:53:47 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 20 +
.../src/main/resources/yarn-default.xml | 26 +
.../server/nodemanager/ContainerExecutor.java | 9 +
.../nodemanager/LinuxContainerExecutor.java | 23 +
.../launcher/ContainerLaunch.java | 34 +-
.../DelegatingLinuxContainerRuntime.java | 30 +-
.../JavaSandboxLinuxContainerRuntime.java | 495 +++++++++++++++++++
.../runtime/LinuxContainerRuntimeConstants.java | 2 +
.../executor/ContainerPrepareContext.java | 119 +++++
.../src/main/resources/java.policy | 63 +++
.../TestJavaSandboxLinuxContainerRuntime.java | 364 ++++++++++++++
11 files changed, 1155 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index cdccec6..b366855 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1465,6 +1465,26 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK =
"host";
+ /** The mode in which the Java Container Sandbox should run detailed by
+ * the JavaSandboxLinuxContainerRuntime. */
+ public static final String YARN_CONTAINER_SANDBOX =
+ LINUX_CONTAINER_RUNTIME_PREFIX + "sandbox-mode";
+ public static final String DEFAULT_YARN_CONTAINER_SANDBOX = "disabled";
+
+ /** Permissions for application local directories.*/
+ public static final String YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS =
+ YARN_CONTAINER_SANDBOX + ".local-dirs.permissions";
+ public static final String DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS =
+ "read";
+
+ /** Location for non-default java policy file.*/
+ public static final String YARN_CONTAINER_SANDBOX_POLICY =
+ YARN_CONTAINER_SANDBOX + ".policy";
+
+ /** The group which will run by default without the java security manager.*/
+ public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP =
+ YARN_CONTAINER_SANDBOX + ".whitelist-group";
+
/** The path to the Linux container executor.*/
public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH =
NM_PREFIX + "linux-container-executor.path";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 368946e..645a342 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1584,6 +1584,32 @@
</property>
<property>
+ <description>The mode in which the Java Container Sandbox should run detailed by
+ the JavaSandboxLinuxContainerRuntime.</description>
+ <name>yarn.nodemanager.runtime.linux.sandbox-mode</name>
+ <value>disabled</value>
+ </property>
+
+ <property>
+ <description>Permissions for application local directories.</description>
+ <name>yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions</name>
+ <value>read</value>
+ </property>
+
+ <property>
+ <description>Location for non-default java policy file.</description>
+ <name>yarn.nodemanager.runtime.linux.sandbox-mode.policy</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>The group which will run by default without the java security
+ manager.</description>
+ <name>yarn.nodemanager.runtime.linux.sandbox-mode.whitelist-group</name>
+ <value></value>
+ </property>
+
+ <property>
<description>This flag determines whether memory limit will be set for the Windows Job
Object of the containers launched by the default container executor.</description>
<name>yarn.nodemanager.windows-container.memory-limit.enabled</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 158585e..64d2af9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext;
@@ -148,6 +149,14 @@ public abstract class ContainerExecutor implements Configurable {
public abstract void startLocalizer(LocalizerStartContext ctx)
throws IOException, InterruptedException;
+ /**
+ * Prepare the container prior to the launch environment being written.
+ * @param ctx Encapsulates information necessary for launching containers.
+ * @throws IOException if errors occur during container preparation
+ */
+ public void prepareContainer(ContainerPrepareContext ctx) throws
+ IOException{
+ }
/**
* Launch the container on the node. This is a blocking call and returns only
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 323c443..46b6dfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
@@ -363,6 +364,28 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ public void prepareContainer(ContainerPrepareContext ctx) throws IOException {
+
+ ContainerRuntimeContext.Builder builder =
+ new ContainerRuntimeContext.Builder(ctx.getContainer());
+
+ builder.setExecutionAttribute(LOCALIZED_RESOURCES,
+ ctx.getLocalizedResources())
+ .setExecutionAttribute(USER, ctx.getUser())
+ .setExecutionAttribute(CONTAINER_LOCAL_DIRS,
+ ctx.getContainerLocalDirs())
+ .setExecutionAttribute(CONTAINER_RUN_CMDS, ctx.getCommands())
+ .setExecutionAttribute(CONTAINER_ID_STR,
+ ctx.getContainer().getContainerId().toString());
+
+ try {
+ linuxContainerRuntime.prepareContainer(builder.build());
+ } catch (ContainerExecutionException e) {
+ throw new IOException("Unable to prepare container: ", e);
+ }
+ }
+
+ @Override
public int launchContainer(ContainerStartContext ctx) throws IOException {
Container container = ctx.getContainer();
Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 823457f..a1c407f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
@@ -198,18 +199,14 @@ public class ContainerLaunch implements Callable<Integer> {
FileContext lfs = FileContext.getLocalFSFileContext();
- Path nmPrivateContainerScriptPath =
- dirsHandler.getLocalPathForWrite(
+ Path nmPrivateContainerScriptPath = dirsHandler.getLocalPathForWrite(
getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ CONTAINER_SCRIPT);
- Path nmPrivateTokensPath =
- dirsHandler.getLocalPathForWrite(
- getContainerPrivateDir(appIdStr, containerIdStr)
- + Path.SEPARATOR
+ Path nmPrivateTokensPath = dirsHandler.getLocalPathForWrite(
+ getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
containerIdStr));
- Path nmPrivateClasspathJarDir =
- dirsHandler.getLocalPathForWrite(
+ Path nmPrivateClasspathJarDir = dirsHandler.getLocalPathForWrite(
getContainerPrivateDir(appIdStr, containerIdStr));
DataOutputStream containerScriptOutStream = null;
DataOutputStream tokensOutStream = null;
@@ -224,7 +221,6 @@ public class ContainerLaunch implements Callable<Integer> {
recordContainerWorkDir(containerID, containerWorkDir.toString());
String pidFileSubpath = getPidFileSubpath(appIdStr, containerIdStr);
-
// pid file should be in nm private dir so that it is not
// accessible by users
pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
@@ -240,11 +236,9 @@ public class ContainerLaunch implements Callable<Integer> {
throw new IOException("Most of the disks failed. "
+ dirsHandler.getDisksHealthReport(false));
}
-
try {
// /////////// Write out the container-script in the nmPrivate space.
List<Path> appDirs = new ArrayList<Path>(localDirs.size());
-
for (String localDir : localDirs) {
Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
Path userdir = new Path(usersdir, user);
@@ -258,18 +252,22 @@ public class ContainerLaunch implements Callable<Integer> {
// Set the token location too.
environment.put(
ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME,
- new Path(containerWorkDir,
+ new Path(containerWorkDir,
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
// Sanitize the container's environment
sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs,
- containerLogDirs,
- localResources, nmPrivateClasspathJarDir);
-
+ containerLogDirs, localResources, nmPrivateClasspathJarDir);
+
+ exec.prepareContainer(new ContainerPrepareContext.Builder()
+ .setContainer(container)
+ .setLocalizedResources(localResources)
+ .setUser(user)
+ .setContainerLocalDirs(containerLocalDirs)
+ .setCommands(launchContext.getCommands()).build());
// Write out the environment
exec.writeLaunchEnv(containerScriptOutStream, environment,
localResources, launchContext.getCommands(),
new Path(containerLogDirs.get(0)), user);
-
// /////////// End of writing out container-script
// /////////// Write out the container-tokens in the nmPrivate space.
@@ -295,8 +293,7 @@ public class ContainerLaunch implements Callable<Integer> {
.setFilecacheDirs(filecacheDirs)
.setUserLocalDirs(userLocalDirs)
.setContainerLocalDirs(containerLocalDirs)
- .setContainerLogDirs(containerLogDirs)
- .build());
+ .setContainerLogDirs(containerLogDirs).build());
} catch (Throwable e) {
LOG.warn("Failed to launch container.", e);
dispatcher.getEventHandler().handle(new ContainerExitEvent(
@@ -308,7 +305,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
handleContainerExitCode(ret, containerLogDir);
-
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index c12858e..896ffd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -35,10 +35,10 @@ import java.util.Map;
/**
* This class is a {@link ContainerRuntime} implementation that delegates all
- * operations to either a {@link DefaultLinuxContainerRuntime} instance or a
- * {@link DockerLinuxContainerRuntime} instance, depending on whether the
- * {@link DockerLinuxContainerRuntime} instance believes the operation to be
- * requesting a Docker container.
+ * operations to a {@link DefaultLinuxContainerRuntime} instance, a
+ * {@link DockerLinuxContainerRuntime} instance, or a
+ * {@link JavaSandboxLinuxContainerRuntime} instance depending on whether
+ * each instance believes the operation to be within its scope.
*
* @see DockerLinuxContainerRuntime#isDockerContainerRequested
*/
@@ -49,6 +49,7 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
.getLog(DelegatingLinuxContainerRuntime.class);
private DefaultLinuxContainerRuntime defaultLinuxContainerRuntime;
private DockerLinuxContainerRuntime dockerLinuxContainerRuntime;
+ private JavaSandboxLinuxContainerRuntime javaSandboxLinuxContainerRuntime;
@Override
public void initialize(Configuration conf)
@@ -61,15 +62,20 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
dockerLinuxContainerRuntime = new DockerLinuxContainerRuntime(
privilegedOperationExecutor);
dockerLinuxContainerRuntime.initialize(conf);
+ javaSandboxLinuxContainerRuntime = new JavaSandboxLinuxContainerRuntime(
+ privilegedOperationExecutor);
+ javaSandboxLinuxContainerRuntime.initialize(conf);
}
- private LinuxContainerRuntime pickContainerRuntime(Container container) {
- Map<String, String> env = container.getLaunchContext().getEnvironment();
+ private LinuxContainerRuntime pickContainerRuntime(
+ Map<String, String> environment){
LinuxContainerRuntime runtime;
- if (DockerLinuxContainerRuntime.isDockerContainerRequested(env)){
+ if (DockerLinuxContainerRuntime.isDockerContainerRequested(environment)){
runtime = dockerLinuxContainerRuntime;
- } else {
+ } else if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()) {
+ runtime = javaSandboxLinuxContainerRuntime;
+ } else {
runtime = defaultLinuxContainerRuntime;
}
@@ -81,12 +87,14 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
return runtime;
}
+ private LinuxContainerRuntime pickContainerRuntime(Container container) {
+ return pickContainerRuntime(container.getLaunchContext().getEnvironment());
+ }
+
@Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
- Container container = ctx.getContainer();
- LinuxContainerRuntime runtime = pickContainerRuntime(container);
-
+ LinuxContainerRuntime runtime = pickContainerRuntime(ctx.getContainer());
runtime.prepareContainer(ctx);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
new file mode 100644
index 0000000..6dc627b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.linux.runtime;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+import org.apache.log4j.Logger;
+
+import java.io.FilePermission;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.security.AllPermission;
+import java.util.Formatter;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.fs.Path.SEPARATOR;
+import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR;
+import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCALIZED_RESOURCES;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER;
+/**
+ * <p>This class extends the {@link DefaultLinuxContainerRuntime} specifically
+ * for containers which run Java commands. It generates a new java security
+ * policy file per container and modifies the java command to enable the
+ * Java Security Manager with the generated policy.</p>
+ *
+ * The behavior of the {@link JavaSandboxLinuxContainerRuntime} can be modified
+ * using the following settings:
+ *
+ * <ul>
+ * <li>
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX} :
+ * This yarn-site.xml setting has three options:
+ * <ul>
+ * <li>disabled - Default behavior. {@link LinuxContainerRuntime}
+ * is disabled</li>
+ * <li>permissive - JVM containers will run with Java Security Manager
+ * enabled. Non-JVM containers will run normally</li>
+ * <li>enforcing - JVM containers will run with Java Security Manager
+ * enabled. Non-JVM containers will be prevented from executing and an
+ * {@link ContainerExecutionException} will be thrown.</li>
+ * </ul>
+ * </li>
+ * <li>
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS} :
+ * Determines the file permissions for the application directories. The
+ * permissions come in the form of comma separated values
+ * (e.g. read,write,execute,delete). Defaults to {@code read} for read-only.
+ * </li>
+ * <li>
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY} :
+ * Accepts canonical path to a java policy file on the local filesystem.
+ * This file will be loaded as the base policy, any additional container
+ * grants will be appended to this base file. If not specified, the default
+ * java.policy file provided with hadoop resources will be used.
+ * </li>
+ * <li>
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} :
+ * Optional setting to specify a YARN queue which will be exempt from the
+ * sand-boxing process.
+ * </li>
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class JavaSandboxLinuxContainerRuntime
+ extends DefaultLinuxContainerRuntime {
+ private static final Log LOG =
+ LogFactory.getLog(DefaultLinuxContainerRuntime.class);
+ private Configuration configuration;
+ private SandboxMode sandboxMode;
+
+ public static final String POLICY_FILE_DIR = "nm-sandbox-policies";
+
+ private static Path policyFileDir;
+ private static final FileAttribute<Set<PosixFilePermission>> POLICY_ATTR =
+ PosixFilePermissions.asFileAttribute(
+ PosixFilePermissions.fromString("rwxr-xr-x"));
+
+ private Map<String, Path> containerPolicies = new HashMap<>();
+
+ /**
+ * Create an instance using the given {@link PrivilegedOperationExecutor}
+ * instance for performing operations.
+ *
+ * @param privilegedOperationExecutor the {@link PrivilegedOperationExecutor}
+ * instance
+ */
+ public JavaSandboxLinuxContainerRuntime(
+ PrivilegedOperationExecutor privilegedOperationExecutor) {
+ super(privilegedOperationExecutor);
+ }
+
+ @Override
+ public void initialize(Configuration conf)
+ throws ContainerExecutionException {
+ this.configuration = conf;
+ this.sandboxMode =
+ SandboxMode.get(
+ this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
+ YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
+
+ initializePolicyDir();
+
+ super.initialize(conf);
+ }
+
+ /**
+ * Initialize the Java Security Policy directory. Either creates the
+ * directory if it doesn't exist, or clears the contents of the directory if
+ * already created.
+ * @throws ContainerExecutionException If unable to resolve policy directory
+ */
+ private void initializePolicyDir() throws ContainerExecutionException {
+ String hadoopTempDir = configuration.get("hadoop.tmp.dir");
+ if (hadoopTempDir == null) {
+ throw new ContainerExecutionException("hadoop.tmp.dir not set!");
+ }
+ policyFileDir = Paths.get(hadoopTempDir, POLICY_FILE_DIR);
+ //Delete any existing policy files if the directory has already been created
+ if(Files.exists(policyFileDir)){
+ try (DirectoryStream<Path> stream =
+ Files.newDirectoryStream(policyFileDir)){
+ for(Path policyFile : stream){
+ Files.delete(policyFile);
+ }
+ }catch(IOException e){
+ throw new ContainerExecutionException("Unable to initialize policy "
+ + "directory: " + e);
+ }
+ } else {
+ try {
+ policyFileDir = Files.createDirectories(
+ Paths.get(hadoopTempDir, POLICY_FILE_DIR), POLICY_ATTR);
+ } catch (IOException e) {
+ throw new ContainerExecutionException("Unable to create policy file " +
+ "directory: " + e);
+ }
+ }
+ }
+
+ /**
+ * Prior to environment from being written locally need to generate
+ * policy file which limits container access to a small set of directories.
+ * Additionally the container run command needs to be modified to include
+ * flags to enable the java security manager with the generated policy.
+ * <br>
+ * The Java Sandbox will be circumvented if the user is a member of the
+ * group specified in:
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} and if
+ * they do not include the JVM flag:
+ * {@value NMContainerPolicyUtils#SECURITY_FLAG}
+ *
+ * @param ctx The {@link ContainerRuntimeContext} containing container
+ * setup properties.
+ * @throws ContainerExecutionException Exception thrown if temporary policy
+ * file directory can't be created, or if any exceptions occur during policy
+ * file parsing and generation.
+ */
+ @Override
+ public void prepareContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+
+ @SuppressWarnings("unchecked")
+ List<String> localDirs =
+ ctx.getExecutionAttribute(CONTAINER_LOCAL_DIRS);
+ @SuppressWarnings("unchecked")
+ Map<org.apache.hadoop.fs.Path, List<String>> resources =
+ ctx.getExecutionAttribute(LOCALIZED_RESOURCES);
+ @SuppressWarnings("unchecked")
+ List<String> commands =
+ ctx.getExecutionAttribute(CONTAINER_RUN_CMDS);
+ Map<String, String> env =
+ ctx.getContainer().getLaunchContext().getEnvironment();
+
+ if(!isSandboxContainerWhitelisted(ctx, commands)) {
+ String tmpDirBase = configuration.get("hadoop.tmp.dir");
+ if (tmpDirBase == null) {
+ throw new ContainerExecutionException("hadoop.tmp.dir not set!");
+ }
+
+ OutputStream policyOutputStream = null;
+ try {
+ String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
+
+ Path policyFilePath = Files.createFile(
+ Paths.get(policyFileDir.toString(),
+ containerID + "-" + NMContainerPolicyUtils.POLICY_FILE),
+ POLICY_ATTR);
+ policyOutputStream = Files.newOutputStream(policyFilePath);
+
+ containerPolicies.put(containerID, policyFilePath);
+
+ NMContainerPolicyUtils.generatePolicyFile(
+ policyOutputStream, localDirs, resources, configuration);
+ NMContainerPolicyUtils.appendSecurityFlags(
+ commands, env, policyFilePath, sandboxMode);
+
+ } catch (Exception e) {
+ throw new ContainerExecutionException(e);
+ } finally {
+ IOUtils.cleanup(LOG, policyOutputStream);
+ }
+ }
+ }
+
+ @Override
+ public void launchContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ try {
+ super.launchContainer(ctx);
+ } finally {
+ deletePolicyFiles(ctx);
+ }
+ }
+
+ /**
+ * Determine if JVMSandboxLinuxContainerRuntime should be used. This is
+ * decided based on the value of
+ * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX}
+ * @return true if Sandbox is requested, false otherwise
+ */
+ boolean isSandboxContainerRequested() {
+ return sandboxMode != SandboxMode.disabled;
+ }
+
+ /**
+ * Determine if the container should be whitelisted (i.e. exempt from the
+ * Java Security Manager).
+ * @param ctx The container runtime context for the requested container
+ * @param commands The list of run commands for the container
+ * @return boolean value denoting whether the container should be whitelisted.
+ * @throws ContainerExecutionException If container user can not be resolved
+ */
+ private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx,
+ List<String> commands) throws ContainerExecutionException {
+ String whitelistGroup = configuration.get(
+ YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
+ Groups groups = Groups.getUserToGroupsMappingService(configuration);
+ List<String> userGroups;
+ boolean isWhitelisted = false;
+
+ try {
+ userGroups = groups.getGroups(ctx.getExecutionAttribute(USER));
+ } catch (IOException e) {
+ throw new ContainerExecutionException("Container user does not exist");
+ }
+
+ if(whitelistGroup != null && userGroups.contains(whitelistGroup)) {
+ // If any command has security flag, whitelisting is disabled
+ for(String cmd : commands) {
+ if(cmd.contains(NMContainerPolicyUtils.SECURITY_FLAG)){
+ isWhitelisted = false;
+ break;
+ } else {
+ isWhitelisted = true;
+ }
+ }
+ }
+ return isWhitelisted;
+ }
+
+ /**
+ * Deletes policy files for container specified by parameter. Additionally
+ * this method will age off any stale policy files generated by
+ * {@link JavaSandboxLinuxContainerRuntime}
+ * @param ctx Container context for files to be deleted
+ * @throws ContainerExecutionException if unable to access or delete policy
+ * files or generated policy file directory
+ */
+ private void deletePolicyFiles(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ try {
+ Files.delete(containerPolicies.remove(
+ ctx.getExecutionAttribute(CONTAINER_ID_STR)));
+ } catch (IOException e) {
+ throw new ContainerExecutionException("Unable to delete policy file: "
+ + e);
+ }
+ }
+
+ /**
+ * Enumeration of the modes the JavaSandboxLinuxContainerRuntime can use.
+ * See {@link JavaSandboxLinuxContainerRuntime} for details on the
+ * behavior of each setting.
+ */
+ public enum SandboxMode {
+ enforcing("enforcing"),
+ permissive("permissive"),
+ disabled("disabled");
+
+ private final String mode;
+ SandboxMode(String mode){
+ this.mode = mode;
+ }
+
+ public static SandboxMode get(String mode) {
+
+ if(enforcing.mode.equals(mode)) {
+ return enforcing;
+ } else if(permissive.mode.equals(mode)) {
+ return permissive;
+ } else {
+ return disabled;
+ }
+ }
+
+ public String toString(){
+ return mode;
+ }
+ }
+
+ /**
+ * Static utility class defining String constants and static methods for the
+ * use of the {@link JavaSandboxLinuxContainerRuntime}.
+ */
+ static final class NMContainerPolicyUtils{
+
+ static final String POLICY_FILE = "java.policy";
+ static final String SECURITY_DEBUG = " -Djava.security.debug=all";
+ static final String SECURITY_FLAG = "-Djava.security.manager";
+ static final String POLICY_APPEND_FLAG = "-Djava.security.policy=";
+ static final String POLICY_FLAG = POLICY_APPEND_FLAG + "=";
+ static final String JAVA_CMD = "/bin/java ";
+ static final String JVM_SECURITY_CMD =
+ JAVA_CMD + SECURITY_FLAG + " " + POLICY_FLAG;
+
+ static final String STRIP_POLICY_FLAG = POLICY_APPEND_FLAG + "[^ ]+";
+ static final String CONTAINS_JAVA_CMD = "\\$" + JAVA_HOME + JAVA_CMD + ".*";
+ static final String CHAINED_COMMAND_REGEX =
+ "^.*(&&.+$)|(\\|\\|.+$).*$"; //Matches any occurrences of '||' or '&&'
+ static final String CLEAN_CMD_REGEX =
+ "(" + SECURITY_FLAG + ")|" +
+ "(" + STRIP_POLICY_FLAG + ")";
+
+ static final String FILE_PERMISSION_FORMAT = " permission "
+ + FilePermission.class.getCanonicalName()
+ + " \"%1$s" + SEPARATOR + "-\", \"%2$s\";%n";
+ static final String HADOOP_HOME_PERMISSION = "%ngrant codeBase \"file:"
+ + Paths.get(System.getProperty(SYSPROP_HADOOP_HOME_DIR))
+ + SEPARATOR + "-\" {%n" +
+ " permission " + AllPermission.class.getCanonicalName() + ";%n};%n";
+ static final Logger LOG =
+ Logger.getLogger(NMContainerPolicyUtils.class);
+
+ /**
+ * Write new policy file to policyOutStream which will include read access
+ * to localize resources. Optionally a default policyFilePath can be
+ * specified to append a custom policy implementation to the new policy file
+ * @param policyOutStream OutputStream pointing to java.policy file
+ * @param localDirs Container local directories
+ * @param resources List of local container resources
+ * @param conf YARN configuration
+ * @throws IOException - If policy file generation is unable to read the
+ * base policy file or if it is unable to create a new policy file.
+ */
+ static void generatePolicyFile(OutputStream policyOutStream,
+ List<String> localDirs, Map<org.apache.hadoop.fs.Path,
+ List<String>> resources, Configuration conf)
+ throws IOException {
+
+ String policyFilePath =
+ conf.get(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY);
+ String filePermissions =
+ conf.get(YarnConfiguration.YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS,
+ YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS);
+
+ Set<String> cacheDirs = new HashSet<>();
+ for(org.apache.hadoop.fs.Path path : resources.keySet()) {
+ cacheDirs.add(path.getParent().toString());
+ }
+
+ if(policyFilePath == null) {
+ IOUtils.copyBytes(
+ NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE),
+ policyOutStream, conf, false);
+ } else {
+ Files.copy(Paths.get(policyFilePath), policyOutStream);
+ policyOutStream.flush();
+ }
+
+ Formatter filePermissionFormat = new Formatter(policyOutStream,
+ StandardCharsets.UTF_8.name());
+ filePermissionFormat.format(HADOOP_HOME_PERMISSION);
+ filePermissionFormat.format("grant {%n");
+ for(String localDir : localDirs) {
+ filePermissionFormat.format(
+ FILE_PERMISSION_FORMAT, localDir, filePermissions);
+ }
+ for(String cacheDir : cacheDirs) {
+ filePermissionFormat.format(
+ FILE_PERMISSION_FORMAT, cacheDir, filePermissions);
+ }
+ filePermissionFormat.format("};%n");
+ filePermissionFormat.flush();
+ }
+
+ /**
+ * Modify command to enable the Java Security Manager and specify
+ * java.policy file. Will modify the passed commands to strip any
+ * existing java security configurations. Expects a java command to be the
+ * first and only executable provided in enforcing mode. In passive mode
+ * any commands with '||' or '&&' will not be modified.
+ * @param commands List of container commands
+ * @param env Container environment variables
+ * @param policyPath Path to the container specific policy file
+ * @param sandboxMode (enforcing, permissive, disabled) Determines
+ * whether non-java containers will be launched
+ * @throws ContainerExecutionException - Exception thrown if
+ * JVM Sandbox enabled in 'enforcing' mode and a non-java command is
+ * provided in the list of commands
+ */
+ static void appendSecurityFlags(List<String> commands,
+ Map<String, String> env, Path policyPath, SandboxMode sandboxMode)
+ throws ContainerExecutionException {
+
+ for(int i = 0; i < commands.size(); i++){
+ String command = commands.get(i);
+ if(validateJavaHome(env.get(JAVA_HOME.name()))
+ && command.matches(CONTAINS_JAVA_CMD)
+ && !command.matches(CHAINED_COMMAND_REGEX)){
+ command = command.replaceAll(CLEAN_CMD_REGEX, "");
+ String securityString = JVM_SECURITY_CMD + policyPath + " ";
+ if(LOG.isDebugEnabled()) {
+ securityString += SECURITY_DEBUG;
+ }
+ commands.set(i, command.replaceFirst(JAVA_CMD, securityString));
+ } else if (sandboxMode == SandboxMode.enforcing){
+ throw new ContainerExecutionException(
+ "Only JVM containers permitted in YARN sandbox mode (enforcing). "
+ + "The following command can not be executed securely: " + command);
+ } else if (sandboxMode == SandboxMode.permissive){
+ LOG.warn("The container will run without the java security manager"
+ + " due to an unsupported container command. The command"
+ + " will be permitted to run in Sandbox permissive mode: "
+ + command);
+ }
+ }
+ }
+
+ private static boolean validateJavaHome(String containerJavaHome)
+ throws ContainerExecutionException{
+ if (System.getenv(JAVA_HOME.name()) == null){
+ throw new ContainerExecutionException(
+ "JAVA_HOME is not set for NodeManager");
+ }
+ if (containerJavaHome == null) {
+ throw new ContainerExecutionException(
+ "JAVA_HOME is not set for container");
+ }
+ return System.getenv(JAVA_HOME.name()).equals(containerJavaHome);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
index 0c1ec3e..2e632fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
@@ -67,6 +67,8 @@ public final class LinuxContainerRuntimeConstants {
String.class, "resources_options");
public static final Attribute<String> TC_COMMAND_FILE = Attribute.attribute(
String.class, "tc_command_file");
+ public static final Attribute<List> CONTAINER_RUN_CMDS = Attribute.attribute(
+ List.class, "container_run_cmds");
public static final Attribute<String> CGROUP_RELATIVE_PATH = Attribute
.attribute(String.class, "cgroup_relative_path");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerPrepareContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerPrepareContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerPrepareContext.java
new file mode 100644
index 0000000..f711da2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerPrepareContext.java
@@ -0,0 +1,119 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.executor;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Encapsulates information required for preparing containers.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class ContainerPrepareContext {
+ private final Container container;
+ private final Map<Path, List<String>> localizedResources;
+ private final String user;
+ private final List<String> containerLocalDirs;
+ private final List<String> commands;
+
+ /**
+ * Builder for ContainerPrepareContext.
+ */
+ public static final class Builder {
+ private Container container;
+ private Map<Path, List<String>> localizedResources;
+ private String user;
+ private List<String> containerLocalDirs;
+ private List<String> commands;
+
+ public Builder() {
+ }
+
+ public ContainerPrepareContext.Builder setContainer(Container container) {
+ this.container = container;
+ return this;
+ }
+
+ public ContainerPrepareContext.Builder setLocalizedResources(Map<Path,
+ List<String>> localizedResources) {
+ this.localizedResources = localizedResources;
+ return this;
+ }
+
+ public ContainerPrepareContext.Builder setUser(String user) {
+ this.user = user;
+ return this;
+ }
+ public ContainerPrepareContext.Builder setContainerLocalDirs(
+ List<String> containerLocalDirs) {
+ this.containerLocalDirs = containerLocalDirs;
+ return this;
+ }
+
+ public ContainerPrepareContext build() {
+ return new ContainerPrepareContext(this);
+ }
+
+ public ContainerPrepareContext.Builder setCommands(List<String> commands) {
+ this.commands = commands;
+ return this;
+ }
+ }
+
+ private ContainerPrepareContext(ContainerPrepareContext.Builder builder) {
+ this.container = builder.container;
+ this.localizedResources = builder.localizedResources;
+ this.user = builder.user;
+ this.containerLocalDirs = builder.containerLocalDirs;
+ this.commands = builder.commands;
+ }
+
+ public Container getContainer() {
+ return this.container;
+ }
+
+ public Map<Path, List<String>> getLocalizedResources() {
+ if (this.localizedResources != null) {
+ return Collections.unmodifiableMap(this.localizedResources);
+ } else {
+ return null;
+ }
+ }
+
+ public String getUser() {
+ return this.user;
+ }
+
+ public List<String> getContainerLocalDirs() {
+ return Collections.unmodifiableList(this.containerLocalDirs);
+ }
+
+ public List<String> getCommands(){
+ return this.commands;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/java.policy
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/java.policy b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/java.policy
new file mode 100644
index 0000000..d9320c2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/java.policy
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Standard extensions get all permissions by default
+grant codeBase "file:${{java.ext.dirs}}/*" {
+ permission java.security.AllPermission;
+};
+
+// default permissions granted to all domains
+grant {
+ permission java.lang.RuntimePermission "accessDeclaredMembers";
+
+ permission java.util.PropertyPermission "java.version", "read";
+ permission java.util.PropertyPermission "java.vendor", "read";
+ permission java.util.PropertyPermission "java.vendor.url", "read";
+ permission java.util.PropertyPermission "java.class.version", "read";
+ permission java.util.PropertyPermission "os.name", "read";
+ permission java.util.PropertyPermission "os.version", "read";
+ permission java.util.PropertyPermission "os.arch", "read";
+ permission java.util.PropertyPermission "file.separator", "read";
+ permission java.util.PropertyPermission "path.separator", "read";
+ permission java.util.PropertyPermission "line.separator", "read";
+
+ permission java.util.PropertyPermission "java.specification.version", "read";
+ permission java.util.PropertyPermission "java.specification.vendor", "read";
+ permission java.util.PropertyPermission "java.specification.name", "read";
+
+ permission java.util.PropertyPermission "java.vm.specification.version", "read";
+ permission java.util.PropertyPermission "java.vm.specification.vendor", "read";
+ permission java.util.PropertyPermission "java.vm.specification.name", "read";
+ permission java.util.PropertyPermission "java.vm.version", "read";
+ permission java.util.PropertyPermission "java.vm.vendor", "read";
+ permission java.util.PropertyPermission "java.vm.name", "read";
+
+ //additional hadoop permissions
+ permission java.util.PropertyPermission "awt.Toolkit", "read";
+ permission java.util.PropertyPermission "file.encoding", "read";
+ permission java.util.PropertyPermission "file.encoding.pkg", "read";
+ permission java.util.PropertyPermission "hadoop.metrics.log.level", "read";
+ permission java.util.PropertyPermission "hadoop.root.logger", "read";
+ permission java.util.PropertyPermission "java.awt.graphicsenv" ,"read";
+ permission java.util.PropertyPermission "java.awt.printerjob", "read";
+ permission java.util.PropertyPermission "java.class.path", "read";
+ permission java.util.PropertyPermission "yarn.app.container.log.dir", "read";
+ permission java.util.PropertyPermission "yarn.app.container.log.filesize", "read";
+ permission java.lang.RuntimePermission "loadLibrary.gplcompression";
+ permission javax.security.auth.AuthPermission "getSubject";
+};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6dfe02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
new file mode 100644
index 0000000..e482c8d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.linux.runtime;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilePermission;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CHAINED_COMMAND_REGEX;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FLAG;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.SECURITY_FLAG;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.POLICY_FILE_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPID;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_WORK_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.FILECACHE_DIRS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCALIZED_RESOURCES;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCAL_DIRS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOG_DIRS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.RUN_AS_USER;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER_LOCAL_DIRS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test policy file generation and policy enforcement for the
+ * {@link JavaSandboxLinuxContainerRuntime}.
+ */
+public class TestJavaSandboxLinuxContainerRuntime {
+
+ private static final String HADOOP_HOME = "hadoop.home.dir";
+ private static String hadoopHomeDir = System.getProperty(HADOOP_HOME);
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ private static File grantFile, denyFile, policyFile,
+ grantDir, denyDir, containerDir;
+ private static java.nio.file.Path policyFilePath;
+ private static SecurityManager securityManager;
+ private Map<Path, List<String>> resources;
+ private Map<String, String> env;
+ private List<String> whitelistGroup;
+
+ private PrivilegedOperationExecutor mockExecutor;
+ private JavaSandboxLinuxContainerRuntime runtime;
+ private ContainerRuntimeContext.Builder runtimeContextBuilder;
+ private Configuration conf;
+
+ private final static String NORMAL_USER = System.getProperty("user.name");
+ private final static String NORMAL_GROUP = "normalGroup";
+ private final static String WHITELIST_USER = "picard";
+ private final static String WHITELIST_GROUP = "captains";
+ private final static String CONTAINER_ID = "container_1234567890";
+ private final static String APPLICATION_ID = "application_1234567890";
+
+ @Before
+ public void setup() throws Exception {
+
+ File baseTestDirectory = new File(System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir", "target")),
+ TestJavaSandboxLinuxContainerRuntime.class.getName());
+
+ whitelistGroup = new ArrayList<>();
+ whitelistGroup.add(WHITELIST_GROUP);
+
+ conf = new Configuration();
+ conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
+ WHITELIST_USER + "=" + WHITELIST_GROUP + ";"
+ + NORMAL_USER + "=" + NORMAL_GROUP + ";");
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+ WHITELIST_GROUP);
+ conf.set("hadoop.tmp.dir", baseTestDirectory.getAbsolutePath());
+
+ Files.deleteIfExists(Paths.get(baseTestDirectory.getAbsolutePath(),
+ POLICY_FILE_DIR, CONTAINER_ID + "-" + POLICY_FILE));
+
+ mockExecutor = mock(PrivilegedOperationExecutor.class);
+ runtime = new JavaSandboxLinuxContainerRuntime(mockExecutor);
+ runtime.initialize(conf);
+
+ resources = new HashMap<>();
+ grantDir = new File(baseTestDirectory, "grantDir");
+ denyDir = new File(baseTestDirectory, "denyDir");
+ containerDir = new File(baseTestDirectory,
+ APPLICATION_ID + Path.SEPARATOR + CONTAINER_ID);
+ grantDir.mkdirs();
+ denyDir.mkdirs();
+ containerDir.mkdirs();
+
+ grantFile = File.createTempFile("grantFile", "tmp", grantDir);
+ denyFile = File.createTempFile("denyFile", "tmp", denyDir);
+
+ List<String> symLinks = new ArrayList<>();
+ symLinks.add(grantFile.getName());
+ resources.put(new Path(grantFile.getCanonicalPath()), symLinks);
+
+ env = new HashMap();
+ env.put(JAVA_HOME.name(), System.getenv(JAVA_HOME.name()));
+
+ policyFile = File.createTempFile("java", "policy", containerDir);
+ policyFilePath = Paths.get(policyFile.getAbsolutePath());
+
+ runtimeContextBuilder = createRuntimeContext();
+
+ if (hadoopHomeDir == null) {
+ System.setProperty(HADOOP_HOME, policyFile.getParent());
+ }
+
+ OutputStream outStream = new FileOutputStream(policyFile);
+ JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
+ .generatePolicyFile(outStream, symLinks, resources, conf);
+ outStream.close();
+
+ System.setProperty("java.security.policy", policyFile.getCanonicalPath());
+ securityManager = new SecurityManager();
+
+ }
+
+ public ContainerRuntimeContext.Builder createRuntimeContext(){
+
+ Container container = mock(Container.class);
+ ContainerLaunchContext ctx = mock(ContainerLaunchContext.class);
+
+ when(container.getLaunchContext()).thenReturn(ctx);
+ when(ctx.getEnvironment()).thenReturn(env);
+
+ ContainerRuntimeContext.Builder builder =
+ new ContainerRuntimeContext.Builder(container);
+
+ List<String> localDirs = new ArrayList<>();
+
+ builder.setExecutionAttribute(LOCALIZED_RESOURCES, resources)
+ .setExecutionAttribute(RUN_AS_USER, NORMAL_USER)
+ .setExecutionAttribute(CONTAINER_ID_STR, CONTAINER_ID)
+ .setExecutionAttribute(APPID, APPLICATION_ID)
+ .setExecutionAttribute(CONTAINER_WORK_DIR,
+ new Path(containerDir.toString()))
+ .setExecutionAttribute(LOCAL_DIRS, localDirs)
+ .setExecutionAttribute(LOG_DIRS, localDirs)
+ .setExecutionAttribute(FILECACHE_DIRS, localDirs)
+ .setExecutionAttribute(USER_LOCAL_DIRS, localDirs)
+ .setExecutionAttribute(CONTAINER_LOCAL_DIRS, localDirs)
+ .setExecutionAttribute(CONTAINER_RUN_CMDS, localDirs);
+
+ return builder;
+ }
+
+ @Test
+ public void testGrant() throws Exception {
+ FilePermission grantPermission =
+ new FilePermission(grantFile.getAbsolutePath(), "read");
+ securityManager.checkPermission(grantPermission);
+ }
+
+ @Test
+ public void testDeny() throws Exception {
+ FilePermission denyPermission =
+ new FilePermission(denyFile.getAbsolutePath(), "read");
+ exception.expect(java.security.AccessControlException.class);
+ securityManager.checkPermission(denyPermission);
+ }
+
+ @Test
+ public void testEnforcingMode() throws ContainerExecutionException {
+ String[] nonJavaCommands = {
+ "bash malicious_script.sh",
+ "python malicious_script.py"
+ };
+
+ List<String> commands = Arrays.asList(nonJavaCommands);
+ exception.expect(ContainerExecutionException.class);
+ JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
+ .appendSecurityFlags(commands, env, policyFilePath,
+ JavaSandboxLinuxContainerRuntime.SandboxMode.enforcing);
+
+ }
+
+ @Test
+ public void testPermissiveMode() throws ContainerExecutionException {
+ String[] nonJavaCommands = {
+ "bash non-java-script.sh",
+ "python non-java-script.py"
+ };
+
+ List<String> commands = Arrays.asList(nonJavaCommands);
+ JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
+ .appendSecurityFlags(commands, env, policyFilePath,
+ JavaSandboxLinuxContainerRuntime.SandboxMode.permissive);
+
+ }
+
+ @Test
+ public void testDisabledSandboxWithWhitelist()
+ throws ContainerExecutionException {
+
+ String[] inputCommand = {
+ "java jar MyJob.jar"
+ };
+ List<String> commands = Arrays.asList(inputCommand);
+
+ runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
+ runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
+ runtime.prepareContainer(runtimeContextBuilder.build());
+
+ Assert.assertTrue("Command should not be modified when user is " +
+ "member of whitelisted group",
+ inputCommand[0].equals(commands.get(0)));
+
+ }
+
+ @Test
+ public void testEnabledSandboxWithWhitelist()
+ throws ContainerExecutionException{
+ String[] inputCommand = {
+ "$JAVA_HOME/bin/java jar -Djava.security.manager MyJob.jar"
+ };
+ List<String> commands = Arrays.asList(inputCommand);
+
+ runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
+ runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
+ runtime.prepareContainer(runtimeContextBuilder.build());
+
+ Assert.assertTrue("Command should be modified to include " +
+ "policy file in whitelisted Sandbox mode",
+ commands.get(0).contains(SECURITY_FLAG)
+ && commands.get(0).contains(POLICY_FLAG));
+ }
+
+ @Test
+ public void testDeniedWhitelistGroup() throws ContainerExecutionException {
+
+ String[] inputCommand = {
+ "$JAVA_HOME/bin/java jar MyJob.jar"
+ };
+ List<String> commands = Arrays.asList(inputCommand);
+
+ runtimeContextBuilder.setExecutionAttribute(USER, NORMAL_USER);
+ runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
+ runtime.prepareContainer(runtimeContextBuilder.build());
+
+ Assert.assertTrue("Java security manager must be enabled for "
+ + "unauthorized users",
+ commands.get(0).contains(SECURITY_FLAG));
+ }
+
+ @Test
+ public void testChainedCmdRegex(){
+ Assert.assertTrue("cmd1 && cmd2 || cmd3".matches(CHAINED_COMMAND_REGEX));
+ Assert.assertFalse("cmd1 &> logfile".matches(CHAINED_COMMAND_REGEX));
+ }
+
+ @Test
+ public void testContainsJavaRegex(){
+ String[] javaCmds = {
+ "$JAVA_HOME/bin/java -cp App.jar AppClass",
+ "$JAVA_HOME/bin/java -jar App.jar AppClass &> logfile"
+ };
+ String[] nonJavaCmds = {
+ "$JAVA_HOME/bin/jajavava -cp App.jar AppClass",
+ "/nm/app/container/usercache/badjava -cp Bad.jar ChaosClass"
+ };
+ for(String javaCmd : javaCmds) {
+ Assert.assertTrue(javaCmd.matches(CONTAINS_JAVA_CMD));
+ }
+ for(String nonJavaCmd : nonJavaCmds) {
+ Assert.assertFalse(nonJavaCmd.matches(CONTAINS_JAVA_CMD));
+ }
+ }
+
+ @Test
+ public void testCleanCmdRegex(){
+ String[] securityManagerCmds = {
+ "/usr/bin/java -Djava.security.manager -cp $CLASSPATH $MainClass",
+ "-Djava.security.manager -Djava.security.policy==testpolicy keepThis"
+ };
+ String[] cleanedCmdsResult = {
+ "/usr/bin/java -cp $CLASSPATH $MainClass",
+ "keepThis"
+ };
+ for(int i = 0; i < securityManagerCmds.length; i++){
+ Assert.assertEquals(
+ securityManagerCmds[i].replaceAll(CLEAN_CMD_REGEX, "").trim(),
+ cleanedCmdsResult[i]);
+ }
+ }
+
+ @Test
+ public void testAppendSecurityFlags() throws ContainerExecutionException {
+ String securityString = "-Djava.security.manager -Djava.security.policy=="
+ + policyFile.getAbsolutePath();
+ String[] badCommands = {
+ "$JAVA_HOME/bin/java -Djava.security.manager "
+ + "-Djava.security.policy=/home/user/java.policy",
+ "$JAVA_HOME/bin/java -cp MyApp.jar MrAppMaster"
+ };
+ String[] cleanCommands = {
+ "$JAVA_HOME/bin/java " + securityString,
+ "$JAVA_HOME/bin/java " + securityString
+ + " -cp MyApp.jar MrAppMaster"
+ };
+
+ List<String> commands = Arrays.asList(badCommands);
+ JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
+ .appendSecurityFlags(commands, env, policyFilePath,
+ JavaSandboxLinuxContainerRuntime.SandboxMode.enforcing);
+
+ for(int i = 0; i < commands.size(); i++) {
+ Assert.assertTrue(commands.get(i).trim().equals(cleanCommands[i].trim()));
+ }
+ }
+
+ @After
+ public void cleanup(){
+ System.setSecurityManager(null);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] hadoop git commit: HDFS-11419. HDFS specific network topology
classes with storage type info included. Contributed by Chen Liang.
Posted by kk...@apache.org.
HDFS-11419. HDFS specific network topology classes with storage type info included. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eeca8b0c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eeca8b0c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eeca8b0c
Branch: refs/heads/YARN-5972
Commit: eeca8b0c4e2804b0fee5b012ea14b58383425ec3
Parents: 4e14ead
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Mar 2 09:21:27 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Mar 2 09:21:27 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/net/InnerNode.java | 2 +-
.../org/apache/hadoop/net/InnerNodeImpl.java | 22 +-
.../org/apache/hadoop/net/NetworkTopology.java | 27 +-
.../apache/hadoop/net/TestClusterTopology.java | 5 +-
.../blockmanagement/DatanodeDescriptor.java | 9 +
.../apache/hadoop/hdfs/DFSNetworkTopology.java | 36 +++
.../apache/hadoop/hdfs/DFSTopologyNodeImpl.java | 253 ++++++++++++++++++
.../hadoop/hdfs/TestBlockStoragePolicy.java | 2 +-
.../hadoop/hdfs/TestDFSNetworkTopology.java | 260 +++++++++++++++++++
.../apache/hadoop/net/TestNetworkTopology.java | 6 +-
10 files changed, 601 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
index d07929b..65eaf24 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
@@ -61,7 +61,7 @@ public interface InnerNode extends Node {
*
* @param leafIndex an indexed leaf of the node
* @param excludedNode an excluded node (can be null)
- * @return
+ * @return the leaf node corresponding to the given index.
*/
Node getLeaf(int leafIndex, Node excludedNode);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
index e6aa0f7..81eaf7f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
@@ -25,9 +25,9 @@ import java.util.Map;
/** InnerNode represents a switch/router of a data center or rack.
* Different from a leaf node, it has non-null children.
*/
-class InnerNodeImpl extends NodeBase implements InnerNode {
- static class Factory implements InnerNode.Factory<InnerNodeImpl> {
- private Factory() {}
+public class InnerNodeImpl extends NodeBase implements InnerNode {
+ protected static class Factory implements InnerNode.Factory<InnerNodeImpl> {
+ protected Factory() {}
@Override
public InnerNodeImpl newInnerNode(String path) {
@@ -37,18 +37,18 @@ class InnerNodeImpl extends NodeBase implements InnerNode {
static final Factory FACTORY = new Factory();
- private final List<Node> children = new ArrayList<>();
- private final Map<String, Node> childrenMap = new HashMap<>();
- private int numOfLeaves;
+ protected final List<Node> children = new ArrayList<>();
+ protected final Map<String, Node> childrenMap = new HashMap<>();
+ protected int numOfLeaves;
/** Construct an InnerNode from a path-like string */
- InnerNodeImpl(String path) {
+ protected InnerNodeImpl(String path) {
super(path);
}
/** Construct an InnerNode
* from its name, its network location, its parent, and its level */
- InnerNodeImpl(String name, String location, InnerNode parent, int level) {
+ protected InnerNodeImpl(String name, String location, InnerNode parent, int level) {
super(name, location, parent, level);
}
@@ -81,7 +81,7 @@ class InnerNodeImpl extends NodeBase implements InnerNode {
* @param n a node
* @return true if this node is an ancestor of <i>n</i>
*/
- boolean isAncestor(Node n) {
+ protected boolean isAncestor(Node n) {
return getPath(this).equals(NodeBase.PATH_SEPARATOR_STR) ||
(n.getNetworkLocation()+NodeBase.PATH_SEPARATOR_STR).
startsWith(getPath(this)+NodeBase.PATH_SEPARATOR_STR);
@@ -92,12 +92,12 @@ class InnerNodeImpl extends NodeBase implements InnerNode {
* @param n a node
* @return true if this node is the parent of <i>n</i>
*/
- boolean isParent(Node n) {
+ protected boolean isParent(Node n) {
return n.getNetworkLocation().equals(getPath(this));
}
/* Return a child name of this node who is an ancestor of node <i>n</i> */
- private String getNextAncestorName(Node n) {
+ protected String getNextAncestorName(Node n) {
if (!isAncestor(n)) {
throw new IllegalArgumentException(
this + "is not an ancestor of " + n);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 38c02f8..051012b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -68,16 +68,32 @@ public class NetworkTopology {
* @return an instance of NetworkTopology
*/
public static NetworkTopology getInstance(Configuration conf){
- return ReflectionUtils.newInstance(
+ return getInstance(conf, InnerNodeImpl.FACTORY);
+ }
+
+ public static NetworkTopology getInstance(Configuration conf,
+ InnerNode.Factory factory) {
+ NetworkTopology nt = ReflectionUtils.newInstance(
conf.getClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
- NetworkTopology.class, NetworkTopology.class), conf);
+ NetworkTopology.class, NetworkTopology.class), conf);
+ return nt.init(factory);
+ }
+
+ protected NetworkTopology init(InnerNode.Factory factory) {
+ if (!factory.equals(this.factory)) {
+ // the constructor has initialized the factory to default. So only init
+ // again if another factory is specified.
+ this.factory = factory;
+ this.clusterMap = factory.newInnerNode(NodeBase.ROOT);
+ }
+ return this;
}
- InnerNode.Factory factory = InnerNodeImpl.FACTORY;
+ InnerNode.Factory factory;
/**
* the root cluster map
*/
- InnerNode clusterMap = factory.newInnerNode(NodeBase.ROOT);
+ InnerNode clusterMap;
/** Depth of all leaf nodes */
private int depthOfAllLeaves = -1;
/** rack counter */
@@ -92,7 +108,10 @@ public class NetworkTopology {
/** the lock used to manage access */
protected ReadWriteLock netlock = new ReentrantReadWriteLock();
+ // keeping the constructor because other components like MR still uses this.
public NetworkTopology() {
+ this.factory = InnerNodeImpl.FACTORY;
+ this.clusterMap = factory.newInnerNode(NodeBase.ROOT);
}
/** Add a leaf node
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
index 72fc5cb..09f0914 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.List;
import org.apache.commons.math3.stat.inference.ChiSquareTest;
+import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
import org.junit.Test;
@@ -80,7 +81,7 @@ public class TestClusterTopology extends Assert {
@Test
public void testCountNumNodes() throws Exception {
// create the topology
- NetworkTopology cluster = new NetworkTopology();
+ NetworkTopology cluster = NetworkTopology.getInstance(new Configuration());
NodeElement node1 = getNewNode("node1", "/d1/r1");
cluster.add(node1);
NodeElement node2 = getNewNode("node2", "/d1/r2");
@@ -128,7 +129,7 @@ public class TestClusterTopology extends Assert {
@Test
public void testChooseRandom() {
// create the topology
- NetworkTopology cluster = new NetworkTopology();
+ NetworkTopology cluster = NetworkTopology.getInstance(new Configuration());
NodeElement node1 = getNewNode("node1", "/d1/r1");
cluster.add(node1);
NodeElement node2 = getNewNode("node2", "/d1/r2");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index c180432..15af6ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -300,6 +301,14 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
}
+ public EnumSet<StorageType> getStorageTypes() {
+ EnumSet<StorageType> storageTypes = EnumSet.noneOf(StorageType.class);
+ for (DatanodeStorageInfo dsi : getStorageInfos()) {
+ storageTypes.add(dsi.getStorageType());
+ }
+ return storageTypes;
+ }
+
public StorageReport[] getStorageReports() {
final DatanodeStorageInfo[] infos = getStorageInfos();
final StorageReport[] reports = new StorageReport[infos.length];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java
new file mode 100644
index 0000000..a6b8c00
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetworkTopology;
+
+/**
+ * The HDFS specific network topology class. The main purpose of doing this
+ * subclassing is to add storage-type-aware chooseRandom method. All the
+ * remaining parts should be the same.
+ *
+ * Currently a placeholder to test storage type info.
+ * TODO : add "chooseRandom with storageType info" function.
+ */
+public class DFSNetworkTopology extends NetworkTopology {
+ public static DFSNetworkTopology getInstance(Configuration conf) {
+ DFSNetworkTopology nt = new DFSNetworkTopology();
+ return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java
new file mode 100644
index 0000000..e746823
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.net.InnerNode;
+import org.apache.hadoop.net.InnerNodeImpl;
+import org.apache.hadoop.net.Node;
+
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.HashMap;
+
+/**
+ * The HDFS-specific representation of a network topology inner node. The
+ * difference is this class includes the information about the storage type
+ * info of this subtree. This info will be used when selecting subtrees
+ * in block placement.
+ */
+public class DFSTopologyNodeImpl extends InnerNodeImpl {
+
+ static final InnerNodeImpl.Factory FACTORY
+ = new DFSTopologyNodeImpl.Factory();
+
+ static final class Factory extends InnerNodeImpl.Factory {
+ private Factory() {}
+
+ @Override
+ public InnerNodeImpl newInnerNode(String path) {
+ return new DFSTopologyNodeImpl(path);
+ }
+ }
+
+ /**
+ * The core data structure of this class. The information about what storage
+ * types this subtree has. Basically, a map whose key is a child
+ * id, value is a enum map including the counts of each storage type. e.g.
+ * DISK type has count 5 means there are 5 leaf datanodes with DISK type
+ * available. This value is set/updated upon datanode joining and leaving.
+ *
+ * NOTE : It might be sufficient to keep only a map from storage type
+ * to count, omitting the child node id. But this might make it hard to keep
+ * consistency when there are updates from children.
+ *
+ * For example, if currently R has two children A and B with storage X, Y, and
+ * A : X=1 Y=1
+ * B : X=2 Y=2
+ * so we store X=3 Y=3 as total on R.
+ *
+ * Now say A has a new X plugged in and becomes X=2 Y=1.
+ *
+ * If we know that "A adds one X", it is easy to update R by +1 on X. However,
+ * if we don't know "A adds one X", but instead got "A now has X=2 Y=1",
+ * (which seems to be the case in current heartbeat) we will not know how to
+ * update R. While if we store on R "A has X=1 and Y=1" then we can simply
+ * update R by completely replacing the A entry and all will be good.
+ */
+ private final HashMap
+ <String, EnumMap<StorageType, Integer>> childrenStorageInfo;
+
+ DFSTopologyNodeImpl(String path) {
+ super(path);
+ childrenStorageInfo = new HashMap<>();
+ }
+
+ DFSTopologyNodeImpl(
+ String name, String location, InnerNode parent, int level) {
+ super(name, location, parent, level);
+ childrenStorageInfo = new HashMap<>();
+ }
+
+ int getNumOfChildren() {
+ return children.size();
+ }
+
+ @Override
+ public boolean add(Node n) {
+ if (!isAncestor(n)) {
+ throw new IllegalArgumentException(n.getName()
+ + ", which is located at " + n.getNetworkLocation()
+ + ", is not a descendant of " + getPath(this));
+ }
+ // In HDFS topology, the leaf node should always be DatanodeDescriptor
+ if (!(n instanceof DatanodeDescriptor)) {
+ throw new IllegalArgumentException("Unexpected node type "
+ + n.getClass().getName());
+ }
+ DatanodeDescriptor dnDescriptor = (DatanodeDescriptor) n;
+ if (isParent(n)) {
+ // this node is the parent of n; add n directly
+ n.setParent(this);
+ n.setLevel(this.level + 1);
+ Node prev = childrenMap.put(n.getName(), n);
+ if (prev != null) {
+ for(int i=0; i<children.size(); i++) {
+ if (children.get(i).getName().equals(n.getName())) {
+ children.set(i, n);
+ return false;
+ }
+ }
+ }
+ children.add(n);
+ numOfLeaves++;
+ synchronized (childrenStorageInfo) {
+ if (!childrenStorageInfo.containsKey(dnDescriptor.getName())) {
+ childrenStorageInfo.put(
+ dnDescriptor.getName(), new EnumMap<>(StorageType.class));
+ }
+ for (StorageType st : dnDescriptor.getStorageTypes()) {
+ childrenStorageInfo.get(dnDescriptor.getName()).put(st, 1);
+ }
+ }
+ return true;
+ } else {
+ // find the next ancestor node
+ String parentName = getNextAncestorName(n);
+ InnerNode parentNode = (InnerNode)childrenMap.get(parentName);
+ if (parentNode == null) {
+ // create a new InnerNode
+ parentNode = createParentNode(parentName);
+ children.add(parentNode);
+ childrenMap.put(parentNode.getName(), parentNode);
+ }
+ // add n to the subtree of the next ancestor node
+ if (parentNode.add(n)) {
+ numOfLeaves++;
+ synchronized (childrenStorageInfo) {
+ if (!childrenStorageInfo.containsKey(parentNode.getName())) {
+ childrenStorageInfo.put(
+ parentNode.getName(), new EnumMap<>(StorageType.class));
+ for (StorageType st : dnDescriptor.getStorageTypes()) {
+ childrenStorageInfo.get(parentNode.getName()).put(st, 1);
+ }
+ } else {
+ EnumMap<StorageType, Integer> currentCount =
+ childrenStorageInfo.get(parentNode.getName());
+ for (StorageType st : dnDescriptor.getStorageTypes()) {
+ if (currentCount.containsKey(st)) {
+ currentCount.put(st, currentCount.get(st) + 1);
+ } else {
+ currentCount.put(st, 1);
+ }
+ }
+ }
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ @VisibleForTesting
+ HashMap <String, EnumMap<StorageType, Integer>> getChildrenStorageInfo() {
+ return childrenStorageInfo;
+ }
+
+
+ private DFSTopologyNodeImpl createParentNode(String parentName) {
+ return new DFSTopologyNodeImpl(
+ parentName, getPath(this), this, this.getLevel() + 1);
+ }
+
+ @Override
+ public boolean remove(Node n) {
+ if (!isAncestor(n)) {
+ throw new IllegalArgumentException(n.getName()
+ + ", which is located at " + n.getNetworkLocation()
+ + ", is not a descendant of " + getPath(this));
+ }
+ // In HDFS topology, the leaf node should always be DatanodeDescriptor
+ if (!(n instanceof DatanodeDescriptor)) {
+ throw new IllegalArgumentException("Unexpected node type "
+ + n.getClass().getName());
+ }
+ DatanodeDescriptor dnDescriptor = (DatanodeDescriptor) n;
+ if (isParent(n)) {
+ // this node is the parent of n; remove n directly
+ if (childrenMap.containsKey(n.getName())) {
+ for (int i=0; i<children.size(); i++) {
+ if (children.get(i).getName().equals(n.getName())) {
+ children.remove(i);
+ childrenMap.remove(n.getName());
+ synchronized (childrenStorageInfo) {
+ childrenStorageInfo.remove(dnDescriptor.getName());
+ }
+ numOfLeaves--;
+ n.setParent(null);
+ return true;
+ }
+ }
+ }
+ return false;
+ } else {
+ // find the next ancestor node: the parent node
+ String parentName = getNextAncestorName(n);
+ DFSTopologyNodeImpl parentNode =
+ (DFSTopologyNodeImpl)childrenMap.get(parentName);
+ if (parentNode == null) {
+ return false;
+ }
+ // remove n from the parent node
+ boolean isRemoved = parentNode.remove(n);
+ if (isRemoved) {
+ // if the parent node has no children, remove the parent node too
+ synchronized (childrenStorageInfo) {
+ EnumMap<StorageType, Integer> currentCount =
+ childrenStorageInfo.get(parentNode.getName());
+ EnumSet<StorageType> toRemove = EnumSet.noneOf(StorageType.class);
+ for (StorageType st : dnDescriptor.getStorageTypes()) {
+ int newCount = currentCount.get(st) - 1;
+ if (newCount == 0) {
+ toRemove.add(st);
+ }
+ currentCount.put(st, newCount);
+ }
+ for (StorageType st : toRemove) {
+ currentCount.remove(st);
+ }
+ }
+ if (parentNode.getNumOfChildren() == 0) {
+ for(int i=0; i < children.size(); i++) {
+ if (children.get(i).getName().equals(parentName)) {
+ children.remove(i);
+ childrenMap.remove(parentName);
+ childrenStorageInfo.remove(parentNode.getName());
+ break;
+ }
+ }
+ }
+ numOfLeaves--;
+ }
+ return isRemoved;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 6d55c46..a3c0aa4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -1268,7 +1268,7 @@ public class TestBlockStoragePolicy {
}
FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
- conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
File baseDir = PathUtils.getTestDir(TestReplicationPolicy.class);
conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
new File(baseDir, "name").getPath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java
new file mode 100644
index 0000000..ac1edf9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.util.EnumMap;
+import java.util.HashMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This class tests the correctness of storage type info stored in
+ * DFSNetworkTopology.
+ */
+public class TestDFSNetworkTopology {
+ private static final Log LOG =
+ LogFactory.getLog(TestDFSNetworkTopology.class);
+ private final static DFSNetworkTopology CLUSTER =
+ DFSNetworkTopology.getInstance(new Configuration());
+ private DatanodeDescriptor[] dataNodes;
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30000);
+
+ @Before
+ public void setupDatanodes() {
+ final String[] racks = {
+ "/l1/d1/r1", "/l1/d1/r1", "/l1/d1/r2", "/l1/d1/r2", "/l1/d1/r2",
+
+ "/l1/d2/r3", "/l1/d2/r3", "/l1/d2/r3",
+
+ "/l2/d3/r1", "/l2/d3/r2", "/l2/d3/r3", "/l2/d3/r4", "/l2/d3/r5",
+
+ "/l2/d4/r1", "/l2/d4/r1", "/l2/d4/r1", "/l2/d4/r1", "/l2/d4/r1",
+ "/l2/d4/r1", "/l2/d4/r1"};
+ final String[] hosts = {
+ "host1", "host2", "host3", "host4", "host5",
+ "host6", "host7", "host8", "host9", "host10",
+ "host11", "host12", "host13", "host14", "host15",
+ "host16", "host17", "host18", "host19", "host20"};
+ final StorageType[] types = {
+ StorageType.ARCHIVE, StorageType.DISK, StorageType.ARCHIVE,
+ StorageType.DISK, StorageType.DISK,
+
+ StorageType.DISK, StorageType.RAM_DISK, StorageType.SSD,
+
+ StorageType.DISK, StorageType.RAM_DISK, StorageType.DISK,
+ StorageType.ARCHIVE, StorageType.ARCHIVE,
+
+ StorageType.DISK, StorageType.DISK, StorageType.RAM_DISK,
+ StorageType.RAM_DISK, StorageType.ARCHIVE, StorageType.ARCHIVE,
+ StorageType.SSD};
+ final DatanodeStorageInfo[] storages =
+ DFSTestUtil.createDatanodeStorageInfos(20, racks, hosts, types);
+ dataNodes = DFSTestUtil.toDatanodeDescriptor(storages);
+ for (int i = 0; i < dataNodes.length; i++) {
+ CLUSTER.add(dataNodes[i]);
+ }
+ dataNodes[9].setDecommissioned();
+ dataNodes[10].setDecommissioned();
+ }
+
+ /**
+ * Test getting the storage type info of subtree.
+ * @throws Exception
+ */
+ @Test
+ public void testGetStorageTypeInfo() throws Exception {
+ // checking level = 2 nodes
+ DFSTopologyNodeImpl d1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1/d1");
+ HashMap<String, EnumMap<StorageType, Integer>> d1info =
+ d1.getChildrenStorageInfo();
+ assertEquals(2, d1info.keySet().size());
+ assertTrue(d1info.get("r1").size() == 2 && d1info.get("r2").size() == 2);
+ assertEquals(1, (int)d1info.get("r1").get(StorageType.DISK));
+ assertEquals(1, (int)d1info.get("r1").get(StorageType.ARCHIVE));
+ assertEquals(2, (int)d1info.get("r2").get(StorageType.DISK));
+ assertEquals(1, (int)d1info.get("r2").get(StorageType.ARCHIVE));
+
+ DFSTopologyNodeImpl d2 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1/d2");
+ HashMap<String, EnumMap<StorageType, Integer>> d2info =
+ d2.getChildrenStorageInfo();
+ assertEquals(1, d2info.keySet().size());
+ assertTrue(d2info.get("r3").size() == 3);
+ assertEquals(1, (int)d2info.get("r3").get(StorageType.DISK));
+ assertEquals(1, (int)d2info.get("r3").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)d2info.get("r3").get(StorageType.SSD));
+
+ DFSTopologyNodeImpl d3 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l2/d3");
+ HashMap<String, EnumMap<StorageType, Integer>> d3info =
+ d3.getChildrenStorageInfo();
+ assertEquals(5, d3info.keySet().size());
+ assertEquals(1, (int)d3info.get("r1").get(StorageType.DISK));
+ assertEquals(1, (int)d3info.get("r2").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)d3info.get("r3").get(StorageType.DISK));
+ assertEquals(1, (int)d3info.get("r4").get(StorageType.ARCHIVE));
+ assertEquals(1, (int)d3info.get("r5").get(StorageType.ARCHIVE));
+
+ DFSTopologyNodeImpl d4 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l2/d4");
+ HashMap<String, EnumMap<StorageType, Integer>> d4info =
+ d4.getChildrenStorageInfo();
+ assertEquals(1, d4info.keySet().size());
+ assertEquals(2, (int)d4info.get("r1").get(StorageType.DISK));
+ assertEquals(2, (int)d4info.get("r1").get(StorageType.RAM_DISK));
+ assertEquals(2, (int)d4info.get("r1").get(StorageType.ARCHIVE));
+ assertEquals(1, (int)d4info.get("r1").get(StorageType.SSD));
+
+ DFSTopologyNodeImpl l1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1");
+ HashMap<String, EnumMap<StorageType, Integer>> l1info =
+ l1.getChildrenStorageInfo();
+ assertEquals(2, l1info.keySet().size());
+ assertTrue(l1info.get("d1").size() == 2
+ && l1info.get("d2").size() == 3);
+ assertEquals(2, (int)l1info.get("d1").get(StorageType.ARCHIVE));
+ assertEquals(3, (int)l1info.get("d1").get(StorageType.DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.SSD));
+
+ // checking level = 1 nodes
+ DFSTopologyNodeImpl l2 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l2");
+ HashMap<String, EnumMap<StorageType, Integer>> l2info =
+ l2.getChildrenStorageInfo();
+ assertTrue(l2info.get("d3").size() == 3
+ && l2info.get("d4").size() == 4);
+ assertEquals(2, l2info.keySet().size());
+ assertEquals(2, (int)l2info.get("d3").get(StorageType.DISK));
+ assertEquals(2, (int)l2info.get("d3").get(StorageType.ARCHIVE));
+ assertEquals(1, (int)l2info.get("d3").get(StorageType.RAM_DISK));
+ assertEquals(2, (int)l2info.get("d4").get(StorageType.DISK));
+ assertEquals(2, (int)l2info.get("d4").get(StorageType.ARCHIVE));
+ assertEquals(2, (int)l2info.get("d4").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)l2info.get("d4").get(StorageType.SSD));
+ }
+
+ /**
+ * Test the correctness of storage type info when nodes are added and removed.
+ * @throws Exception
+ */
+ @Test
+ public void testAddAndRemoveTopology() throws Exception {
+ String[] newRack = {"/l1/d1/r1", "/l1/d1/r3", "/l1/d3/r3", "/l1/d3/r3"};
+ String[] newHost = {"nhost1", "nhost2", "nhost3", "nhost4"};
+ String[] newips = {"30.30.30.30", "31.31.31.31", "32.32.32.32",
+ "33.33.33.33"};
+ StorageType[] newTypes = {StorageType.DISK, StorageType.SSD,
+ StorageType.SSD, StorageType.SSD};
+ DatanodeDescriptor[] newDD = new DatanodeDescriptor[4];
+
+ for (int i = 0; i<4; i++) {
+ DatanodeStorageInfo dsi = DFSTestUtil.createDatanodeStorageInfo(
+ "s" + newHost[i], newips[i], newRack[i], newHost[i],
+ newTypes[i], null);
+ newDD[i] = dsi.getDatanodeDescriptor();
+ CLUSTER.add(newDD[i]);
+ }
+
+ DFSTopologyNodeImpl d1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1/d1");
+ HashMap<String, EnumMap<StorageType, Integer>> d1info =
+ d1.getChildrenStorageInfo();
+ assertEquals(3, d1info.keySet().size());
+ assertTrue(d1info.get("r1").size() == 2 && d1info.get("r2").size() == 2
+ && d1info.get("r3").size() == 1);
+ assertEquals(2, (int)d1info.get("r1").get(StorageType.DISK));
+ assertEquals(1, (int)d1info.get("r1").get(StorageType.ARCHIVE));
+ assertEquals(2, (int)d1info.get("r2").get(StorageType.DISK));
+ assertEquals(1, (int)d1info.get("r2").get(StorageType.ARCHIVE));
+ assertEquals(1, (int)d1info.get("r3").get(StorageType.SSD));
+
+ DFSTopologyNodeImpl d3 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1/d3");
+ HashMap<String, EnumMap<StorageType, Integer>> d3info =
+ d3.getChildrenStorageInfo();
+ assertEquals(1, d3info.keySet().size());
+ assertTrue(d3info.get("r3").size() == 1);
+ assertEquals(2, (int)d3info.get("r3").get(StorageType.SSD));
+
+ DFSTopologyNodeImpl l1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1");
+ HashMap<String, EnumMap<StorageType, Integer>> l1info =
+ l1.getChildrenStorageInfo();
+ assertEquals(3, l1info.keySet().size());
+ assertTrue(l1info.get("d1").size() == 3 &&
+ l1info.get("d2").size() == 3 && l1info.get("d3").size() == 1);
+ assertEquals(4, (int)l1info.get("d1").get(StorageType.DISK));
+ assertEquals(2, (int)l1info.get("d1").get(StorageType.ARCHIVE));
+ assertEquals(1, (int)l1info.get("d1").get(StorageType.SSD));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.SSD));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.DISK));
+ assertEquals(2, (int)l1info.get("d3").get(StorageType.SSD));
+
+
+ for (int i = 0; i<4; i++) {
+ CLUSTER.remove(newDD[i]);
+ }
+
+ // /d1/r3 should've been out, /d1/r1 should've been resumed
+ DFSTopologyNodeImpl nd1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1/d1");
+ HashMap<String, EnumMap<StorageType, Integer>> nd1info =
+ nd1.getChildrenStorageInfo();
+ assertEquals(2, nd1info.keySet().size());
+ assertTrue(nd1info.get("r1").size() == 2 && nd1info.get("r2").size() == 2);
+ assertEquals(1, (int)nd1info.get("r1").get(StorageType.DISK));
+ assertEquals(1, (int)nd1info.get("r1").get(StorageType.ARCHIVE));
+ assertEquals(2, (int)nd1info.get("r2").get(StorageType.DISK));
+ assertEquals(1, (int)nd1info.get("r2").get(StorageType.ARCHIVE));
+
+ // /l1/d3 should've been out, and /l1/d1 should've been resumed
+ DFSTopologyNodeImpl nl1 =
+ (DFSTopologyNodeImpl) CLUSTER.getNode("/l1");
+ HashMap<String, EnumMap<StorageType, Integer>> nl1info =
+ nl1.getChildrenStorageInfo();
+ assertEquals(2, nl1info.keySet().size());
+ assertTrue(l1info.get("d1").size() == 2
+ && l1info.get("d2").size() == 3);
+ assertEquals(2, (int)nl1info.get("d1").get(StorageType.ARCHIVE));
+ assertEquals(3, (int)nl1info.get("d1").get(StorageType.DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.RAM_DISK));
+ assertEquals(1, (int)l1info.get("d2").get(StorageType.SSD));
+
+ assertNull(CLUSTER.getNode("/l1/d3"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eeca8b0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
index 62bd262..923a73f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
@@ -47,7 +47,8 @@ import org.junit.rules.Timeout;
public class TestNetworkTopology {
private static final Log LOG = LogFactory.getLog(TestNetworkTopology.class);
- private final static NetworkTopology cluster = new NetworkTopology();
+ private final static NetworkTopology cluster =
+ NetworkTopology.getInstance(new Configuration());
private DatanodeDescriptor dataNodes[];
@Rule
@@ -101,7 +102,8 @@ public class TestNetworkTopology {
@Test
public void testCreateInvalidTopology() throws Exception {
- NetworkTopology invalCluster = new NetworkTopology();
+ NetworkTopology invalCluster =
+ NetworkTopology.getInstance(new Configuration());
DatanodeDescriptor invalDataNodes[] = new DatanodeDescriptor[] {
DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1"),
DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1"),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] hadoop git commit: YARN-6271. yarn rmadin -getGroups returns
information from standby RM. Contributed by Jian He.
Posted by kk...@apache.org.
YARN-6271. yarn rmadin -getGroups returns information from standby RM. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8db7a8c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8db7a8c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8db7a8c3
Branch: refs/heads/YARN-5972
Commit: 8db7a8c3aea3d989361f32cca5b271e9653773b6
Parents: ac5ae00
Author: Junping Du <ju...@apache.org>
Authored: Fri Mar 3 14:39:31 2017 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Fri Mar 3 14:39:31 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/server/resourcemanager/AdminService.java | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8db7a8c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 9dda57e..bcf7309 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -572,6 +572,15 @@ public class AdminService extends CompositeService implements
@Override
public String[] getGroupsForUser(String user) throws IOException {
+ String operation = "getGroupsForUser";
+ UserGroupInformation ugi;
+ try {
+ ugi = checkAcls(operation);
+ } catch (YarnException e) {
+ // The interface is from hadoop-common which does not accept YarnException
+ throw new IOException(e);
+ }
+ checkRMStatus(ugi.getShortUserName(), operation, "get groups for user");
return UserGroupInformation.createRemoteUser(user).getGroupNames();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] hadoop git commit: HADOOP-13930. Azure: Add Authorization
support to WASB. Contributed by Sivaguru Sankaridurg and Dushyanth
Posted by kk...@apache.org.
HADOOP-13930. Azure: Add Authorization support to WASB. Contributed by Sivaguru Sankaridurg and Dushyanth
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68682352
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68682352
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68682352
Branch: refs/heads/YARN-5972
Commit: 686823529be09bea2a6cecb3503ef722017475bc
Parents: 52d7d5a
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Mar 6 17:16:36 2017 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Mar 6 17:16:36 2017 -0800
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 10 +
.../conf/TestCommonConfigurationFields.java | 1 +
.../fs/azure/AzureNativeFileSystemStore.java | 5 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 116 ++++++-
.../fs/azure/RemoteWasbAuthorizerImpl.java | 190 ++++++++++
.../fs/azure/WasbAuthorizationException.java | 40 +++
.../fs/azure/WasbAuthorizationOperations.java | 44 +++
.../fs/azure/WasbAuthorizerInterface.java | 53 +++
.../hadoop/fs/azure/WasbRemoteCallHelper.java | 71 +++-
.../hadoop-azure/src/site/markdown/index.md | 34 ++
.../fs/azure/AzureBlobStorageTestAccount.java | 61 ++--
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 102 ++++++
.../TestNativeAzureFileSystemAuthorization.java | 344 +++++++++++++++++++
.../fs/azure/TestWasbRemoteCallHelper.java | 344 +++++++++++++++++++
.../src/test/resources/azure-test.xml | 28 +-
15 files changed, 1373 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 35be56b..52b58ed 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1292,6 +1292,16 @@
to specify the time (such as 2s, 2m, 1h, etc.).
</description>
</property>
+<property>
+ <name>fs.azure.authorization</name>
+ <value>false</value>
+ <description>
+ Config flag to enable authorization support in WASB. Setting it to "true" enables
+ authorization support to WASB. Currently WASB authorization requires a remote service
+ to provide authorization that needs to be specified via fs.azure.authorization.remote.service.url
+ configuration
+ </description>
+</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 966a8ac..cbfb6d1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -114,6 +114,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPropsToSkipCompare.add("fs.azure.sas.expiry.period");
xmlPropsToSkipCompare.add("fs.azure.local.sas.key.mode");
xmlPropsToSkipCompare.add("fs.azure.secure.mode");
+ xmlPropsToSkipCompare.add("fs.azure.authorization");
// Deprecated properties. These should eventually be removed from the
// class.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 07c389c..a8708ec 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -249,7 +249,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
* Default values to control SAS Key mode.
* By default we set the values to false.
*/
- private static final boolean DEFAULT_USE_SECURE_MODE = false;
+ public static final boolean DEFAULT_USE_SECURE_MODE = false;
private static final boolean DEFAULT_USE_LOCAL_SAS_KEY_MODE = false;
/**
@@ -849,6 +849,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
rootDirectory = container.getDirectoryReference("");
canCreateOrModifyContainer = true;
+
+ configureAzureStorageSession();
+ tolerateOobAppends = false;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 4184a53..6de0a28 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -1106,7 +1106,31 @@ public class NativeAzureFileSystem extends FileSystem {
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
private boolean appendSupportEnabled = false;
-
+
+ /**
+ * Configuration key to enable authorization support in WASB.
+ */
+ public static final String KEY_AZURE_AUTHORIZATION =
+ "fs.azure.authorization";
+
+ /**
+ * Default value for the authorization support in WASB.
+ */
+ private static final boolean DEFAULT_AZURE_AUTHORIZATION = false;
+
+ /**
+ * Flag controlling authorization support in WASB.
+ */
+ private boolean azureAuthorization = false;
+
+ /**
+ * Authorizer to use when authorization support is enabled in
+ * WASB.
+ */
+ private WasbAuthorizerInterface authorizer = null;
+
+ private String delegationToken = null;
+
public NativeAzureFileSystem() {
// set store in initialize()
}
@@ -1146,11 +1170,11 @@ public class NativeAzureFileSystem extends FileSystem {
return baseName + number;
}
}
-
+
/**
* Checks if the given URI scheme is a scheme that's affiliated with the Azure
* File System.
- *
+ *
* @param scheme
* The URI scheme.
* @return true iff it's an Azure File System URI scheme.
@@ -1167,7 +1191,7 @@ public class NativeAzureFileSystem extends FileSystem {
/**
* Puts in the authority of the default file system if it is a WASB file
* system and the given URI's authority is null.
- *
+ *
* @return The URI with reconstructed authority if necessary and possible.
*/
private static URI reconstructAuthorityIfNeeded(URI uri, Configuration conf) {
@@ -1237,6 +1261,24 @@ public class NativeAzureFileSystem extends FileSystem {
// Initialize thread counts from user configuration
deleteThreadCount = conf.getInt(AZURE_DELETE_THREADS, DEFAULT_AZURE_DELETE_THREADS);
renameThreadCount = conf.getInt(AZURE_RENAME_THREADS, DEFAULT_AZURE_RENAME_THREADS);
+
+ boolean useSecureMode = conf.getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE,
+ AzureNativeFileSystemStore.DEFAULT_USE_SECURE_MODE);
+
+ this.azureAuthorization = useSecureMode &&
+ conf.getBoolean(KEY_AZURE_AUTHORIZATION, DEFAULT_AZURE_AUTHORIZATION);
+
+ if (this.azureAuthorization) {
+
+ this.authorizer =
+ new RemoteWasbAuthorizerImpl();
+ authorizer.init(conf);
+ }
+ }
+
+ @VisibleForTesting
+ public void updateWasbAuthorizer(WasbAuthorizerInterface authorizer) {
+ this.authorizer = authorizer;
}
private NativeFileSystemStore createDefaultStore(Configuration conf) {
@@ -1338,18 +1380,28 @@ public class NativeAzureFileSystem extends FileSystem {
/**
* For unit test purposes, retrieves the AzureNativeFileSystemStore store
* backing this file system.
- *
+ *
* @return The store object.
*/
@VisibleForTesting
public AzureNativeFileSystemStore getStore() {
return actualStore;
}
-
+
NativeFileSystemStore getStoreInterface() {
return store;
}
+ private void performAuthCheck(String path, String accessType,
+ String operation) throws WasbAuthorizationException, IOException {
+
+ if (azureAuthorization && this.authorizer != null &&
+ !this.authorizer.authorize(path, accessType, delegationToken)) {
+ throw new WasbAuthorizationException(operation
+ + " operation for Path : " + path + " not allowed");
+ }
+ }
+
/**
* Gets the metrics source for this file system.
* This is mainly here for unit testing purposes.
@@ -1372,6 +1424,10 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Opening file: {} for append", f);
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.WRITE.toString(), "append");
+
String key = pathToKey(absolutePath);
FileMetadata meta = null;
try {
@@ -1434,6 +1490,7 @@ public class NativeAzureFileSystem extends FileSystem {
* Get a self-renewing lease on the specified file.
* @param path path whose lease to be renewed.
* @return Lease
+ * @throws AzureException when not being able to acquire a lease on the path
*/
public SelfRenewingLease acquireLease(Path path) throws AzureException {
String fullKey = pathToKey(makeAbsolute(path));
@@ -1572,6 +1629,10 @@ public class NativeAzureFileSystem extends FileSystem {
}
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.WRITE.toString(), "create");
+
String key = pathToKey(absolutePath);
FileMetadata existingMetadata = store.retrieveMetadata(key);
@@ -1652,10 +1713,10 @@ public class NativeAzureFileSystem extends FileSystem {
// Construct the data output stream from the buffered output stream.
FSDataOutputStream fsOut = new FSDataOutputStream(bufOutStream, statistics);
-
+
// Increment the counter
instrumentation.fileCreated();
-
+
// Return data output stream to caller.
return fsOut;
}
@@ -1694,6 +1755,10 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Deleting file: {}", f.toString());
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "delete");
+
String key = pathToKey(absolutePath);
// Capture the metadata for the path.
@@ -1964,6 +2029,10 @@ public class NativeAzureFileSystem extends FileSystem {
// Capture the absolute path and the path to key.
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "getFileStatus");
+
String key = pathToKey(absolutePath);
if (key.length() == 0) { // root always exists
return newDirectory(null, absolutePath);
@@ -2062,6 +2131,10 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Listing status for {}", f.toString());
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "list");
+
String key = pathToKey(absolutePath);
Set<FileStatus> status = new TreeSet<FileStatus>();
FileMetadata meta = null;
@@ -2228,7 +2301,7 @@ public class NativeAzureFileSystem extends FileSystem {
/**
* Applies the applicable UMASK's on the given permission.
- *
+ *
* @param permission
* The permission to mask.
* @param applyMode
@@ -2250,7 +2323,7 @@ public class NativeAzureFileSystem extends FileSystem {
/**
* Creates the PermissionStatus object to use for the given permission, based
* on the current user in context.
- *
+ *
* @param permission
* The permission for the file.
* @return The permission status object to use.
@@ -2284,6 +2357,10 @@ public class NativeAzureFileSystem extends FileSystem {
}
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "mkdirs");
+
PermissionStatus permissionStatus = null;
if(noUmask) {
// ensure owner still has wx permissions at the minimum
@@ -2337,6 +2414,10 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Opening file: {}", f.toString());
Path absolutePath = makeAbsolute(f);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.READ.toString(), "read");
+
String key = pathToKey(absolutePath);
FileMetadata meta = null;
try {
@@ -2393,7 +2474,12 @@ public class NativeAzureFileSystem extends FileSystem {
+ " through WASB that has colons in the name");
}
- String srcKey = pathToKey(makeAbsolute(src));
+ Path absolutePath = makeAbsolute(src);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "rename");
+
+ String srcKey = pathToKey(absolutePath);
if (srcKey.length() == 0) {
// Cannot rename root of file system
@@ -2695,6 +2781,10 @@ public class NativeAzureFileSystem extends FileSystem {
@Override
public void setPermission(Path p, FsPermission permission) throws FileNotFoundException, IOException {
Path absolutePath = makeAbsolute(p);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "setPermission");
+
String key = pathToKey(absolutePath);
FileMetadata metadata = null;
try {
@@ -2733,6 +2823,10 @@ public class NativeAzureFileSystem extends FileSystem {
public void setOwner(Path p, String username, String groupname)
throws IOException {
Path absolutePath = makeAbsolute(p);
+
+ performAuthCheck(absolutePath.toString(),
+ WasbAuthorizationOperations.EXECUTE.toString(), "setOwner");
+
String key = pathToKey(absolutePath);
FileMetadata metadata = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
new file mode 100644
index 0000000..5f2265b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.azure.WasbRemoteCallHelper.REMOTE_CALL_SUCCESS_CODE;
+
+/**
+ * Class implementing WasbAuthorizerInterface using a remote
+ * service that implements the authorization operation. This
+ * class expects the url of the remote service to be passed
+ * via config.
+ */
+public class RemoteWasbAuthorizerImpl implements WasbAuthorizerInterface {
+
+ private String remoteAuthorizerServiceUrl = "";
+
+ /**
+ * Configuration parameter name expected in the Configuration object to
+ * provide the url of the remote service. {@value}
+ */
+ public static final String KEY_REMOTE_AUTH_SERVICE_URL =
+ "fs.azure.authorization.remote.service.url";
+
+ /**
+ * Authorization operation OP name in the remote service {@value}
+ */
+ private static final String CHECK_AUTHORIZATION_OP =
+ "CHECK_AUTHORIZATION";
+
+ /**
+ * Query parameter specifying the access operation type. {@value}
+ */
+ private static final String ACCESS_OPERATION_QUERY_PARAM_NAME =
+ "operation_type";
+
+ /**
+ * Query parameter specifying the wasb absolute path. {@value}
+ */
+ private static final String WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME =
+ "wasb_absolute_path";
+
+ /**
+ * Query parameter name for user info {@value}
+ */
+ private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME =
+ "delegation_token";
+
+ private WasbRemoteCallHelper remoteCallHelper = null;
+
+ @VisibleForTesting
+ public void updateWasbRemoteCallHelper(WasbRemoteCallHelper helper) {
+ this.remoteCallHelper = helper;
+ }
+
+ @Override
+ public void init(Configuration conf)
+ throws WasbAuthorizationException, IOException {
+
+ remoteAuthorizerServiceUrl = conf.get(KEY_REMOTE_AUTH_SERVICE_URL);
+
+ if (remoteAuthorizerServiceUrl == null
+ || remoteAuthorizerServiceUrl.isEmpty()) {
+ throw new WasbAuthorizationException(
+ "fs.azure.authorization.remote.service.url config not set"
+ + " in configuration.");
+ }
+
+ this.remoteCallHelper = new WasbRemoteCallHelper();
+ }
+
+ @Override
+ public boolean authorize(String wasbAbsolutePath, String accessType,
+ String delegationToken) throws WasbAuthorizationException, IOException {
+
+ try {
+ URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl);
+ uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP);
+ uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME,
+ wasbAbsolutePath);
+ uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME,
+ accessType);
+ uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
+ delegationToken);
+
+ String responseBody = remoteCallHelper.makeRemoteGetRequest(
+ new HttpGet(uriBuilder.build()));
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ RemoteAuthorizerResponse authorizerResponse =
+ objectMapper.readValue(responseBody, RemoteAuthorizerResponse.class);
+
+ if (authorizerResponse == null) {
+ throw new WasbAuthorizationException(
+ "RemoteAuthorizerResponse object null from remote call");
+ } else if (authorizerResponse.getResponseCode()
+ == REMOTE_CALL_SUCCESS_CODE) {
+ return authorizerResponse.getAuthorizationResult();
+ } else {
+ throw new WasbAuthorizationException("Remote authorization"
+ + " service encountered an error "
+ + authorizerResponse.getResponseMessage());
+ }
+ } catch (URISyntaxException | WasbRemoteCallException
+ | JsonParseException | JsonMappingException ex) {
+ throw new WasbAuthorizationException(ex);
+ }
+ }
+}
+
+/**
+ * POJO representing the response expected from a remote
+ * authorization service.
+ * The remote service is expected to return the authorization
+ * response in the following JSON format
+ * {
+ * "responseCode" : 0 or non-zero <int>,
+ * "responseMessage" : relavant message of failure <String>
+ * "authorizationResult" : authorization result <boolean>
+ * true - if auhorization allowed
+ * false - otherwise.
+ *
+ * }
+ */
+class RemoteAuthorizerResponse {
+
+ private int responseCode;
+ private boolean authorizationResult;
+ private String responseMessage;
+
+ public RemoteAuthorizerResponse(int responseCode,
+ boolean authorizationResult, String message) {
+ this.responseCode = responseCode;
+ this.authorizationResult = authorizationResult;
+ this.responseMessage = message;
+ }
+
+ public RemoteAuthorizerResponse() {
+ }
+
+ public int getResponseCode() {
+ return responseCode;
+ }
+
+ public void setResponseCode(int responseCode) {
+ this.responseCode = responseCode;
+ }
+
+ public boolean getAuthorizationResult() {
+ return authorizationResult;
+ }
+
+ public void setAuthorizationResult(boolean authorizationResult) {
+ this.authorizationResult = authorizationResult;
+ }
+
+ public String getResponseMessage() {
+ return responseMessage;
+ }
+
+ public void setResponseMessage(String message) {
+ this.responseMessage = message;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
new file mode 100644
index 0000000..eff9248
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+/**
+ * Exception that gets thrown during the authorization failures
+ * in WASB.
+ */
+public class WasbAuthorizationException extends AzureException {
+
+ private static final long serialVersionUID = 1L;
+
+ public WasbAuthorizationException(String message) {
+ super(message);
+ }
+
+ public WasbAuthorizationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public WasbAuthorizationException(Throwable t) {
+ super(t);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
new file mode 100644
index 0000000..41ca2b3
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+/**
+ * Different authorization operations supported
+ * in WASB.
+ */
+
+public enum WasbAuthorizationOperations {
+
+ READ, WRITE, EXECUTE;
+
+ @Override
+ public String toString() {
+ switch(this) {
+ case READ:
+ return "read";
+ case WRITE:
+ return "write";
+ case EXECUTE:
+ return "execute";
+ default:
+ throw new IllegalArgumentException(
+ "Invalid Authorization Operation");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
new file mode 100644
index 0000000..f391851
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Interface to implement authorization support in WASB.
+ * API's of this interface will be implemented in the
+ * StorageInterface Layer before making calls to Azure
+ * Storage.
+ */
+public interface WasbAuthorizerInterface {
+ /**
+ * Initializer method
+ * @param conf - Configuration object
+ * @throws WasbAuthorizationException - On authorization exceptions
+ * @throws IOException - When not able to reach the authorizer
+ */
+ public void init(Configuration conf)
+ throws WasbAuthorizationException, IOException;
+
+ /**
+ * Authorizer API to authorize access in WASB.
+
+ * @param wasbAbolutePath : Absolute WASB Path used for access.
+ * @param accessType : Type of access
+ * @param delegationToken : The user information.
+ * @return : true - If access allowed false - If access is not allowed.
+ * @throws WasbAuthorizationException - On authorization exceptions
+ * @throws IOException - When not able to reach the authorizer
+ */
+ public boolean authorize(String wasbAbolutePath, String accessType,
+ String delegationToken) throws WasbAuthorizationException, IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
index 543c899..09ea084 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
@@ -18,18 +18,21 @@
package org.apache.hadoop.fs.azure;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
+import org.apache.http.StatusLine;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+
/**
* Helper class the has constants and helper methods
* used in WASB when integrating with a remote http cred
@@ -48,6 +51,11 @@ class WasbRemoteCallHelper {
*/
private HttpClient client = null;
+ @VisibleForTesting
+ public void updateHttpClient(HttpClient client) {
+ this.client = client;
+ }
+
public WasbRemoteCallHelper() {
this.client = HttpClientBuilder.create().build();
}
@@ -58,17 +66,54 @@ class WasbRemoteCallHelper {
* @return Http Response body returned as a string. The caller
* is expected to semantically understand the response.
* @throws WasbRemoteCallException
+ * @throws IOException
*/
public String makeRemoteGetRequest(HttpGet getRequest)
- throws WasbRemoteCallException {
+ throws WasbRemoteCallException, IOException {
try {
+ final String APPLICATION_JSON = "application/json";
+ final int MAX_CONTENT_LENGTH = 1024;
+
+ getRequest.setHeader("Accept", APPLICATION_JSON);
+
HttpResponse response = client.execute(getRequest);
- if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
- throw new WasbRemoteCallException(
- response.getStatusLine().toString());
+ StatusLine statusLine = response.getStatusLine();
+ if (statusLine == null || statusLine.getStatusCode() != HttpStatus.SC_OK) {
+ throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" +
+ ((statusLine!=null) ? statusLine.toString() : "NULL")
+ );
+ }
+
+ Header contentTypeHeader = response.getFirstHeader("Content-Type");
+ if (contentTypeHeader == null || contentTypeHeader.getValue() != APPLICATION_JSON) {
+ throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" +
+ "Content-Type mismatch: expected: " + APPLICATION_JSON +
+ ", got " + ((contentTypeHeader!=null) ? contentTypeHeader.getValue() : "NULL")
+ );
+ }
+
+ Header contentLengthHeader = response.getFirstHeader("Content-Length");
+ if (contentLengthHeader == null) {
+ throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" +
+ "Content-Length header missing"
+ );
+ }
+
+ try {
+ if (Integer.parseInt(contentLengthHeader.getValue()) > MAX_CONTENT_LENGTH) {
+ throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" +
+ "Content-Length:" + contentLengthHeader.getValue() +
+ "exceeded max:" + MAX_CONTENT_LENGTH
+ );
+ }
+ }
+ catch (NumberFormatException nfe) {
+ throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" +
+ "Invalid Content-Length value :" + contentLengthHeader.getValue()
+ );
}
BufferedReader rd = new BufferedReader(
@@ -83,11 +128,11 @@ class WasbRemoteCallHelper {
return responseBody.toString();
} catch (ClientProtocolException clientProtocolEx) {
- throw new WasbRemoteCallException("Encountered ClientProtocolException"
- + " while making remote call", clientProtocolEx);
+ throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" +
+ "Encountered ClientProtocolException while making remote call", clientProtocolEx);
} catch (IOException ioEx) {
- throw new WasbRemoteCallException("Encountered IOException while making"
- + " remote call", ioEx);
+ throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" +
+ "Encountered IOException while making remote call", ioEx);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index 2865223..1d1274b 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -333,6 +333,40 @@ The service is expected to return a response in JSON format:
"sasKey" : Requested SAS Key <String>
}
```
+
+## <a name="WASB Authorization" />Authorization Support in WASB.
+
+Authorization support can be enabled in WASB using the following configuration:
+
+```
+ <property>
+ <name>fs.azure.authorization</name>
+ <value>true</value>
+ </property>
+```
+ The current implementation of authorization relies on the presence of an external service that can enforce
+ the authorization. The service is expected to be running on a URL provided by the following config.
+
+```
+ <property>
+ <name>fs.azure.authorization.remote.service.url</name>
+ <value>{URL}</value>
+ </property>
+```
+
+ The remote service is expected to provide support for the following REST call: ```{URL}/CHECK_AUTHORIZATION```
+ An example request:
+ ```{URL}/CHECK_AUTHORIZATION?wasb_absolute_path=<absolute_path>&operation_type=<operation type>&delegation_token=<delegation token>```
+
+ The service is expected to return a response in JSON format:
+ ```
+ {
+ "responseCode" : 0 or non-zero <int>,
+ "responseMessage" : relevant message on failure <String>,
+ "authorizationResult" : true/false <boolean>
+ }
+ ```
+
## <a name="Testing_the_hadoop-azure_Module" />Testing the hadoop-azure Module
The hadoop-azure module includes a full suite of unit tests. Most of the tests
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
index 5353663..5f66fd2f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
@@ -18,17 +18,9 @@
package org.apache.hadoop.fs.azure;
-import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.GregorianCalendar;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
+import com.microsoft.azure.storage.*;
+import com.microsoft.azure.storage.blob.*;
+import com.microsoft.azure.storage.core.Base64;
import org.apache.commons.configuration2.SubsetConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -39,22 +31,14 @@ import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsSink;
import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.impl.TestMetricsConfig;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-
-import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.StorageCredentials;
-import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
-import com.microsoft.azure.storage.StorageCredentialsAnonymous;
-import com.microsoft.azure.storage.blob.BlobContainerPermissions;
-import com.microsoft.azure.storage.blob.BlobContainerPublicAccessType;
-import com.microsoft.azure.storage.blob.BlobOutputStream;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
-import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
-import com.microsoft.azure.storage.core.Base64;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME;
+import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_LOCAL_SAS_KEY_MODE;
/**
* Helper class to create WASB file systems backed by either a mock in-memory
@@ -92,7 +76,7 @@ public final class AzureBlobStorageTestAccount {
private static final ConcurrentLinkedQueue<MetricsRecord> allMetrics =
new ConcurrentLinkedQueue<MetricsRecord>();
private static boolean metricsConfigSaved = false;
-
+
private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
CloudStorageAccount account,
CloudBlobContainer container) {
@@ -272,6 +256,7 @@ public final class AzureBlobStorageTestAccount {
store.setAzureStorageInteractionLayer(mockStorage);
NativeAzureFileSystem fs = new NativeAzureFileSystem(store);
setMockAccountKey(conf);
+ configureSecureModeTestSettings(conf);
// register the fs provider.
fs.initialize(new URI(MOCK_WASB_URI), conf);
@@ -332,6 +317,8 @@ public final class AzureBlobStorageTestAccount {
// Set account URI and initialize Azure file system.
URI accountUri = createAccountUri(DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME,
containerName);
+ configureSecureModeTestSettings(conf);
+
fs.initialize(accountUri, conf);
// Create test account initializing the appropriate member variables.
@@ -368,6 +355,7 @@ public final class AzureBlobStorageTestAccount {
// out-of-band appends.
conf.setBoolean(KEY_DISABLE_THROTTLING, true);
conf.setBoolean(KEY_READ_TOLERATE_CONCURRENT_APPEND, true);
+ configureSecureModeTestSettings(conf);
// Set account URI and initialize Azure file system.
URI accountUri = createAccountUri(accountName, containerName);
@@ -409,6 +397,17 @@ public final class AzureBlobStorageTestAccount {
}
/**
+ * Configure default values for Secure Mode testing.
+ * These values are relevant only when testing in Secure Mode.
+ *
+ * @param conf
+ * The configuration.
+ */
+ public static void configureSecureModeTestSettings(Configuration conf) {
+ conf.set(KEY_USE_LOCAL_SAS_KEY_MODE, "true"); // always use local sas-key mode for testing
+ }
+
+ /**
* Sets the mock account key in the given configuration.
*
* @param conf
@@ -556,6 +555,8 @@ public final class AzureBlobStorageTestAccount {
conf.setBoolean(KEY_DISABLE_THROTTLING, true);
}
+ configureSecureModeTestSettings(conf);
+
// Set account URI and initialize Azure file system.
URI accountUri = createAccountUri(accountName, containerName);
fs.initialize(accountUri, conf);
@@ -693,6 +694,8 @@ public final class AzureBlobStorageTestAccount {
// Capture the account URL and the account name.
String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+ configureSecureModeTestSettings(conf);
+
// Generate a container name and create a shared access signature string for
// it.
//
@@ -764,6 +767,8 @@ public final class AzureBlobStorageTestAccount {
// Capture the account URL and the account name.
String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+ configureSecureModeTestSettings(conf);
+
// Set up public container with the specified blob name.
CloudBlockBlob blobRoot = primeRootContainer(blobClient, accountName,
blobName, fileSize);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
new file mode 100644
index 0000000..8f7cb2a
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * A mock wasb authorizer implementation.
+ */
+
+public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface {
+
+ private Map<AuthorizationComponent, Boolean> authRules;
+
+ @Override
+ public void init(Configuration conf) {
+ authRules = new HashMap<AuthorizationComponent, Boolean>();
+ }
+
+ public void addAuthRule(String wasbAbsolutePath,
+ String accessType, boolean access) {
+ AuthorizationComponent component =
+ new AuthorizationComponent(wasbAbsolutePath, accessType);
+ this.authRules.put(component, access);
+ }
+
+ @Override
+ public boolean authorize(String wasbAbsolutePath, String accessType,
+ String delegationToken) throws WasbAuthorizationException {
+
+ AuthorizationComponent component =
+ new AuthorizationComponent(wasbAbsolutePath, accessType);
+
+ if (authRules.containsKey(component)) {
+ return authRules.get(component);
+ } else {
+ return false;
+ }
+ }
+}
+
+class AuthorizationComponent {
+
+ private String wasbAbsolutePath;
+ private String accessType;
+
+ public AuthorizationComponent(String wasbAbsolutePath,
+ String accessType) {
+ this.wasbAbsolutePath = wasbAbsolutePath;
+ this.accessType = accessType;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.wasbAbsolutePath.hashCode() ^ this.accessType.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj == null
+ || !(obj instanceof AuthorizationComponent)) {
+ return false;
+ }
+
+ return ((AuthorizationComponent)obj).
+ getWasbAbsolutePath().equals(this.wasbAbsolutePath)
+ && ((AuthorizationComponent)obj).
+ getAccessType().equals(this.accessType);
+ }
+
+ public String getWasbAbsolutePath() {
+ return this.wasbAbsolutePath;
+ }
+
+ public String getAccessType() {
+ return accessType;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
new file mode 100644
index 0000000..a2bbeb1
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.sun.tools.javac.util.Assert;
+import org.junit.rules.ExpectedException;
+
+import java.io.Console;
+
+import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE;
+
+/**
+ * Test class to hold all WASB authorization tests.
+ */
+public class TestNativeAzureFileSystemAuthorization
+ extends AbstractWasbTestBase {
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true");
+ conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URL, "http://localhost/");
+ return AzureBlobStorageTestAccount.create(conf);
+ }
+
+
+ @Before
+ public void beforeMethod() {
+ boolean useSecureMode = fs.getConf().getBoolean(KEY_USE_SECURE_MODE, false);
+ boolean useAuthorization = fs.getConf().getBoolean(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, false);
+ Assume.assumeTrue("Test valid when both SecureMode and Authorization are enabled .. skipping",
+ useSecureMode && useAuthorization);
+
+ Assume.assumeTrue(
+ useSecureMode && useAuthorization
+ );
+ }
+
+
+ @Rule
+ public ExpectedException expectedEx = ExpectedException.none();
+
+ /**
+ * Positive test to verify Create and delete access check
+ * @throws Throwable
+ */
+ @Test
+ public void testCreateAccessCheckPositive() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ Path parentDir = new Path("/testCreateAccessCheckPositive");
+ Path testPath = new Path(parentDir, "test.dat");
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ fs.create(testPath);
+ ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath);
+ fs.delete(parentDir, true);
+ }
+
+ /**
+ * Negative test to verify Create access check
+ * @throws Throwable
+ */
+
+ @Test // (expected=WasbAuthorizationException.class)
+ public void testCreateAccessCheckNegative() throws Throwable {
+
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage("create operation for Path : /testCreateAccessCheckNegative/test.dat not allowed");
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ Path parentDir = new Path("/testCreateAccessCheckNegative");
+ Path testPath = new Path(parentDir, "test.dat");
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(),WasbAuthorizationOperations.WRITE.toString(), false);
+ authorizer.addAuthRule(parentDir.toString(),WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testPath);
+ }
+ finally {
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Positive test to verify Create and delete access check
+ * @throws Throwable
+ */
+ @Test
+ public void testListAccessCheckPositive() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ Path parentDir = new Path("/testListAccessCheckPositive");
+ Path testPath = new Path(parentDir, "test.dat");
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ fs.create(testPath);
+ ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath);
+
+ try {
+ fs.listStatus(testPath);
+ }
+ finally {
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Negative test to verify Create access check
+ * @throws Throwable
+ */
+
+ @Test //(expected=WasbAuthorizationException.class)
+ public void testListAccessCheckNegative() throws Throwable {
+
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage("getFileStatus operation for Path : /testListAccessCheckNegative/test.dat not allowed");
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ Path parentDir = new Path("/testListAccessCheckNegative");
+ Path testPath = new Path(parentDir, "test.dat");
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), false);
+ authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ fs.create(testPath);
+ ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath);
+
+ try {
+ fs.listStatus(testPath);
+ }
+ finally {
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Positive test to verify rename access check.
+ * @throws Throwable
+ */
+ @Test
+ public void testRenameAccessCheckPositive() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ Path parentDir = new Path("/testRenameAccessCheckPositive");
+ Path testPath = new Path(parentDir, "test.dat");
+ Path renamePath = new Path(parentDir, "test2.dat");
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(renamePath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ fs.create(testPath);
+ ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", testPath);
+
+ try {
+ fs.rename(testPath, renamePath);
+ ContractTestUtils.assertPathExists(fs, "destPath does not exist", renamePath);
+ }
+ finally {
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Negative test to verify rename access check.
+ * @throws Throwable
+ */
+ @Test //(expected=WasbAuthorizationException.class)
+ public void testRenameAccessCheckNegative() throws Throwable {
+
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage("rename operation for Path : /testRenameAccessCheckNegative/test.dat not allowed");
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+ Path parentDir = new Path("/testRenameAccessCheckNegative");
+ Path testPath = new Path(parentDir, "test.dat");
+ Path renamePath = new Path(parentDir, "test2.dat");
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ // set EXECUTE to true for initial assert right after creation.
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ fs.create(testPath);
+ ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", testPath);
+
+ // Set EXECUTE to false for actual rename-failure test
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), false);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.rename(testPath, renamePath);
+ ContractTestUtils.assertPathExists(fs, "destPath does not exist", renamePath);
+ } finally {
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Positive test for read access check.
+ * @throws Throwable
+ */
+ @Test
+ public void testReadAccessCheckPositive() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+ Path parentDir = new Path("/testReadAccessCheckPositive");
+ Path testPath = new Path(parentDir, "test.dat");
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
+ authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ FSDataOutputStream fso = fs.create(testPath);
+ String data = "Hello World";
+ fso.writeBytes(data);
+ fso.close();
+ ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath);
+
+ FSDataInputStream inputStream = null;
+ try {
+ inputStream = fs.open(testPath);
+ ContractTestUtils.verifyRead(inputStream, data.getBytes(), 0, data.length());
+ }
+ finally {
+ if(inputStream != null) {
+ inputStream.close();
+ }
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Negative test to verify read access check.
+ * @throws Throwable
+ */
+
+ @Test //(expected=WasbAuthorizationException.class)
+ public void testReadAccessCheckNegative() throws Throwable {
+
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage("read operation for Path : /testReadAccessCheckNegative/test.dat not allowed");
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+ Path parentDir = new Path("/testReadAccessCheckNegative");
+ Path testPath = new Path(parentDir, "test.dat");
+
+ MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl();
+ authorizer.init(null);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false);
+ authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ FSDataOutputStream fso = fs.create(testPath);
+ String data = "Hello World";
+ fso.writeBytes(data);
+ fso.close();
+ ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath);
+
+ FSDataInputStream inputStream = null;
+ try {
+ inputStream = fs.open(testPath);
+ ContractTestUtils.verifyRead(inputStream, data.getBytes(), 0, data.length());
+ } finally {
+ if (inputStream != null) {
+ inputStream.close();
+ }
+ fs.delete(parentDir, true);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java
new file mode 100644
index 0000000..b13e5e9
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.http.*;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE;
+
+/**
+ * Test class to hold all WasbRemoteCallHelper tests
+ */
+public class TestWasbRemoteCallHelper
+ extends AbstractWasbTestBase {
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true");
+ conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URL, "http://localhost/");
+ return AzureBlobStorageTestAccount.create(conf);
+ }
+
+ @Before
+ public void beforeMethod() {
+ boolean useSecureMode = fs.getConf().getBoolean(KEY_USE_SECURE_MODE, false);
+ boolean useAuthorization = fs.getConf().getBoolean(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, false);
+ Assume.assumeTrue("Test valid when both SecureMode and Authorization are enabled .. skipping",
+ useSecureMode && useAuthorization);
+
+ Assume.assumeTrue(
+ useSecureMode && useAuthorization
+ );
+ }
+
+ @Rule
+ public ExpectedException expectedEx = ExpectedException.none();
+
+ /**
+ * Test invalid status-code
+ * @throws Throwable
+ */
+ @Test // (expected = WasbAuthorizationException.class)
+ public void testInvalidStatusCode() throws Throwable {
+
+ setupExpectations();
+
+ // set up mocks
+ HttpClient mockHttpClient = Mockito.mock(HttpClient.class);
+ HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class);
+ Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse);
+ Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(999));
+ // finished setting up mocks
+
+ performop(mockHttpClient);
+ }
+
+ /**
+ * Test invalid Content-Type
+ * @throws Throwable
+ */
+ @Test // (expected = WasbAuthorizationException.class)
+ public void testInvalidContentType() throws Throwable {
+
+ setupExpectations();
+
+ // set up mocks
+ HttpClient mockHttpClient = Mockito.mock(HttpClient.class);
+ HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class);
+ Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse);
+ Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Type"))
+ .thenReturn(newHeader("Content-Type", "text/plain"));
+ // finished setting up mocks
+
+ performop(mockHttpClient);
+ }
+
+ /**
+ * Test missing Content-Length
+ * @throws Throwable
+ */
+ @Test // (expected = WasbAuthorizationException.class)
+ public void testMissingContentLength() throws Throwable {
+
+ setupExpectations();
+
+ // set up mocks
+ HttpClient mockHttpClient = Mockito.mock(HttpClient.class);
+ HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class);
+ Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse);
+ Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Type"))
+ .thenReturn(newHeader("Content-Type", "application/json"));
+ // finished setting up mocks
+
+ performop(mockHttpClient);
+ }
+
+ /**
+ * Test Content-Length exceeds max
+ * @throws Throwable
+ */
+ @Test // (expected = WasbAuthorizationException.class)
+ public void testContentLengthExceedsMax() throws Throwable {
+
+ setupExpectations();
+
+ // set up mocks
+ HttpClient mockHttpClient = Mockito.mock(HttpClient.class);
+ HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class);
+ Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse);
+ Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Type"))
+ .thenReturn(newHeader("Content-Type", "application/json"));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Length"))
+ .thenReturn(newHeader("Content-Length", "2048"));
+ // finished setting up mocks
+
+ performop(mockHttpClient);
+ }
+
+ /**
+ * Test invalid Content-Length value
+ * @throws Throwable
+ */
+ @Test // (expected = WasbAuthorizationException.class)
+ public void testInvalidContentLengthValue() throws Throwable {
+
+ setupExpectations();
+
+ // set up mocks
+ HttpClient mockHttpClient = Mockito.mock(HttpClient.class);
+ HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class);
+ Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse);
+ Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Type"))
+ .thenReturn(newHeader("Content-Type", "application/json"));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Length"))
+ .thenReturn(newHeader("Content-Length", "20abc48"));
+ // finished setting up mocks
+
+ performop(mockHttpClient);
+ }
+
+ /**
+ * Test valid JSON response
+ * @throws Throwable
+ */
+ @Test
+ public void testValidJSONResponse() throws Throwable {
+
+ // set up mocks
+ HttpClient mockHttpClient = Mockito.mock(HttpClient.class);
+
+ HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class);
+ HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class);
+
+ Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse);
+ Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Type"))
+ .thenReturn(newHeader("Content-Type", "application/json"));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Length"))
+ .thenReturn(newHeader("Content-Length", "1024"));
+ Mockito.when(mockHttpResponse.getEntity()).thenReturn(mockHttpEntity);
+ Mockito.when(mockHttpEntity.getContent())
+ .thenReturn(new ByteArrayInputStream(validJsonResponse().getBytes(StandardCharsets.UTF_8)))
+ .thenReturn(new ByteArrayInputStream(validJsonResponse().getBytes(StandardCharsets.UTF_8)))
+ .thenReturn(new ByteArrayInputStream(validJsonResponse().getBytes(StandardCharsets.UTF_8)));
+ // finished setting up mocks
+
+ performop(mockHttpClient);
+ }
+
+ /**
+ * Test malformed JSON response
+ * @throws Throwable
+ */
+ @Test // (expected = WasbAuthorizationException.class)
+ public void testMalFormedJSONResponse() throws Throwable {
+
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage("com.fasterxml.jackson.core.JsonParseException: Unexpected end-of-input in FIELD_NAME");
+
+ // set up mocks
+ HttpClient mockHttpClient = Mockito.mock(HttpClient.class);
+
+ HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class);
+ HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class);
+
+ Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse);
+ Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Type"))
+ .thenReturn(newHeader("Content-Type", "application/json"));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Length"))
+ .thenReturn(newHeader("Content-Length", "1024"));
+ Mockito.when(mockHttpResponse.getEntity()).thenReturn(mockHttpEntity);
+ Mockito.when(mockHttpEntity.getContent())
+ .thenReturn(new ByteArrayInputStream(malformedJsonResponse().getBytes(StandardCharsets.UTF_8)));
+ // finished setting up mocks
+
+ performop(mockHttpClient);
+ }
+
+ /**
+ * Test valid JSON response failure response code
+ * @throws Throwable
+ */
+ @Test // (expected = WasbAuthorizationException.class)
+ public void testFailureCodeJSONResponse() throws Throwable {
+
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage("Remote authorization service encountered an error Unauthorized");
+
+ // set up mocks
+ HttpClient mockHttpClient = Mockito.mock(HttpClient.class);
+
+ HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class);
+ HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class);
+
+ Mockito.when(mockHttpClient.execute(Mockito.<HttpGet>any())).thenReturn(mockHttpResponse);
+ Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Type"))
+ .thenReturn(newHeader("Content-Type", "application/json"));
+ Mockito.when(mockHttpResponse.getFirstHeader("Content-Length"))
+ .thenReturn(newHeader("Content-Length", "1024"));
+ Mockito.when(mockHttpResponse.getEntity()).thenReturn(mockHttpEntity);
+ Mockito.when(mockHttpEntity.getContent())
+ .thenReturn(new ByteArrayInputStream(failureCodeJsonResponse().getBytes(StandardCharsets.UTF_8)));
+ // finished setting up mocks
+
+ performop(mockHttpClient);
+ }
+
+ private void setupExpectations() {
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage("org.apache.hadoop.fs.azure.WasbRemoteCallException: " +
+ "http://localhost/CHECK_AUTHORIZATION?wasb_absolute_path=%2Ftest.dat&" +
+ "operation_type=write&delegation_token:Encountered IOException while making remote call");
+ }
+
+ private void performop(HttpClient mockHttpClient) throws Throwable {
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+
+ Path testPath = new Path("/", "test.dat");
+
+ RemoteWasbAuthorizerImpl authorizer = new RemoteWasbAuthorizerImpl();
+ authorizer.init(fs.getConf());
+ WasbRemoteCallHelper mockWasbRemoteCallHelper = new WasbRemoteCallHelper();
+ mockWasbRemoteCallHelper.updateHttpClient(mockHttpClient);
+ authorizer.updateWasbRemoteCallHelper(mockWasbRemoteCallHelper);
+ fs.updateWasbAuthorizer(authorizer);
+
+ fs.create(testPath);
+ ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath);
+ fs.delete(testPath, false);
+ }
+
+ private String validJsonResponse() {
+ return new String(
+ "{\"responseCode\": 0, \"authorizationResult\": true, \"responseMessage\": \"Authorized\"}"
+ );
+ }
+
+ private String malformedJsonResponse() {
+ return new String(
+ "{\"responseCode\": 0, \"authorizationResult\": true, \"responseMessage\":"
+ );
+ }
+
+ private String failureCodeJsonResponse() {
+ return new String(
+ "{\"responseCode\": 1, \"authorizationResult\": false, \"responseMessage\": \"Unauthorized\"}"
+ );
+ }
+
+ private StatusLine newStatusLine(int statusCode) {
+ return new StatusLine() {
+ @Override
+ public ProtocolVersion getProtocolVersion() {
+ return new ProtocolVersion("HTTP", 1, 1);
+ }
+
+ @Override
+ public int getStatusCode() {
+ return statusCode;
+ }
+
+ @Override
+ public String getReasonPhrase() {
+ return "Reason Phrase";
+ }
+ };
+ }
+
+ private Header newHeader(String name, String value) {
+ return new Header() {
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public HeaderElement[] getElements() throws ParseException {
+ return new HeaderElement[0];
+ }
+ };
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68682352/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
index e898aa6..c73d6d8 100644
--- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
+++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
@@ -19,27 +19,21 @@
<!-- For tests against live azure, provide the following account information -->
<!--
- <property>
- <name>fs.azure.test.account.name</name>
- <value>{ACCOUNTNAME}.blob.core.windows.net</value>
- </property>
- <property>
- <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
- <value>{ACCOUNTKEY}</value>
- </property>
+ <property>
+ <name>fs.azure.test.account.name</name>
+ <value>{ACCOUNTNAME}.blob.core.windows.net</value>
+ </property>
+ <property>
+ <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
+ <value>{ACCOUNTKEY}</value>
+ </property>
+ -->
+
<property>
<name>fs.azure.secure.mode</name>
<value>false</value>
</property>
- <property>
- <name>fs.azure.local.sas.key.mode</name>
- <value>false</value>
- </property>
- <property>
- <name>fs.azure.cred.service.url</name>
- <value>{CRED_SERIVCE_URL}</value>
- </property>
- -->
+
<!-- Save the above configuration properties in a separate file named -->
<!-- azure-auth-keys.xml in the same directory as this file. -->
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] hadoop git commit: YARN-6278. Enforce to use correct node and
npm version in new YARN-UI build. (Sunil G via wangda)
Posted by kk...@apache.org.
YARN-6278. Enforce to use correct node and npm version in new YARN-UI build. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c61bc2d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c61bc2d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c61bc2d8
Branch: refs/heads/YARN-5972
Commit: c61bc2d843ace614078be24b2dcbf3fb43d6f358
Parents: 0f336ba
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Mar 3 23:43:25 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Mar 3 23:43:25 2017 -0800
----------------------------------------------------------------------
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 50 +++++++++++---------
.../hadoop-yarn-ui/src/main/webapp/bower.json | 2 +
.../hadoop-yarn-ui/src/main/webapp/package.json | 11 +++--
3 files changed, 36 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c61bc2d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
index 22031fa..ae7ab0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
@@ -32,8 +32,8 @@
<packaging.type>pom</packaging.type>
<webappTgtDir>${basedir}/target/src/main/webapp</webappTgtDir>
<node.executable>node</node.executable>
- <nodeVersion>v0.12.2</nodeVersion>
- <npmVersion>2.10.0</npmVersion>
+ <nodeVersion>v4.4.5</nodeVersion>
+ <npmVersion>2.15.5</npmVersion>
<keep-ui-build-cache>false</keep-ui-build-cache>
</properties>
@@ -120,41 +120,47 @@
</executions>
</plugin>
-
- <!-- Bower install & grunt build-->
<plugin>
- <artifactId>exec-maven-plugin</artifactId>
- <groupId>org.codehaus.mojo</groupId>
+ <groupId>com.github.eirslett</groupId>
+ <artifactId>frontend-maven-plugin</artifactId>
+ <configuration>
+ <workingDirectory>${webappTgtDir}</workingDirectory>
+ </configuration>
<executions>
<execution>
<phase>generate-sources</phase>
- <id>npm install</id>
+ <id>install node and npm</id>
<goals>
- <goal>exec</goal>
+ <goal>install-node-and-npm</goal>
</goals>
<configuration>
- <workingDirectory>${webappTgtDir}</workingDirectory>
- <executable>npm</executable>
- <arguments>
- <argument>install</argument>
- </arguments>
+ <nodeVersion>${nodeVersion}</nodeVersion>
+ <npmVersion>${npmVersion}</npmVersion>
</configuration>
</execution>
<execution>
<phase>generate-sources</phase>
+ <id>npm install</id>
+ <goals>
+ <goal>npm</goal>
+ </goals>
+ </execution>
+ <execution>
+ <phase>generate-sources</phase>
<id>bower install</id>
<goals>
- <goal>exec</goal>
+ <goal>bower</goal>
</goals>
- <configuration>
- <workingDirectory>${webappTgtDir}</workingDirectory>
- <executable>bower</executable>
- <arguments>
- <argument>--allow-root</argument>
- <argument>install</argument>
- </arguments>
- </configuration>
</execution>
+ </executions>
+ </plugin>
+
+
+ <!-- Bower install & grunt build-->
+ <plugin>
+ <artifactId>exec-maven-plugin</artifactId>
+ <groupId>org.codehaus.mojo</groupId>
+ <executions>
<execution>
<id>ember build</id>
<phase>generate-sources</phase>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c61bc2d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json
index fe5f289..e1ab943 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json
@@ -12,6 +12,8 @@
"loader.js": "3.3.0",
"qunit": "1.19.0",
"jquery-ui": "1.11.4",
+ "moment": "2.12.0",
+ "moment-timezone": "0.5.0",
"more-js": "0.8.2",
"bootstrap": "3.3.6",
"d3": "~3.5.6",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c61bc2d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
index 4cd0fdc..573e505 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
@@ -18,12 +18,13 @@
"author": "",
"license": "Apache",
"devDependencies": {
+ "bower": "1.7.7",
"broccoli-asset-rev": "2.4.2",
"broccoli-funnel": "1.0.1",
- "em-table": "0.7.0",
+ "broccoli-merge-trees": "1.1.1",
"ember-array-contains-helper": "1.0.2",
"ember-bootstrap": "0.5.1",
- "ember-cli": "1.13.13",
+ "ember-cli": "^1.13.13",
"ember-cli-app-version": "1.0.0",
"ember-cli-babel": "5.1.6",
"ember-cli-content-security-policy": "0.4.0",
@@ -33,6 +34,8 @@
"ember-cli-ic-ajax": "0.2.1",
"ember-cli-inject-live-reload": "1.4.0",
"ember-cli-jquery-ui": "0.0.20",
+ "ember-cli-moment-shim": "0.7.3",
+ "ember-cli-numeral": "^0.2.0",
"ember-cli-qunit": "1.2.1",
"ember-cli-release": "0.2.8",
"ember-cli-sri": "1.2.1",
@@ -48,8 +51,6 @@
},
"dependencies": {
"em-helpers": "^0.8.0",
- "em-table": "^0.7.0",
- "ember-cli-moment-shim": "^3.0.1",
- "ember-cli-numeral": "^0.2.0"
+ "em-table": "^0.7.0"
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] hadoop git commit: HADOOP-14138. Remove S3A ref from META-INF
service discovery,
rely on existing core-default entry. Contributed by Steve Loughran
Posted by kk...@apache.org.
HADOOP-14138. Remove S3A ref from META-INF service discovery, rely on existing core-default entry.
Contributed by Steve Loughran
(cherry picked from commit 51fefbe38bf40373d2855163fa6444821b0f994c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a97833e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a97833e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a97833e0
Branch: refs/heads/YARN-5972
Commit: a97833e0ed4b31f0403ee3d789163615c7cdd9af
Parents: b3ec531
Author: Steve Loughran <st...@apache.org>
Authored: Thu Mar 2 21:44:30 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Mar 2 21:44:30 2017 +0000
----------------------------------------------------------------------
.../resources/META-INF/services/org.apache.hadoop.fs.FileSystem | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a97833e0/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
index e606fd9..1938bde 100644
--- a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
+++ b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
@@ -14,4 +14,3 @@
# limitations under the License.
org.apache.hadoop.fs.s3native.NativeS3FileSystem
-org.apache.hadoop.fs.s3a.S3AFileSystem
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] hadoop git commit: YARN-5665. Enhance documentation for
yarn.resourcemanager.scheduler.class property. (Yufei Gu via rchiang)
Posted by kk...@apache.org.
YARN-5665. Enhance documentation for yarn.resourcemanager.scheduler.class property. (Yufei Gu via rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9dc444d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9dc444d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9dc444d
Branch: refs/heads/YARN-5972
Commit: d9dc444dc73fbe23f9e553d63baf83f12c636fa7
Parents: ec839b9
Author: Ray Chiang <rc...@apache.org>
Authored: Mon Mar 6 14:02:49 2017 -0800
Committer: Ray Chiang <rc...@apache.org>
Committed: Mon Mar 6 14:02:49 2017 -0800
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/ClusterSetup.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9dc444d/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
index 1d9e9da..7be6a19 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
@@ -139,7 +139,7 @@ This section deals with important parameters to be specified in the given config
| `yarn.resourcemanager.admin.address` | `ResourceManager` host:port for administrative commands. | *host:port*�If set, overrides the hostname set in `yarn.resourcemanager.hostname`. |
| `yarn.resourcemanager.webapp.address` | `ResourceManager` web-ui host:port. | *host:port*�If set, overrides the hostname set in `yarn.resourcemanager.hostname`. |
| `yarn.resourcemanager.hostname` | `ResourceManager` host. | *host*�Single hostname that can be set in place of setting all `yarn.resourcemanager*address` resources. Results in default ports for ResourceManager components. |
-| `yarn.resourcemanager.scheduler.class` | `ResourceManager` Scheduler class. | `CapacityScheduler` (recommended), `FairScheduler` (also recommended), or `FifoScheduler` |
+| `yarn.resourcemanager.scheduler.class` | `ResourceManager` Scheduler class. | `CapacityScheduler` (recommended), `FairScheduler` (also recommended), or `FifoScheduler`. Use a fully qualified class name, e.g., `org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler`. |
| `yarn.scheduler.minimum-allocation-mb` | Minimum limit of memory to allocate to each container request at the `Resource Manager`. | In MBs |
| `yarn.scheduler.maximum-allocation-mb` | Maximum limit of memory to allocate to each container request at the `Resource Manager`. | In MBs |
| `yarn.resourcemanager.nodes.include-path` / `yarn.resourcemanager.nodes.exclude-path` | List of permitted/excluded NodeManagers. | If necessary, use these files to control the list of allowable NodeManagers. |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] hadoop git commit: YARN-6274. Documentation refers to
incorrect nodemanager health checker interval property. Contributed by WeiWei
Yang
Posted by kk...@apache.org.
YARN-6274. Documentation refers to incorrect nodemanager health checker interval property. Contributed by WeiWei Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05237636
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05237636
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05237636
Branch: refs/heads/YARN-5972
Commit: 05237636d3a359dc502e8fa2d66bfdcea274f86c
Parents: eb5a179
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Fri Mar 3 07:57:37 2017 -0600
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Fri Mar 3 07:57:37 2017 -0600
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/ClusterSetup.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05237636/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
index e2ccbf0..1d9e9da 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
@@ -202,7 +202,7 @@ The following parameters can be used to control the node health monitoring scrip
|:---- |:---- |:---- |
| `yarn.nodemanager.health-checker.script.path` | Node health script | Script to check for node's health status. |
| `yarn.nodemanager.health-checker.script.opts` | Node health script options | Options for script to check for node's health status. |
-| `yarn.nodemanager.health-checker.script.interval-ms` | Node health script interval | Time interval for running health script. |
+| `yarn.nodemanager.health-checker.interval-ms` | Node health script interval | Time interval for running health script. |
| `yarn.nodemanager.health-checker.script.timeout-ms` | Node health script timeout interval | Timeout for health script execution. |
The health checker script is not supposed to give ERROR if only some of the local disks become bad. NodeManager has the ability to periodically check the health of the local disks (specifically checks nodemanager-local-dirs and nodemanager-log-dirs) and after reaching the threshold of number of bad directories based on the value set for the config property yarn.nodemanager.disk-health-checker.min-healthy-disks, the whole node is marked unhealthy and this info is sent to resource manager also. The boot disk is either raided or a failure in the boot disk is identified by the health checker script.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] hadoop git commit: HADOOP-14087. S3A typo in pom.xml test
exclusions. Contributed by Aaron Fabbri.
Posted by kk...@apache.org.
HADOOP-14087. S3A typo in pom.xml test exclusions. Contributed by Aaron Fabbri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f597f4c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f597f4c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f597f4c4
Branch: refs/heads/YARN-5972
Commit: f597f4c43e0a6e2304b9bcaf727d6d8d15a365f9
Parents: f01a69f
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Mar 7 15:14:55 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Mar 7 15:14:55 2017 +0900
----------------------------------------------------------------------
hadoop-tools/hadoop-aws/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f597f4c4/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index e5bbbfd..c188055 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -184,7 +184,7 @@
<exclude>**/ITest*Root*.java</exclude>
<exclude>**/ITestS3AFileContextStatistics.java</exclude>
<exclude>**/ITestS3AEncryptionSSE*.java</exclude>
- <include>**/ITestS3AHuge*.java</include>
+ <exclude>**/ITestS3AHuge*.java</exclude>
</excludes>
</configuration>
</execution>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] hadoop git commit: Revert "HDFS-11477. Simplify file IO
profiling configuration. Contributed by Hanisha Koneru."
Posted by kk...@apache.org.
Revert "HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru."
This reverts commit e61491d4769536ce8b77b416005ce531278ce729.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f4817f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f4817f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f4817f2
Branch: refs/heads/YARN-5972
Commit: 8f4817f2c5524c1891a3b3b50e396dd4ca294a2d
Parents: e61491d
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Mar 2 16:20:48 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Mar 2 16:20:48 2017 -0800
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 7 +----
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 ++++-
.../hdfs/server/datanode/FileIoProvider.java | 2 +-
.../server/datanode/ProfilingFileIoEvents.java | 27 +++++---------------
.../datanode/TestDataNodeVolumeMetrics.java | 4 +--
.../hadoop/tools/TestHdfsConfigFields.java | 2 ++
6 files changed, 17 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index a8bdbeb..7900692 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -332,12 +332,7 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
FsVolume
--------
-Per-volume metrics contain Datanode Volume IO related statistics. Per-volume
-metrics are off by default. They can be enabled by setting `dfs.datanode
-.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0.
-Setting this value to 0.0 would mean profiling is not enabled. But enabling
-per-volume metrics may have a performance impact. Each metrics record
-contains tags such as Hostname as additional information along with metrics.
+Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics.
| Name | Description |
|:---- |:---- |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 82d6073..be20829 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -718,6 +718,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT = 0; //no throttling
// Datanode File IO Stats
+ public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
+ "dfs.datanode.enable.fileio.profiling";
+ public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
+ false;
public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY =
"dfs.datanode.enable.fileio.fault.injection";
public static final boolean
@@ -726,7 +730,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
"dfs.datanode.fileio.profiling.sampling.fraction";
public static final double
- DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0;
+ DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
//Keys with no defaults
public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index 5508e0b..9def2e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*;
*
* Behavior can be injected into these events by enabling the
* profiling and/or fault injection event hooks through
- * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and
+ * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and
* {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}.
* These event hooks are disabled by default.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 7eab969..9af84fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.datanode;
-import static java.lang.Math.abs;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -44,32 +43,18 @@ class ProfilingFileIoEvents {
public ProfilingFileIoEvents(@Nullable Configuration conf) {
if (conf != null) {
+ isEnabled = conf.getBoolean(DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT);
- if (abs(fileIOSamplingFraction) < 0.000001) {
- LOG.info(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
- + fileIOSamplingFraction + ". Disabling file IO profiling");
- isEnabled = false;
- } else if (fileIOSamplingFraction < 0.000001) {
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
+ if (fileIOSamplingFraction > 1) {
LOG.warn(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
- " value cannot be less than 0. Disabling file IO profiling.");
- isEnabled = false;
- } else if (fileIOSamplingFraction > 1) {
- LOG.warn(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
- " value cannot be more than 1. Setting value to 1 and enabling " +
- "file IO profiling");
- isEnabled = true;
+ " value cannot be more than 1. Setting value to 1");
fileIOSamplingFraction = 1;
- } else {
- LOG.info(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
- + fileIOSamplingFraction + ". Enabling file IO profiling");
- isEnabled = true;
}
sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
index 03e1dee..6a8ac9c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
@@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics {
private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException {
Configuration conf = new HdfsConfiguration();
- conf.setDouble(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
+ conf.setBoolean(DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true);
SimulatedFSDataset.setFactory(conf);
return new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATANODES)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 1fdf713..cdce342 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -103,6 +103,8 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
.add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
configurationPropsToSkipCompare
.add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
.DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] hadoop git commit: MAPREDUCE-6852. Job#updateStatus() failed
with NPE due to race condition. Contributed by Junping Du
Posted by kk...@apache.org.
MAPREDUCE-6852. Job#updateStatus() failed with NPE due to race condition. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/747bafaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/747bafaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/747bafaf
Branch: refs/heads/YARN-5972
Commit: 747bafaf969857b66233a8b4660590bdd712ed7d
Parents: eeca8b0
Author: Jian He <ji...@apache.org>
Authored: Thu Mar 2 10:58:56 2017 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Mar 2 10:58:56 2017 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/mapreduce/Job.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/747bafaf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
index 45c065d..84d90de 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
@@ -323,7 +323,7 @@ public class Job extends JobContextImpl implements JobContext {
this.status = ugi.doAs(new PrivilegedExceptionAction<JobStatus>() {
@Override
public JobStatus run() throws IOException, InterruptedException {
- return cluster.getClient().getJobStatus(status.getJobID());
+ return cluster.getClient().getJobStatus(getJobID());
}
});
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] hadoop git commit: Treat encrypted files as private.
Contributed by Daniel Templeton.
Posted by kk...@apache.org.
Treat encrypted files as private. Contributed by Daniel Templeton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f01a69f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f01a69f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f01a69f8
Branch: refs/heads/YARN-5972
Commit: f01a69f84f4cc7d925d078a7ce32e5800da4e429
Parents: 1441398
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Mar 7 13:22:11 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Mar 7 13:22:11 2017 +0900
----------------------------------------------------------------------
.../filecache/ClientDistributedCacheManager.java | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f01a69f8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
index 73a0330..9f8edb5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
@@ -294,10 +294,21 @@ public class ClientDistributedCacheManager {
FsAction action, Map<URI, FileStatus> statCache) throws IOException {
FileStatus status = getFileStatus(fs, path.toUri(), statCache);
FsPermission perms = status.getPermission();
- FsAction otherAction = perms.getOtherAction();
- if (otherAction.implies(action)) {
- return true;
+
+ // Encrypted files are always treated as private. This stance has two
+ // important side effects. The first is that the encrypted files will be
+ // downloaded as the job owner instead of the YARN user, which is required
+ // for the KMS ACLs to work as expected. Second, it prevent a file with
+ // world readable permissions that is stored in an encryption zone from
+ // being localized as a publicly shared file with world readable
+ // permissions.
+ if (!perms.getEncryptedBit()) {
+ FsAction otherAction = perms.getOtherAction();
+ if (otherAction.implies(action)) {
+ return true;
+ }
}
+
return false;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] hadoop git commit: HDFS-11508. Fix bind failure in
SimpleTCPServer & Portmap where bind fails because socket is in TIME_WAIT
state. Contributed by Mukul Kumar Singh.
Posted by kk...@apache.org.
HDFS-11508. Fix bind failure in SimpleTCPServer & Portmap where bind fails because socket is in TIME_WAIT state. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f9848df
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f9848df
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f9848df
Branch: refs/heads/YARN-5972
Commit: 1f9848dfe1fc9148cbbcfcc3dfed948b9e0f3c3c
Parents: 959940b
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Mar 7 11:41:05 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Mar 7 11:41:05 2017 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java | 1 +
.../src/main/java/org/apache/hadoop/portmap/Portmap.java | 2 ++
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f9848df/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
index 99d1d6f..f7ab52e 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
@@ -81,6 +81,7 @@ public class SimpleTcpServer {
});
server.setOption("child.tcpNoDelay", true);
server.setOption("child.keepAlive", true);
+ server.setOption("reuseAddress", true);
// Listen to TCP port
ch = server.bind(new InetSocketAddress(port));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f9848df/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java
index 2b88791..94d76d0 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java
@@ -109,12 +109,14 @@ final class Portmap {
RpcUtil.STAGE_RPC_TCP_RESPONSE);
}
});
+ tcpServer.setOption("reuseAddress", true);
udpServer = new ConnectionlessBootstrap(new NioDatagramChannelFactory(
Executors.newCachedThreadPool()));
udpServer.setPipeline(Channels.pipeline(RpcUtil.STAGE_RPC_MESSAGE_PARSER,
handler, RpcUtil.STAGE_RPC_UDP_RESPONSE));
+ udpServer.setOption("reuseAddress", true);
tcpChannel = tcpServer.bind(tcpAddress);
udpChannel = udpServer.bind(udpAddress);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org