You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/01/05 00:05:26 UTC
[45/50] [abbrv] ambari git commit: AMBARI-19331. Setup correct
authentication and authorization mechanism between Yarn and Zookeeper (Attila
Magyar via rlevas)
AMBARI-19331. Setup correct authentication and authorization mechanism between Yarn and Zookeeper (Attila Magyar via rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e96dee0f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e96dee0f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e96dee0f
Branch: refs/heads/branch-dev-patch-upgrade
Commit: e96dee0fecfafeda637c339217b2746b337f729f
Parents: 30b27b1
Author: Attila Magyar <am...@hortonworks.com>
Authored: Wed Jan 4 11:28:03 2017 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Wed Jan 4 11:29:11 2017 -0500
----------------------------------------------------------------------
.../src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java | 2 ++
.../test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java | 5 +++++
.../python/resource_management/core/resources/zkmigrator.py | 3 +++
.../main/resources/common-services/YARN/2.1.0.2.0/kerberos.json | 4 +++-
.../YARN/2.1.0.2.0/package/scripts/params_linux.py | 2 ++
.../YARN/2.1.0.2.0/package/scripts/resourcemanager.py | 5 +++--
.../main/resources/common-services/YARN/3.0.0.3.0/kerberos.json | 4 +++-
.../YARN/3.0.0.3.0/package/scripts/params_linux.py | 2 ++
.../YARN/3.0.0.3.0/package/scripts/resourcemanager.py | 5 +++--
.../main/resources/stacks/HDP/2.2/services/YARN/kerberos.json | 4 +++-
.../resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json | 4 +++-
.../main/resources/stacks/HDP/2.3/services/YARN/kerberos.json | 4 +++-
.../main/resources/stacks/HDP/2.5/services/YARN/kerberos.json | 4 +++-
13 files changed, 38 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java b/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java
index 15edb69..b4da1ed 100644
--- a/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java
+++ b/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java
@@ -74,6 +74,8 @@ public class ZkMigrator {
ZooKeeper client = ZkConnection.open(connectionString, SESSION_TIMEOUT_MILLIS, CONNECTION_TIMEOUT_MILLIS);
try {
acl.setRecursivelyOn(client, znode);
+ } catch (KeeperException.NoNodeException e) {
+ System.out.println("Could not set ACL on " + znode + ". Reason: " + e.getMessage());
} finally {
client.close();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java b/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java
index 0a2bbac..b2c9899 100644
--- a/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java
+++ b/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java
@@ -105,6 +105,11 @@ public class ZkMigratorTest {
setAcls("/any", "world:anyone:invalid");
}
+ @Test
+ public void testIgnoresNonExistentNode() throws Exception {
+ setAcls("/nonexistent", "world:anyone:rw");
+ }
+
@Before
public void startZookeeper() throws Exception {
zkTestServer = new TestingServer(Port.free());
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py b/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py
index a946e47..5e86e05 100644
--- a/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py
+++ b/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py
@@ -21,6 +21,8 @@ Ambari Agent
"""
from resource_management.core.resources.system import Execute
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import format
class ZkMigrator:
def __init__(self, zk_host, java_exec, java_home, jaas_file, user):
@@ -32,6 +34,7 @@ class ZkMigrator:
self.zkmigrator_jar = "/var/lib/ambari-agent/tools/zkmigrator.jar"
def set_acls(self, znode, acl, tries=1):
+ Logger.info(format("Setting ACL on znode {znode} to {acl}"))
Execute(
self._command(znode, acl), \
user=self.user, \
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json
index a8379ee..c307800 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json
@@ -32,7 +32,9 @@
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
"yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
- "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda"
+ "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
+ "hadoop.registry.secure" : "true",
+ "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
}
},
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 53ea4d6..2fb7bff 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -255,6 +255,8 @@ nodemanager_kinit_cmd = ""
rm_zk_address = config['configurations']['yarn-site']['yarn.resourcemanager.zk-address']
rm_zk_znode = config['configurations']['yarn-site']['yarn.resourcemanager.zk-state-store.parent-path']
rm_zk_store_class = config['configurations']['yarn-site']['yarn.resourcemanager.store.class']
+rm_zk_failover_znode = default('/configurations/yarn-site/yarn.resourcemanager.ha.automatic-failover.zk-base-path', '/yarn-leader-election')
+hadoop_registry_zk_root = default('/configurations/yarn-site/hadoop.registry.zk.root', '/registry')
if security_enabled:
rm_principal_name = config['configurations']['yarn-site']['yarn.resourcemanager.principal']
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
index 3cf5a5b..3207f27 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
@@ -228,7 +228,7 @@ class ResourcemanagerDefault(Resourcemanager):
def disable_security(self, env):
import params
- if 'ZKRMStateStore' not in params.rm_zk_store_class:
+ if not params.rm_zk_address:
Logger.info("Skipping reverting ACL")
return
zkmigrator = ZkMigrator(
@@ -237,8 +237,9 @@ class ResourcemanagerDefault(Resourcemanager):
params.java64_home, \
params.yarn_jaas_file, \
params.yarn_user)
- Logger.info("Reverting ACL of znode %s" % params.rm_zk_znode)
zkmigrator.set_acls(params.rm_zk_znode, 'world:anyone:crdwa')
+ zkmigrator.set_acls(params.rm_zk_failover_znode, 'world:anyone:crdwa')
+ zkmigrator.set_acls(params.hadoop_registry_zk_root, 'world:anyone:crdwa')
def wait_for_dfs_directories_created(self, *dirs):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/kerberos.json b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/kerberos.json
index 4cb18a9..af920f1 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/kerberos.json
@@ -33,7 +33,9 @@
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
"yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
- "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda"
+ "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
+ "hadoop.registry.secure" : "true",
+ "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
}
},
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
index b79fa1a..23a25a0 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
@@ -255,6 +255,8 @@ nodemanager_kinit_cmd = ""
rm_zk_address = config['configurations']['yarn-site']['yarn.resourcemanager.zk-address']
rm_zk_znode = config['configurations']['yarn-site']['yarn.resourcemanager.zk-state-store.parent-path']
rm_zk_store_class = config['configurations']['yarn-site']['yarn.resourcemanager.store.class']
+rm_zk_failover_znode = default('/configurations/yarn-site/yarn.resourcemanager.ha.automatic-failover.zk-base-path', '/yarn-leader-election')
+hadoop_registry_zk_root = default('/configurations/yarn-site/hadoop.registry.zk.root', '/registry')
if security_enabled:
rm_principal_name = config['configurations']['yarn-site']['yarn.resourcemanager.principal']
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
index 16670d1..91d7b89 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
@@ -112,7 +112,7 @@ class ResourcemanagerDefault(Resourcemanager):
def disable_security(self, env):
import params
- if 'ZKRMStateStore' not in params.rm_zk_store_class:
+ if not params.rm_zk_address:
Logger.info("Skipping reverting ACL")
return
zkmigrator = ZkMigrator(
@@ -121,8 +121,9 @@ class ResourcemanagerDefault(Resourcemanager):
params.java64_home, \
params.yarn_jaas_file, \
params.yarn_user)
- Logger.info("Reverting ACL of znode %s" % params.rm_zk_znode)
zkmigrator.set_acls(params.rm_zk_znode, 'world:anyone:crdwa')
+ zkmigrator.set_acls(params.rm_zk_failover_znode, 'world:anyone:crdwa')
+ zkmigrator.set_acls(params.hadoop_registry_zk_root, 'world:anyone:crdwa')
def start(self, env, upgrade_type=None):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json
index 784589c..3a183cc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json
@@ -33,7 +33,9 @@
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
"yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
"yarn.resourcemanager.zk-state-store.parent-path": "/rmstore-secure",
- "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda"
+ "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
+ "hadoop.registry.secure" : "true",
+ "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
}
},
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json
index 74b5746..e11ce84 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json
@@ -35,7 +35,9 @@
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
"yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
- "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda"
+ "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
+ "hadoop.registry.secure" : "true",
+ "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
}
},
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json
index c20bd23..1a6cf5b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json
@@ -33,7 +33,9 @@
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
"yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
- "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda"
+ "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
+ "hadoop.registry.secure" : "true",
+ "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
}
},
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/e96dee0f/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
index 4cb18a9..af920f1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
@@ -33,7 +33,9 @@
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
"yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
- "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda"
+ "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
+ "hadoop.registry.secure" : "true",
+ "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
}
},
{