You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2014/11/07 04:10:50 UTC
ambari git commit: AMBARI-8174. Ambari-deployed cluster can't start
datanode as root from command line. (swagle)
Repository: ambari
Updated Branches:
refs/heads/branch-1.7.0 9502f0c9d -> b7f10831d
AMBARI-8174. Ambari-deployed cluster can't start datanode as root from command line. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b7f10831
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b7f10831
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b7f10831
Branch: refs/heads/branch-1.7.0
Commit: b7f10831d48de70052bc308596237775c12494a4
Parents: 9502f0c
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Thu Nov 6 19:10:05 2014 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Thu Nov 6 19:10:15 2014 -0800
----------------------------------------------------------------------
.../python/resource_management/core/source.py | 3 +-
.../2.0.6/services/HDFS/package/scripts/hdfs.py | 22 -------
.../services/HDFS/package/scripts/utils.py | 18 ++++--
.../services/HDFS/configuration/hadoop-env.xml | 7 +--
.../python/stacks/2.0.6/HDFS/test_datanode.py | 64 +++-----------------
5 files changed, 27 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7f10831/ambari-common/src/main/python/resource_management/core/source.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/source.py b/ambari-common/src/main/python/resource_management/core/source.py
index 22e1c6d..bff22f3 100644
--- a/ambari-common/src/main/python/resource_management/core/source.py
+++ b/ambari-common/src/main/python/resource_management/core/source.py
@@ -114,7 +114,8 @@ else:
self.context = variables.copy() if variables else {}
if not hasattr(self, 'template_env'):
self.template_env = JinjaEnvironment(loader=TemplateLoader(self.env),
- autoescape=False, undefined=StrictUndefined, trim_blocks=True)
+ autoescape=False, undefined=StrictUndefined,
+ trim_blocks=True)
self.template = self.template_env.get_template(self.name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7f10831/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py
index c192682..25c1067 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py
@@ -81,25 +81,3 @@ def hdfs(name=None):
if params.lzo_enabled:
Package(params.lzo_packages_for_current_host)
-
-def setup_hadoop_env(replace=False):
- import params
-
- if params.security_enabled:
- tc_owner = "root"
- else:
- tc_owner = params.hdfs_user
- Directory(params.hadoop_conf_empty_dir,
- recursive=True,
- owner='root',
- group='root'
- )
- Link(params.hadoop_conf_dir,
- to=params.hadoop_conf_empty_dir,
- not_if=format("ls {hadoop_conf_dir}")
- )
- File(os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh'),
- owner=tc_owner,
- content=InlineTemplate(params.hadoop_env_sh_template),
- replace=replace
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7f10831/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py
index c177d9c..28a9ccb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py
@@ -103,11 +103,22 @@ def service(action=None, name=None, user=None, create_pid_dir=False,
pass
pass
+ service_is_up = check_process if action == "start" else None
+
# Set HADOOP_SECURE_DN_USER correctly in hadoop-env if DN is running as root
# in secure mode.
- if name == 'datanode' and user == 'root':
- params.dn_proc_user = 'root'
- hdfs.setup_hadoop_env(replace=True)
+ set_secure_dn_user_cmd="sed -i 's/export HADOOP_SECURE_DN_USER=.*/export " \
+ "HADOOP_SECURE_DN_USER=\"{0}\"/' {1}"
+ if name == 'datanode' and action == 'start':
+ if user == 'root':
+ secure_dn_user = params.hdfs_user
+ else:
+ secure_dn_user = ""
+ pass
+
+ Execute(set_secure_dn_user_cmd.format(secure_dn_user,
+ os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh')),
+ not_if=service_is_up)
pass
hadoop_env_exports_str = ''
@@ -121,7 +132,6 @@ def service(action=None, name=None, user=None, create_pid_dir=False,
daemon_cmd = format("{ulimit_cmd} su -s /bin/bash - {user} -c '{cmd} {action} {name}'")
- service_is_up = check_process if action == "start" else None
#remove pid file from dead process
File(pid_file,
action="delete",
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7f10831/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml
index d186922..b768cb4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml
@@ -135,12 +135,7 @@ export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}
export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS"
-HDFS_DN_PROC_USER={{dn_proc_user}}
-if [ $HDFS_DN_PROC_USER == "root" ]; then
- export HADOOP_SECURE_DN_USER="{{hdfs_user}}"
-else
- export HADOOP_SECURE_DN_USER=""
-fi
+export HADOOP_SECURE_DN_USER=""
</value>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7f10831/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py
index 7f6c43b..40e5be8 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py
@@ -49,6 +49,9 @@ class TestDatanode(RMFTestCase):
owner = 'hdfs',
recursive = True,
)
+ self.assertResourceCalled('Execute', "sed -i 's/export HADOOP_SECURE_DN_USER=.*/export HADOOP_SECURE_DN_USER=\"\"/' /etc/hadoop/conf/hadoop-env.sh",
+ not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1',
+ )
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
action = ['delete'],
not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1',
@@ -109,19 +112,8 @@ class TestDatanode(RMFTestCase):
owner = 'hdfs',
recursive = True,
)
- self.assertResourceCalled('Directory', '/etc/hadoop/conf.empty',
- recursive=True,
- owner='root',
- group='root'
- )
- self.assertResourceCalled('Link', '/etc/hadoop/conf',
- to='/etc/hadoop/conf.empty',
- not_if='ls /etc/hadoop/conf'
- )
- self.assertResourceCalled('File', os.path.join('/etc/hadoop/conf', 'hadoop-env.sh'),
- owner='root',
- content=InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']),
- replace=True
+ self.assertResourceCalled('Execute', "sed -i 's/export HADOOP_SECURE_DN_USER=.*/export HADOOP_SECURE_DN_USER=\"hdfs\"/' /etc/hadoop/conf/hadoop-env.sh",
+ not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1',
)
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
action = ['delete'],
@@ -153,19 +145,8 @@ class TestDatanode(RMFTestCase):
owner = 'hdfs',
recursive = True,
)
- self.assertResourceCalled('Directory', '/etc/hadoop/conf.empty',
- recursive=True,
- owner='root',
- group='root'
- )
- self.assertResourceCalled('Link', '/etc/hadoop/conf',
- to='/etc/hadoop/conf.empty',
- not_if='ls /etc/hadoop/conf'
- )
- self.assertResourceCalled('File', os.path.join('/etc/hadoop/conf', 'hadoop-env.sh'),
- owner='root',
- content=InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']),
- replace=True
+ self.assertResourceCalled('Execute', "sed -i 's/export HADOOP_SECURE_DN_USER=.*/export HADOOP_SECURE_DN_USER=\"hdfs\"/' /etc/hadoop/conf/hadoop-env.sh",
+ not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1',
)
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
action = ['delete'],
@@ -200,6 +181,9 @@ class TestDatanode(RMFTestCase):
owner = 'hdfs',
recursive = True,
)
+ self.assertResourceCalled('Execute', "sed -i 's/export HADOOP_SECURE_DN_USER=.*/export HADOOP_SECURE_DN_USER=\"\"/' /etc/hadoop/conf/hadoop-env.sh",
+ not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1',
+ )
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
action = ['delete'],
not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1',
@@ -224,20 +208,6 @@ class TestDatanode(RMFTestCase):
owner = 'hdfs',
recursive = True,
)
- self.assertResourceCalled('Directory', '/etc/hadoop/conf.empty',
- recursive=True,
- owner='root',
- group='root'
- )
- self.assertResourceCalled('Link', '/etc/hadoop/conf',
- to='/etc/hadoop/conf.empty',
- not_if='ls /etc/hadoop/conf'
- )
- self.assertResourceCalled('File', os.path.join('/etc/hadoop/conf', 'hadoop-env.sh'),
- owner='root',
- content=InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']),
- replace=True
- )
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
action = ['delete'],
not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1',
@@ -272,20 +242,6 @@ class TestDatanode(RMFTestCase):
owner = 'hdfs',
recursive = True,
)
- self.assertResourceCalled('Directory', '/etc/hadoop/conf.empty',
- recursive=True,
- owner='root',
- group='root'
- )
- self.assertResourceCalled('Link', '/etc/hadoop/conf',
- to='/etc/hadoop/conf.empty',
- not_if='ls /etc/hadoop/conf'
- )
- self.assertResourceCalled('File', os.path.join('/etc/hadoop/conf', 'hadoop-env.sh'),
- owner='root',
- content=InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']),
- replace=True
- )
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
action = ['delete'],
not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1',