You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2014/02/08 00:08:51 UTC
git commit: AMBARI-4562. Pig service check fails on secure cluster.
(aonishuk)
Updated Branches:
refs/heads/trunk 25d44e52e -> a687e1b20
AMBARI-4562. Pig service check fails on secure cluster. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a687e1b2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a687e1b2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a687e1b2
Branch: refs/heads/trunk
Commit: a687e1b201a1ec36c2625e3eeca3559644715abd
Parents: 25d44e5
Author: Andrew Onischuk <ao...@hortonworks.com>
Authored: Fri Feb 7 15:03:59 2014 -0800
Committer: Andrew Onischuk <ao...@hortonworks.com>
Committed: Fri Feb 7 15:03:59 2014 -0800
----------------------------------------------------------------------
.../main/python/resource_management/core/base.py | 2 --
.../main/python/resource_management/core/logger.py | 4 ++++
.../libraries/providers/execute_hadoop.py | 6 +-----
.../libraries/resources/execute_hadoop.py | 2 +-
.../1.3.2/services/PIG/package/scripts/params.py | 3 +++
.../services/PIG/package/scripts/service_check.py | 16 +++++++++++-----
.../2.0.6/services/PIG/package/scripts/params.py | 3 +++
.../services/PIG/package/scripts/service_check.py | 16 +++++++++++-----
.../test/python/stacks/1.3.2/PIG/test_pig_client.py | 10 ++++++++++
.../stacks/1.3.2/PIG/test_pig_service_check.py | 9 ++++++++-
.../test/python/stacks/2.0.6/PIG/test_pig_client.py | 10 ++++++++++
.../stacks/2.0.6/PIG/test_pig_service_check.py | 10 ++++++++--
.../src/test/python/stacks/utils/RMFTestCase.py | 13 +++++++++++--
13 files changed, 81 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-agent/src/main/python/resource_management/core/base.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/core/base.py b/ambari-agent/src/main/python/resource_management/core/base.py
index ff1d7f6..52f1dff 100644
--- a/ambari-agent/src/main/python/resource_management/core/base.py
+++ b/ambari-agent/src/main/python/resource_management/core/base.py
@@ -143,8 +143,6 @@ class Resource(object):
self.arguments[key] = arg.validate(value)
except InvalidArgument, exc:
raise InvalidArgument("%s %s" % (self, exc))
-
- Logger.debug("New resource %s: %s" % (self, self.arguments))
if not self.env.test_mode:
self.env.run()
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-agent/src/main/python/resource_management/core/logger.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/core/logger.py b/ambari-agent/src/main/python/resource_management/core/logger.py
index 0aa62ad..a22cd86 100644
--- a/ambari-agent/src/main/python/resource_management/core/logger.py
+++ b/ambari-agent/src/main/python/resource_management/core/logger.py
@@ -22,6 +22,7 @@ Ambari Agent
__all__ = ["Logger"]
import logging
+from resource_management.libraries.script.config_dictionary import UnknownConfiguration
class Logger:
logger = logging.getLogger("resource_management")
@@ -67,6 +68,9 @@ class Logger:
# usually too long
elif isinstance(y, dict):
val = "..."
+ # for configs which didn't come
+ elif isinstance(y, UnknownConfiguration):
+ val = "[EMPTY]"
# correctly output 'mode' (as they are octal values like 0755)
elif y and x == 'mode':
val = oct(y)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py b/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
index afd7bc3..5a159da 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
@@ -29,11 +29,7 @@ class ExecuteHadoopProvider(Provider):
keytab = self.resource.keytab
conf_dir = self.resource.conf_dir
command = self.resource.command
-
- if self.resource.principal:
- principal = self.resource.user
- else:
- principal = self.resource.principal
+ principal = self.resource.principal
if isinstance(command, (list, tuple)):
command = ' '.join(pipes.quote(x) for x in command)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-agent/src/main/python/resource_management/libraries/resources/execute_hadoop.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/resources/execute_hadoop.py b/ambari-agent/src/main/python/resource_management/libraries/resources/execute_hadoop.py
index c271c90..94daf5b 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/resources/execute_hadoop.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/resources/execute_hadoop.py
@@ -31,7 +31,7 @@ class ExecuteHadoop(Resource):
try_sleep = ResourceArgument(default=0) # seconds
user = ResourceArgument()
logoutput = BooleanArgument(default=False)
- principal = ResourceArgument()
+ principal = ResourceArgument(default=lambda obj: obj.user)
conf_dir = ResourceArgument()
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py
index 781e635..636f882 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py
@@ -30,6 +30,9 @@ hadoop_conf_dir = "/etc/hadoop/conf"
hdfs_user = config['configurations']['global']['hdfs_user']
smokeuser = config['configurations']['global']['smokeuser']
user_group = config['configurations']['global']['user_group']
+security_enabled = config['configurations']['global']['security_enabled']
+smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
# not supporting 32 bit jdk.
java64_home = config['hostLevelParams']['java_home']
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/service_check.py
index 3cca087..1cfced2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/service_check.py
@@ -39,7 +39,11 @@ class PigServiceCheck(Script):
tries = 3,
try_sleep = 5,
user = params.smokeuser,
- conf_dir = params.hadoop_conf_dir
+ conf_dir = params.hadoop_conf_dir,
+ # for kinit run
+ keytab = params.smoke_user_keytab,
+ security_enabled = params.security_enabled,
+ kinit_path_local = params.kinit_path_local
)
File( '/tmp/pigSmoke.sh',
@@ -61,11 +65,13 @@ class PigServiceCheck(Script):
)
def main():
+ command_type = "service_check"
+ print "Running "+command_type
+ command_data_file = '/var/lib/ambari-agent/data/command-2.json'
+ basedir = '/root/ambari/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package'
+ stroutfile = '/1.txt'
import sys
- command_type = 'service_check'
- command_data_file = '/root/workspace/Pig/input.json'
- basedir = '/root/workspace/Pig/main'
- sys.argv = ["", command_type, command_data_file, basedir]
+ sys.argv = ["", command_type, command_data_file, basedir, stroutfile]
PigServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
index 781e635..636f882 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
@@ -30,6 +30,9 @@ hadoop_conf_dir = "/etc/hadoop/conf"
hdfs_user = config['configurations']['global']['hdfs_user']
smokeuser = config['configurations']['global']['smokeuser']
user_group = config['configurations']['global']['user_group']
+security_enabled = config['configurations']['global']['security_enabled']
+smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
# not supporting 32 bit jdk.
java64_home = config['hostLevelParams']['java_home']
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
index 3cca087..1cfced2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
@@ -39,7 +39,11 @@ class PigServiceCheck(Script):
tries = 3,
try_sleep = 5,
user = params.smokeuser,
- conf_dir = params.hadoop_conf_dir
+ conf_dir = params.hadoop_conf_dir,
+ # for kinit run
+ keytab = params.smoke_user_keytab,
+ security_enabled = params.security_enabled,
+ kinit_path_local = params.kinit_path_local
)
File( '/tmp/pigSmoke.sh',
@@ -61,11 +65,13 @@ class PigServiceCheck(Script):
)
def main():
+ command_type = "service_check"
+ print "Running "+command_type
+ command_data_file = '/var/lib/ambari-agent/data/command-2.json'
+ basedir = '/root/ambari/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package'
+ stroutfile = '/1.txt'
import sys
- command_type = 'service_check'
- command_data_file = '/root/workspace/Pig/input.json'
- basedir = '/root/workspace/Pig/main'
- sys.argv = ["", command_type, command_data_file, basedir]
+ sys.argv = ["", command_type, command_data_file, basedir, stroutfile]
PigServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py b/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py
index 954c098..bdb0a2f 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py
@@ -39,6 +39,11 @@ class TestHcatClient(RMFTestCase):
self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
owner = 'hdfs',
)
+ self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
+ owner = 'hdfs',
+ group = 'hadoop',
+ mode = 0644,
+ )
self.assertNoMoreResources()
def test_configure_secured(self):
@@ -58,4 +63,9 @@ class TestHcatClient(RMFTestCase):
self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
owner = 'hdfs',
)
+ self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
+ owner = 'hdfs',
+ group = 'hadoop',
+ mode = 0644,
+ )
self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_service_check.py b/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_service_check.py
index ceb5440..af579e3 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_service_check.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_service_check.py
@@ -34,6 +34,9 @@ class TestServiceCheck(RMFTestCase):
tries = 3,
user = 'ambari-qa',
conf_dir = '/etc/hadoop/conf',
+ security_enabled = False,
+ keytab = 'mockOfUnknownConfiguration',
+ kinit_path_local = '/usr/bin/kinit',
)
self.assertResourceCalled('File', '/tmp/pigSmoke.sh',
@@ -53,7 +56,7 @@ class TestServiceCheck(RMFTestCase):
user = 'ambari-qa',
conf_dir = '/etc/hadoop/conf',
)
-
+ self.assertNoMoreResources()
def test_configure_secured(self):
self.executeScript("1.3.2/services/PIG/package/scripts/service_check.py",
@@ -67,6 +70,9 @@ class TestServiceCheck(RMFTestCase):
tries = 3,
user = 'ambari-qa',
conf_dir = '/etc/hadoop/conf',
+ security_enabled = True,
+ keytab = '/etc/security/keytabs/smokeuser.headless.keytab',
+ kinit_path_local = '/usr/bin/kinit',
)
self.assertResourceCalled('File', '/tmp/pigSmoke.sh',
@@ -86,3 +92,4 @@ class TestServiceCheck(RMFTestCase):
user = 'ambari-qa',
conf_dir = '/etc/hadoop/conf',
)
+ self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
index 00c573b..810d6df 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
@@ -39,6 +39,11 @@ class TestHcatClient(RMFTestCase):
self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
owner = 'hdfs',
)
+ self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
+ owner = 'hdfs',
+ group = 'hadoop',
+ mode = 0644,
+ )
self.assertNoMoreResources()
def test_configure_secured(self):
@@ -58,4 +63,9 @@ class TestHcatClient(RMFTestCase):
self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
owner = 'hdfs',
)
+ self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
+ owner = 'hdfs',
+ group = 'hadoop',
+ mode = 0644,
+ )
self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py
index c3d3fa2..a8fd23f 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py
@@ -28,12 +28,14 @@ class TestServiceCheck(RMFTestCase):
command = "service_check",
config_file="default.json"
)
-
self.assertResourceCalled('ExecuteHadoop', 'dfs -rmr pigsmoke.out passwd; hadoop dfs -put /etc/passwd passwd ',
try_sleep = 5,
tries = 3,
user = 'ambari-qa',
conf_dir = '/etc/hadoop/conf',
+ security_enabled = False,
+ keytab = 'mockOfUnknownConfiguration',
+ kinit_path_local = '/usr/bin/kinit'
)
self.assertResourceCalled('File', '/tmp/pigSmoke.sh',
@@ -53,7 +55,7 @@ class TestServiceCheck(RMFTestCase):
user = 'ambari-qa',
conf_dir = '/etc/hadoop/conf',
)
-
+ self.assertNoMoreResources()
def test_configure_secured(self):
self.executeScript("2.0.6/services/PIG/package/scripts/service_check.py",
@@ -67,6 +69,9 @@ class TestServiceCheck(RMFTestCase):
tries = 3,
user = 'ambari-qa',
conf_dir = '/etc/hadoop/conf',
+ security_enabled = True,
+ keytab = '/etc/security/keytabs/smokeuser.headless.keytab',
+ kinit_path_local = '/usr/bin/kinit'
)
self.assertResourceCalled('File', '/tmp/pigSmoke.sh',
@@ -86,3 +91,4 @@ class TestServiceCheck(RMFTestCase):
user = 'ambari-qa',
conf_dir = '/etc/hadoop/conf',
)
+ self.assertNoMoreResources()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a687e1b2/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
index d3e9172..bf21bad 100644
--- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
+++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
@@ -29,6 +29,7 @@ from mock.mock import MagicMock, patch
from resource_management.core.environment import Environment
from resource_management.libraries.script.config_dictionary import ConfigDictionary
from resource_management.libraries.script.script import Script
+from resource_management.libraries.script import config_dictionary
import platform
PATH_TO_STACKS = os.path.normpath("main/resources/stacks/HDP")
@@ -81,7 +82,8 @@ class RMFTestCase(TestCase):
with patch.object(Script, 'install_packages'):
with patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local):
with patch.object(platform, 'linux_distribution', return_value=os_type):
- method(RMFTestCase.env)
+ with patch.object(config_dictionary, 'UnknownConfiguration', return_value='mockOfUnknownConfiguration'):
+ method(RMFTestCase.env)
def getConfig(self):
return self.config_dict
@@ -113,7 +115,14 @@ class RMFTestCase(TestCase):
for resource in RMFTestCase.env.resource_list:
print "'{0}', {1},".format(resource.__class__.__name__, self._ppformat(resource.name))
for k,v in resource.arguments.iteritems():
- print " {0} = {1},".format(k, self._ppformat(v))
+
+ # correctly output octal mode numbers
+ if k == 'mode' and isinstance( v, int ):
+ val = oct(v)
+ else:
+ val = self._ppformat(v)
+
+ print " {0} = {1},".format(k, val)
print
def assertResourceCalled(self, resource_type, name, **kwargs):