You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2013/11/01 16:28:24 UTC
[13/15] git commit: AMBARI-3663. Resource Management. Easy formatting
of configurations (Andrew Onischuk via dlysnichenko)
AMBARI-3663. Resource Management. Easy formatting of configurations (Andrew Onischuk via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/002060f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/002060f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/002060f8
Branch: refs/heads/trunk
Commit: 002060f84dc18cdb03c38924b3fe5287f07c8d96
Parents: fa817f6
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Nov 1 17:23:57 2013 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Nov 1 17:23:57 2013 +0200
----------------------------------------------------------------------
.../python/resource_management/core/base.py | 7 ++--
.../resource_management/core/environment.py | 38 +++++++++++---------
.../python/resource_management/core/source.py | 32 +++++++++--------
.../python/resource_management/core/utils.py | 36 +++++++------------
.../libraries/providers/execute_hadoop.py | 24 ++++++-------
5 files changed, 65 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/002060f8/ambari-agent/src/main/python/resource_management/core/base.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/core/base.py b/ambari-agent/src/main/python/resource_management/core/base.py
index fb2e26d..0ad34ce 100644
--- a/ambari-agent/src/main/python/resource_management/core/base.py
+++ b/ambari-agent/src/main/python/resource_management/core/base.py
@@ -5,7 +5,7 @@ __all__ = ["Resource", "ResourceArgument", "ForcedListArgument",
import logging
from resource_management.core.exceptions import Fail, InvalidArgument
-from resource_management.core.environment import Environment, Substitutor
+from resource_management.core.environment import Environment
class ResourceArgument(object):
def __init__(self, default=None, required=False, allow_override=False):
@@ -95,7 +95,6 @@ class Resource(object):
name = name[0]
env = env or Environment.get_instance()
- name = Substitutor.substitute(name)
provider = provider or getattr(cls, 'provider', None)
r_type = cls.__name__
@@ -123,7 +122,7 @@ class Resource(object):
return
self.env = env or Environment.get_instance()
- self.name = Substitutor.substitute(name)
+ self.name = name
self.provider = provider or getattr(self, 'provider', None)
@@ -135,7 +134,7 @@ class Resource(object):
raise Fail("%s received unsupported argument %s" % (self, key))
else:
try:
- self.arguments[key] = Substitutor.substitute(arg.validate(value))
+ self.arguments[key] = arg.validate(value)
except InvalidArgument, exc:
raise InvalidArgument("%s %s" % (self, exc))
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/002060f8/ambari-agent/src/main/python/resource_management/core/environment.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/core/environment.py b/ambari-agent/src/main/python/resource_management/core/environment.py
index d6557b0..fb0536d 100644
--- a/ambari-agent/src/main/python/resource_management/core/environment.py
+++ b/ambari-agent/src/main/python/resource_management/core/environment.py
@@ -1,19 +1,20 @@
#!/usr/bin/env python
-__all__ = ["Environment"]
+__all__ = ["Environment","format"]
import logging
+import sys
import os
import shutil
import time
from datetime import datetime
+from string import Formatter
from resource_management.core import shell
from resource_management.core.exceptions import Fail
from resource_management.core.providers import find_provider
-from resource_management.core.utils import AttributeDictionary, ParamsAttributeDictionary
+from resource_management.core.utils import AttributeDictionary, checked_unite
from resource_management.core.system import System
-from string import Template
class Environment(object):
@@ -26,6 +27,7 @@ class Environment(object):
@param params: configurations dictionary (this will be accessible in the templates)
"""
self.log = logging.getLogger("resource_management")
+ self.formatter = ConfigurationFormatter()
self.reset(basedir, params)
def reset(self, basedir, params):
@@ -44,7 +46,7 @@ class Environment(object):
# dir where templates,failes dirs are
'basedir': basedir,
# variables, which can be used in templates
- 'params': ParamsAttributeDictionary(Substitutor, params.copy()),
+ 'params': params.copy(),
})
def backup_file(self, path):
@@ -169,18 +171,20 @@ class Environment(object):
self.resource_list = state['resource_list']
self.delayed_actions = state['delayed_actions']
-
-class Substitutor():
- log = logging.getLogger("resource_management.resource")
- @staticmethod
- def substitute(val):
+class ConfigurationFormatter(Formatter):
+ def format(self, format_string, *args, **kwargs):
env = Environment.get_instance()
- dic = env.config.params
+ variables = kwargs
+ params = env.config.params
- if dic and isinstance(val, str):
- result = Template(val).substitute(dic)
- if '$' in val:
- Substitutor.log.debug("%s after substitution is %s", val, result)
- return result
-
- return val
+ result = checked_unite(variables, params)
+ return self.vformat(format_string, args, result)
+
+def format(format_string, *args, **kwargs):
+ env = Environment.get_instance()
+ variables = sys._getframe(1).f_locals
+
+ result = checked_unite(kwargs, variables)
+ result.pop("self", None) # self kwarg would result in an error
+ return env.formatter.format(format_string, args, **result)
+
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/002060f8/ambari-agent/src/main/python/resource_management/core/source.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/core/source.py b/ambari-agent/src/main/python/resource_management/core/source.py
index 2fcd395..b27bacb 100644
--- a/ambari-agent/src/main/python/resource_management/core/source.py
+++ b/ambari-agent/src/main/python/resource_management/core/source.py
@@ -1,5 +1,6 @@
from __future__ import with_statement
-from resource_management.core.environment import Substitutor, Environment
+from resource_management.core.environment import Environment
+from resource_management.core.utils import checked_unite
__all__ = ["Source", "Template", "InlineTemplate", "StaticFile", "DownloadSource"]
@@ -10,9 +11,9 @@ import urlparse
class Source(object):
- def __init__(self, name, env=None):
- self.name = Substitutor.substitute(name)
- self.env = env or Environment.get_instance()
+ def __init__(self, name):
+ self.env = Environment.get_instance()
+ self.name = name
def get_content(self):
raise NotImplementedError()
@@ -25,8 +26,8 @@ class Source(object):
class StaticFile(Source):
- def __init__(self, name, env=None):
- super(StaticFile, self).__init__(name, env)
+ def __init__(self, name):
+ super(StaticFile, self).__init__(name)
def get_content(self):
# absolute path
@@ -42,7 +43,7 @@ class StaticFile(Source):
try:
- from jinja2 import Environment as JinjaEnvironment, BaseLoader, TemplateNotFound, FunctionLoader
+ from jinja2 import Environment as JinjaEnvironment, BaseLoader, TemplateNotFound, FunctionLoader, StrictUndefined
except ImportError:
class Template(Source):
def __init__(self, name, variables=None, env=None):
@@ -73,14 +74,17 @@ else:
return source, path, lambda: mtime == os.path.getmtime(path)
class Template(Source):
- def __init__(self, name, variables=None, env=None):
- super(Template, self).__init__(name, env)
+ def __init__(self, name, **kwargs):
+ """
+ @param kwargs: Additional variables passed to template
+ """
+ super(Template, self).__init__(name)
params = self.env.config.params
- variables = params if params else variables
+ variables = checked_unite(params, kwargs)
self.context = variables.copy() if variables else {}
if not hasattr(self, 'template_env'):
self.template_env = JinjaEnvironment(loader=TemplateLoader(self.env),
- autoescape=False)
+ autoescape=False, undefined=StrictUndefined)
self.template = self.template_env.get_template(self.name)
def get_content(self):
@@ -94,13 +98,13 @@ else:
return rendered + "\n" if not rendered.endswith('\n') else rendered
class InlineTemplate(Template):
- def __init__(self, name, variables=None, env=None):
+ def __init__(self, name, **kwargs):
self.template_env = JinjaEnvironment(loader=FunctionLoader(lambda text: text))
- super(InlineTemplate, self).__init__(name, variables, env)
+ super(InlineTemplate, self).__init__(name, **kwargs)
class DownloadSource(Source):
- def __init__(self, name, cache=True, md5sum=None, env=None):
+ def __init__(self, name, cache=True, md5sum=None):
super(DownloadSource, self).__init__(name)
self.url = self.name
self.md5sum = md5sum
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/002060f8/ambari-agent/src/main/python/resource_management/core/utils.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/core/utils.py b/ambari-agent/src/main/python/resource_management/core/utils.py
index 4cd5207..028b911 100644
--- a/ambari-agent/src/main/python/resource_management/core/utils.py
+++ b/ambari-agent/src/main/python/resource_management/core/utils.py
@@ -54,7 +54,10 @@ class AttributeDictionary(object):
return self._dict.__repr__()
def __unicode__(self):
- return self._dict.__unicode__()
+ if isinstance(self._dict, str):
+ return self._dict.__unicode__()
+ else:
+ return str(self._dict)
def __str__(self):
return self._dict.__str__()
@@ -68,27 +71,12 @@ class AttributeDictionary(object):
def __setstate__(self, state):
super(AttributeDictionary, self).__setattr__("_dict", state)
-class ParamsAttributeDictionary(AttributeDictionary):
- """
- This class can store user parameters
- and it supports some features necessary for substitution to work.
- """
- def __init__(self, substitutor, *args, **kwargs):
- super(ParamsAttributeDictionary, self).__init__(*args, **kwargs)
- super(AttributeDictionary, self).__setattr__("substitutor", substitutor)
-
- def __getitem__(self, name):
- try:
- return self._convert_value(self._dict[name])
- except KeyError as ex:
- raise Fail("Configuration $%s not found!" % str(ex).strip("'"))
+def checked_unite(dict1, dict2):
+ for key in dict1:
+ if key in dict2:
+ raise Fail("Variable '%s' already exists more than once as a variable/configuration/kwarg parameter. Cannot evaluate it." % key)
- def copy(self):
- # don't allow real copying to be able to change params passed to jinja2
- return self
-
- def __unicode__(self):
- if isinstance(self._dict, str):
- return self._dict.__unicode__()
- else:
- return str(self._dict)
\ No newline at end of file
+ result = dict1.copy()
+ result.update(dict2)
+
+ return result
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/002060f8/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py b/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
index 1fbfab0..39788bb 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
@@ -2,21 +2,19 @@ from resource_management import *
class ExecuteHadoopProvider(Provider):
def action_run(self):
- if self.resource.security_enabled and not self.resource.kinit_override:
- kinit_if_needed = "su - {user} -c '{kinit_path_local} -kt {keytab} {principal}'".format(
- user=self.resource.user, kinit_path_local=self.resource.kinit_path_local,
- keytab=self.resource.keytab, principal=self.resource.principal)
- else:
- kinit_if_needed = ""
-
- cmd = "hadoop --config {conf_dir} {command}".format(conf_dir=self.resource.conf_dir, command=self.resource.command)
+ kinit_path_local = self.resource.kinit_path_local
+ keytab = self.resource.keytab
+ principal = self.resource.principal
+ conf_dir = self.resource.conf_dir
+ command = self.resource.command
- if kinit_if_needed != "":
- Execute ( kinit_if_needed,
- path = ['/bin']
+ if self.resource.security_enabled and not self.resource.kinit_override:
+ Execute ((kinit_path_local, '-kt', keytab, principal),
+ path = ['/bin'],
+ user = self.resource.user
)
-
- Execute ( cmd,
+
+ Execute (('hadoop', '--config', conf_dir, command),
user = self.resource.user,
tries = self.resource.tries,
try_sleep = self.resource.try_sleep,