You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/03/25 16:46:18 UTC
ambari git commit: AMBARI-10200. Storm secure config not visible
through Ambari storm config UI (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk 90a88c6aa -> 008cc6bad
AMBARI-10200. Storm secure config not visible through Ambari storm config UI (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/008cc6ba
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/008cc6ba
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/008cc6ba
Branch: refs/heads/trunk
Commit: 008cc6bad32f054de1686af0921ca0a2d024fa5a
Parents: 90a88c6
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Mar 25 17:46:08 2015 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Wed Mar 25 17:46:08 2015 +0200
----------------------------------------------------------------------
.../0.1.0/configuration/storm-site.xml | 34 ++++++++++
.../AMBARI_METRICS/0.1.0/metainfo.xml | 4 ++
.../0.9.1.2.1/configuration/storm-site.xml | 5 ++
.../STORM/0.9.1.2.1/kerberos.json | 7 ++-
.../STORM/0.9.1.2.1/package/scripts/params.py | 6 ++
.../STORM/0.9.1.2.1/package/scripts/storm.py | 10 ++-
.../0.9.1.2.1/package/scripts/yaml_utils.py | 27 ++++++--
.../0.9.1.2.1/package/templates/storm.yaml.j2 | 65 --------------------
.../services/STORM/configuration/storm-site.xml | 6 +-
.../services/STORM/configuration/storm-site.xml | 6 +-
.../python/stacks/2.1/STORM/test_storm_base.py | 29 +++++----
.../stacks/2.1/configs/default-storm-start.json | 1 +
.../stacks/2.1/configs/secured-storm-start.json | 3 +-
13 files changed, 103 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml
new file mode 100644
index 0000000..35456d7
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="true">
+ <property>
+ <name>topology.metrics.consumer.register</name>
+ <value>[{'class': 'org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink', 'parallelism.hint': 1}]</value>
+ <description>Topology metrics consumer register parameters.</description>
+ </property>
+ <property>
+ <name>metrics.reporter.register</name>
+ <value>org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter</value>
+ <description>Topology metrics reporter.</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
index ac954e6..39b640c 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
@@ -129,6 +129,10 @@
<config-type>ams-hbase-log4j</config-type>
</configuration-dependencies>
+ <excluded-config-types>
+ <config-type>storm-site</config-type>
+ </excluded-config-types>
+
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
index b6a2cf2..5e96fb1 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
@@ -37,6 +37,11 @@
have permission to read/write from this location.</description>
</property>
<property>
+ <name>storm.log.dir</name>
+ <value>{{log_dir}}</value>
+ <description>Log directory for Storm.</description>
+ </property>
+ <property>
<name>storm.zookeeper.servers</name>
<value>['localhost']</value>
<description>A list of hosts of ZooKeeper servers used to manage the cluster.</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/kerberos.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/kerberos.json
index 50b92ff..94eaae1 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/kerberos.json
@@ -37,7 +37,12 @@
"drpc.authorizer": "backtype.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer",
"ui.filter": "org.apache.hadoop.security.authentication.server.AuthenticationFilter",
"storm.principal.tolocal": "backtype.storm.security.auth.KerberosPrincipalToLocal",
- "supervisor.enable": "true"
+ "supervisor.enable": "true",
+ "storm.zookeeper.superACL": "sasl:{{storm_bare_jaas_principal}}",
+ "java.security.auth.login.config": "{{conf_dir}}/storm_jaas.conf",
+ "nimbus.admins": "['{{storm_user}}']",
+ "nimbus.supervisor.users": "['{{storm_user}}']",
+ "ui.filter.params": "{'type': 'kerberos', 'kerberos.principal': '{{storm_ui_jaas_principal}}', 'kerberos.keytab': '{{storm_ui_keytab_path}}', 'kerberos.name.rules': 'DEFAULT'}"
}
}
],
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py
index 0e432ba..2d575b0 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py
@@ -111,6 +111,12 @@ if security_enabled:
nimbus_bare_jaas_principal = get_bare_principal(_nimbus_principal_name)
nimbus_keytab_path = config['configurations']['storm-env']['nimbus_keytab']
+if stack_is_hdp22_or_further:
+ if security_enabled:
+ storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.secure.transport']
+ else:
+ storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.nonsecure.transport']
+
ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
index 7a2488d..3ae3f92 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
@@ -18,6 +18,7 @@ limitations under the License.
"""
+import resource_management
from resource_management.core.resources import File
from resource_management.core.resources import Execute
from resource_management.core.resources import Directory
@@ -26,7 +27,7 @@ from resource_management.libraries.resources.template_config import TemplateConf
from resource_management.libraries.functions.format import format
from resource_management.core.source import Template
from resource_management.libraries.functions import compare_versions
-from yaml_utils import escape_yaml_propetry
+from yaml_utils import yaml_config_template
def storm():
import params
@@ -58,12 +59,9 @@ def storm():
)
configurations = params.config['configurations']['storm-site']
-
+
File(format("{conf_dir}/storm.yaml"),
- content=Template(
- "storm.yaml.j2",
- extra_imports=[escape_yaml_propetry],
- configurations = configurations),
+ content=yaml_config_template(configurations),
owner=params.storm_user,
group=params.user_group
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py
index 8187666..0034123 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py
@@ -19,6 +19,8 @@ limitations under the License.
"""
import re
+import resource_management
+from resource_management.core.source import InlineTemplate
def escape_yaml_propetry(value):
unquouted = False
@@ -26,10 +28,10 @@ def escape_yaml_propetry(value):
if value in unquouted_values:
unquouted = True
- # if is list [a,b,c]
- if re.match('^\w*\[.+\]\w*$', value):
+ # if is list [a,b,c] or dictionary {a: v, b: v2, c: v3}
+ if re.match('^\w*\[.+\]\w*$', value) or re.match('^\w*\{.+\}\w*$', value):
unquouted = True
-
+
try:
int(value)
unquouted = True
@@ -46,4 +48,21 @@ def escape_yaml_propetry(value):
value = value.replace("'","''")
value = "'"+value+"'"
- return value
\ No newline at end of file
+ return value
+
+def replace_jaas_placeholder(name, security_enabled, conf_dir):
+ if name.find('_JAAS_PLACEHOLDER') > -1:
+ if security_enabled:
+ return name.replace('_JAAS_PLACEHOLDER', '-Djava.security.auth.login.config=' + conf_dir + '/storm_jaas.conf')
+ else:
+ return name.replace('_JAAS_PLACEHOLDER', '')
+ else:
+ return name
+
+storm_yaml_template = """{% for key, value in configurations|dictsort if not key.startswith('_') %}{{key}} : {{ escape_yaml_propetry(replace_jaas_placeholder(resource_management.core.source.InlineTemplate(value).get_content().strip(), security_enabled, conf_dir)) }}
+{% endfor %}"""
+
+def yaml_config_template(configurations):
+ return InlineTemplate(storm_yaml_template, configurations=configurations,
+ extra_imports=[escape_yaml_propetry, replace_jaas_placeholder, resource_management,
+ resource_management.core, resource_management.core.source])
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.yaml.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.yaml.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.yaml.j2
deleted file mode 100644
index cc3e631..0000000
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.yaml.j2
+++ /dev/null
@@ -1,65 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-{% macro replace_jaas_placeholder(name) -%}
-{% if name.find('_JAAS_PLACEHOLDER') > -1 -%}
- {%- if security_enabled -%}
- {{ name.replace('_JAAS_PLACEHOLDER', '-Djava.security.auth.login.config=' +conf_dir + '/storm_jaas.conf') }}
- {%- else -%}
- {{ name.replace('_JAAS_PLACEHOLDER', '') }}
- {%- endif -%}
-{%- else -%}
- {{name}}
-{%- endif -%}
-{%- endmacro %}
-
-{% for key, value in configurations|dictsort if not key.startswith('_') %}
-{{key}} : {{ escape_yaml_propetry(replace_jaas_placeholder(value)) }}
-{% endfor %}
-
-{% if stack_is_hdp22_or_further %}
-storm.thrift.transport : "{% if security_enabled %}{{configurations['_storm.thrift.secure.transport']}}{% else %}{{configurations['_storm.thrift.nonsecure.transport']}}{% endif %}"
-{% endif %}
-
-{% if security_enabled and stack_is_hdp22_or_further %}
-#
-# Kerberos security section. For the reference please use: https://github.com/hortonworks/storm/blob/champlain/SECURITY.md for details
-#
-
-storm.zookeeper.superACL: "sasl:{{storm_bare_jaas_principal}}"
-java.security.auth.login.config: "{{conf_dir}}/storm_jaas.conf"
-nimbus.admins:
- - "{{storm_user}}"
-nimbus.supervisor.users:
- - "{{storm_user}}"
-
-ui.filter.params:
- "type": "kerberos"
- "kerberos.principal": "{{storm_ui_jaas_principal}}"
- "kerberos.keytab": "{{storm_ui_keytab_path}}"
- "kerberos.name.rules": "DEFAULT"
-
-{% endif %}
-
-{% if has_metric_collector %}
-topology.metrics.consumer.register:
- - class: "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink"
- parallelism.hint: 1
-metrics.reporter.register: "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"
-{% endif %}
-
-storm.log.dir: "{{log_dir}}"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml
index befb5f8..c933099 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml
@@ -24,10 +24,8 @@
<property>
<name>storm.thrift.transport</name>
- <value>--</value>
- <deleted>true</deleted>
- <description>This is calculated field that is based of security state and _storm.thrift.nonsecure.transport and _storm.thrift.secure.transport
- </description>
+ <value>{{storm_thrift_transport}}</value>
+ <description>The transport plug-in that used for Thrift client/server communication.</description>
</property>
<property>
<name>_storm.thrift.nonsecure.transport</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/STORM/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/STORM/configuration/storm-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/STORM/configuration/storm-site.xml
index c0528dd..1a50e8f 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/STORM/configuration/storm-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/STORM/configuration/storm-site.xml
@@ -24,10 +24,8 @@
<property>
<name>storm.thrift.transport</name>
- <value>--</value>
- <deleted>true</deleted>
- <description>This is calculated field that is based of security state and _storm.thrift.nonsecure.transport and _storm.thrift.secure.transport
- </description>
+ <value>{{storm_thrift_transport}}</value>
+ <description>The transport plug-in that used for Thrift client/server communication.</description>
</property>
<property>
<name>_storm.thrift.nonsecure.transport</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py
index 2b475d1..f7b634c 100644
--- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py
+++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py
@@ -19,8 +19,8 @@ limitations under the License.
'''
from mock.mock import MagicMock, call, patch
-from stacks.utils.RMFTestCase import *
import resource_management.core.source
+from stacks.utils.RMFTestCase import *
import re
@@ -62,7 +62,7 @@ class TestStormBase(RMFTestCase):
storm_yarn_content = self.call_storm_template_and_assert()
self.assertTrue(storm_yarn_content.find('_JAAS_PLACEHOLDER') == -1, 'Placeholder have to be substituted')
-
+
self.assertResourceCalled('File', '/etc/storm/conf/storm-env.sh',
owner = 'storm',
content = InlineTemplate(self.getConfig()['configurations']['storm-env']['content'])
@@ -111,18 +111,17 @@ class TestStormBase(RMFTestCase):
owner = 'storm',
)
return storm_yarn_content
-
+
def call_storm_template_and_assert(self):
import yaml_utils
- storm_yarn_template = Template(
- "storm.yaml.j2",
- extra_imports=[yaml_utils.escape_yaml_propetry],
- configurations = self.getConfig()['configurations']['storm-site'])
- storm_yarn_content = storm_yarn_template.get_content()
-
- self.assertResourceCalled('File', '/etc/storm/conf/storm.yaml',
- owner = 'storm',
- content= storm_yarn_template,
- group = 'hadoop'
- )
- return storm_yarn_content
+
+ with RMFTestCase.env as env:
+ storm_yarn_temlate = yaml_utils.yaml_config_template(self.getConfig()['configurations']['storm-site'])
+
+ self.assertResourceCalled('File', '/etc/storm/conf/storm.yaml',
+ owner = 'storm',
+ content= storm_yarn_temlate,
+ group = 'hadoop'
+ )
+
+ return storm_yarn_temlate.get_content()
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json b/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json
index 900eba4..dfcbe92 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json
@@ -76,6 +76,7 @@
"worker.heartbeat.frequency.secs": "1",
"zmq.hwm": "0",
"storm.zookeeper.connection.timeout": "15000",
+ "storm.thrift.transport": "{{storm_thrift_transport}}",
"_storm.thrift.secure.transport": "SECURED_TRANSPORT_CLASS",
"storm.messaging.netty.server_worker_threads": "1",
"supervisor.worker.start.timeout.secs": "120",
http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json b/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json
index 6b8f69a..206f18d 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json
@@ -75,7 +75,8 @@
"topology.skip.missing.kryo.registrations": "false",
"worker.heartbeat.frequency.secs": "1",
"zmq.hwm": "0",
- "storm.zookeeper.connection.timeout": "15000",
+ "storm.zookeeper.connection.timeout": "15000",
+ "storm.thrift.transport": "{{storm_thrift_transport}}",
"_storm.thrift.secure.transport": "SECURED_TRANSPORT_CLASS",
"storm.messaging.netty.server_worker_threads": "1",
"supervisor.worker.start.timeout.secs": "120",