You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2018/06/28 20:21:59 UTC
[ambari] branch trunk updated: BUG-105479. Ranger Storm plugin
toggle behavior changes under different scenarios. (jaimin) (#1636)
This is an automated email from the ASF dual-hosted git repository.
jaimin pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 98da21f BUG-105479. Ranger Storm plugin toggle behavior changes under different scenarios. (jaimin) (#1636)
98da21f is described below
commit 98da21fabbc78a63047b7cbadfd7c7b80e7c1d5e
Author: Jetly <ja...@hortonworks.com>
AuthorDate: Thu Jun 28 13:21:56 2018 -0700
BUG-105479. Ranger Storm plugin toggle behavior changes under different scenarios. (jaimin) (#1636)
---
.../ambari/server/upgrade/UpgradeCatalog270.java | 37 +++++++
.../STORM/0.9.1/package/scripts/params_linux.py | 33 +++---
.../STORM/0.9.1/package/scripts/params_windows.py | 6 -
.../server/upgrade/UpgradeCatalog270Test.java | 121 +++++++++++++++++++++
.../stacks/2.1/configs/default-storm-start.json | 4 +-
.../stacks/2.1/configs/secured-storm-start.json | 6 +-
.../test/python/stacks/2.1/configs/secured.json | 4 +-
7 files changed, 178 insertions(+), 33 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java
index 706701e..d77a1dc 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java
@@ -77,6 +77,7 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.State;
import org.apache.ambari.server.state.kerberos.AbstractKerberosDescriptorContainer;
@@ -1042,6 +1043,7 @@ public class UpgradeCatalog270 extends AbstractUpgradeCatalog {
addUserAuthenticationSequence();
updateSolrConfigurations();
updateAmsConfigs();
+ updateStormConfigs();
}
protected void renameAmbariInfra() throws SQLException {
@@ -1843,4 +1845,39 @@ public class UpgradeCatalog270 extends AbstractUpgradeCatalog {
}
}
+ /**
+ * Removes storm-site configs that were present for ambari needs and
+ * sets the actual property `storm.thrift.transport` to the correct value
+ * @throws AmbariException
+ */
+ protected void updateStormConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ Set<String> removeProperties = Sets.newHashSet("_storm.thrift.nonsecure.transport",
+ "_storm.thrift.secure.transport");
+ String stormSecurityClassKey = "storm.thrift.transport";
+ String stormSecurityClassValue = "org.apache.storm.security.auth.SimpleTransportPlugin";
+ String stormSite = "storm-site";
+ for (final Cluster cluster : clusterMap.values()) {
+ Config config = cluster.getDesiredConfigByType(stormSite);
+ if (config != null) {
+ Map<String, String> stormSiteProperties = config.getProperties();
+ if (stormSiteProperties.containsKey(stormSecurityClassKey)) {
+ LOG.info("Updating " + stormSecurityClassKey);
+ if (cluster.getSecurityType() == SecurityType.KERBEROS) {
+ stormSecurityClassValue = "org.apache.storm.security.auth.kerberos.KerberosSaslTransportPlugin";
+ }
+ Map<String, String> updateProperty = Collections.singletonMap(stormSecurityClassKey, stormSecurityClassValue);
+ updateConfigurationPropertiesForCluster(cluster, stormSite, updateProperty, removeProperties,
+ true, false);
+ }
+ }
+ }
+ }
+ }
+ }
}
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
index 71ab037..7489480 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
@@ -152,27 +152,22 @@ if security_enabled:
_ambari_principal_name = default('/configurations/cluster-env/ambari_principal_name', None)
storm_keytab_path = config['configurations']['storm-env']['storm_keytab']
- if stack_supports_storm_kerberos:
- storm_ui_keytab_path = config['configurations']['storm-env']['storm_ui_keytab']
- _storm_ui_jaas_principal_name = config['configurations']['storm-env']['storm_ui_principal_name']
- storm_ui_jaas_principal = _storm_ui_jaas_principal_name.replace('_HOST',_hostname_lowercase)
- storm_bare_jaas_principal = get_bare_principal(_storm_principal_name)
- if _ambari_principal_name:
- ambari_bare_jaas_principal = get_bare_principal(_ambari_principal_name)
- _nimbus_principal_name = config['configurations']['storm-env']['nimbus_principal_name']
- nimbus_jaas_principal = _nimbus_principal_name.replace('_HOST', _hostname_lowercase)
- nimbus_bare_jaas_principal = get_bare_principal(_nimbus_principal_name)
- nimbus_keytab_path = config['configurations']['storm-env']['nimbus_keytab']
+ storm_ui_keytab_path = config['configurations']['storm-env']['storm_ui_keytab']
+ _storm_ui_jaas_principal_name = config['configurations']['storm-env']['storm_ui_principal_name']
+ storm_ui_jaas_principal = _storm_ui_jaas_principal_name.replace('_HOST',_hostname_lowercase)
+ storm_bare_jaas_principal = get_bare_principal(_storm_principal_name)
+ if _ambari_principal_name:
+ ambari_bare_jaas_principal = get_bare_principal(_ambari_principal_name)
+ _nimbus_principal_name = config['configurations']['storm-env']['nimbus_principal_name']
+ nimbus_jaas_principal = _nimbus_principal_name.replace('_HOST', _hostname_lowercase)
+ nimbus_bare_jaas_principal = get_bare_principal(_nimbus_principal_name)
+ nimbus_keytab_path = config['configurations']['storm-env']['nimbus_keytab']
kafka_bare_jaas_principal = None
-if stack_supports_storm_kerberos:
- if security_enabled:
- storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.secure.transport']
- # generate KafkaClient jaas config if kafka is kerberoized
- _kafka_principal_name = default("/configurations/kafka-env/kafka_principal_name", None)
- kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name)
- else:
- storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.nonsecure.transport']
+if security_enabled:
+ # generate KafkaClient jaas config if kafka is kerberoized
+ _kafka_principal_name = default("/configurations/kafka-env/kafka_principal_name", None)
+ kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name)
set_instanceId = "false"
if 'cluster-env' in config['configurations'] and \
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_windows.py
index a758375..9d040e7 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_windows.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_windows.py
@@ -47,12 +47,6 @@ else:
actual_topology_max_replication_wait_time_sec = default_topology_max_replication_wait_time_sec
actual_topology_min_replication_count = default_topology_min_replication_count
-if stack_is_hdp23_or_further:
- if security_enabled:
- storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.secure.transport']
- else:
- storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.nonsecure.transport']
-
service_map = {
"nimbus" : nimbus_win_service_name,
"supervisor" : supervisor_win_service_name,
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java
index 50eb869..402ff3c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java
@@ -205,6 +205,7 @@ import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponentHostFactory;
import org.apache.ambari.server.state.ServiceFactory;
@@ -324,6 +325,7 @@ public class UpgradeCatalog270Test {
Method updateKerberosDescriptorArtifacts = UpgradeCatalog270.class.getSuperclass().getDeclaredMethod("updateKerberosDescriptorArtifacts");
Method updateSolrConfigurations = UpgradeCatalog270.class.getDeclaredMethod("updateSolrConfigurations");
Method updateAmsConfigurations = UpgradeCatalog270.class.getDeclaredMethod("updateAmsConfigs");
+ Method updateStormConfigurations = UpgradeCatalog270.class.getDeclaredMethod("updateStormConfigs");
UpgradeCatalog270 upgradeCatalog270 = createMockBuilder(UpgradeCatalog270.class)
.addMockedMethod(showHcatDeletedUserMessage)
@@ -339,6 +341,7 @@ public class UpgradeCatalog270Test {
.addMockedMethod(updateKerberosDescriptorArtifacts)
.addMockedMethod(updateSolrConfigurations)
.addMockedMethod(updateAmsConfigurations)
+ .addMockedMethod(updateStormConfigurations)
.createMock();
@@ -379,6 +382,9 @@ public class UpgradeCatalog270Test {
upgradeCatalog270.updateAmsConfigs();
expectLastCall().once();
+ upgradeCatalog270.updateStormConfigs();
+ expectLastCall().once();
+
replay(upgradeCatalog270);
upgradeCatalog270.executeDMLUpdates();
@@ -1531,4 +1537,119 @@ public class UpgradeCatalog270Test {
assertTrue(Maps.difference(newProperties, updatedProperties).areEqual());
}
+ @Test
+ public void testStormConfigs() throws Exception {
+
+ Map<String, String> stormProperties = new HashMap<String, String>() {
+ {
+ put("_storm.thrift.nonsecure.transport", "org.apache.storm.security.auth.SimpleTransportPlugin");
+ put("_storm.thrift.secure.transport", "org.apache.storm.security.auth.kerberos.KerberosSaslTransportPlugin");
+ put("storm.thrift.transport", "{{storm_thrift_transport}}");
+ put("storm.zookeeper.port", "2181");
+ }
+ };
+ Map<String, String> newStormProperties = new HashMap<String, String>() {
+ {
+ put("storm.thrift.transport", "org.apache.storm.security.auth.SimpleTransportPlugin");
+ put("storm.zookeeper.port", "2181");
+ }
+ };
+
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
+ final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
+ Config mockStormSite = easyMockSupport.createNiceMock(Config.class);
+
+ expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", cluster);
+ }}).once();
+ expect(cluster.getDesiredConfigByType("storm-site")).andReturn(mockStormSite).atLeastOnce();
+ expect(cluster.getSecurityType()).andReturn(SecurityType.NONE).anyTimes();
+ expect(mockStormSite.getProperties()).andReturn(stormProperties).anyTimes();
+
+ Injector injector = easyMockSupport.createNiceMock(Injector.class);
+
+ replay(injector, clusters, mockStormSite, cluster);
+
+ AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
+ .addMockedMethod("getClusters", new Class[] { })
+ .addMockedMethod("createConfig")
+ .withConstructor(createNiceMock(ActionManager.class), clusters, injector)
+ .createNiceMock();
+
+ Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
+ Capture<Map> propertiesCapture = EasyMock.newCapture();
+
+ expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
+ expect(controller.getClusters()).andReturn(clusters).anyTimes();
+ expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(propertiesCapture), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+ replay(controller, injector2);
+ new UpgradeCatalog270(injector2).updateStormConfigs();
+ easyMockSupport.verifyAll();
+
+ Map<String, String> updatedProperties = propertiesCapture.getValue();
+ assertTrue(Maps.difference(newStormProperties, updatedProperties).areEqual());
+
+ }
+
+ @Test
+ public void testStormConfigsWithKerberos() throws Exception {
+
+ Map<String, String> stormProperties = new HashMap<String, String>() {
+ {
+ put("_storm.thrift.nonsecure.transport", "org.apache.storm.security.auth.SimpleTransportPlugin");
+ put("_storm.thrift.secure.transport", "org.apache.storm.security.auth.kerberos.KerberosSaslTransportPlugin");
+ put("storm.thrift.transport", "{{storm_thrift_transport}}");
+ put("storm.zookeeper.port", "2181");
+ }
+ };
+ Map<String, String> newStormProperties = new HashMap<String, String>() {
+ {
+ put("storm.thrift.transport", "org.apache.storm.security.auth.kerberos.KerberosSaslTransportPlugin");
+ put("storm.zookeeper.port", "2181");
+ }
+ };
+
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
+ final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
+ Config mockStormSite = easyMockSupport.createNiceMock(Config.class);
+
+ expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", cluster);
+ }}).once();
+ expect(cluster.getDesiredConfigByType("storm-site")).andReturn(mockStormSite).atLeastOnce();
+ expect(cluster.getSecurityType()).andReturn(SecurityType.KERBEROS).anyTimes();
+ expect(mockStormSite.getProperties()).andReturn(stormProperties).anyTimes();
+
+ Injector injector = easyMockSupport.createNiceMock(Injector.class);
+
+ replay(injector, clusters, mockStormSite, cluster);
+
+ AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
+ .addMockedMethod("getClusters", new Class[] { })
+ .addMockedMethod("createConfig")
+ .withConstructor(createNiceMock(ActionManager.class), clusters, injector)
+ .createNiceMock();
+
+ Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
+ Capture<Map> propertiesCapture = EasyMock.newCapture();
+
+ expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
+ expect(controller.getClusters()).andReturn(clusters).anyTimes();
+ expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(propertiesCapture), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+ replay(controller, injector2);
+ new UpgradeCatalog270(injector2).updateStormConfigs();
+ easyMockSupport.verifyAll();
+
+ Map<String, String> updatedProperties = propertiesCapture.getValue();
+ assertTrue(Maps.difference(newStormProperties, updatedProperties).areEqual());
+
+ }
}
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json b/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json
index 0ad32fa..127c04b 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json
@@ -240,8 +240,7 @@
"worker.heartbeat.frequency.secs": "1",
"zmq.hwm": "0",
"storm.zookeeper.connection.timeout": "15000",
- "storm.thrift.transport": "{{storm_thrift_transport}}",
- "_storm.thrift.secure.transport": "SECURED_TRANSPORT_CLASS",
+ "storm.thrift.transport": "NON_SECURED_TRANSPORT_CLASS",
"storm.messaging.netty.server_worker_threads": "1",
"supervisor.worker.start.timeout.secs": "120",
"zmq.threads": "1",
@@ -258,7 +257,6 @@
"storm.cluster.mode": "distributed",
"dev.zookeeper.path": "/tmp/dev-storm-zookeeper",
"drpc.invocations.port": "3773",
- "_storm.thrift.nonsecure.transport": "NON_SECURED_TRANSPORT_CLASS",
"storm.zookeeper.root": "/storm",
"logviewer.childopts": "-Xmx128m",
"transactional.zookeeper.port": "null",
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json b/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json
index 5650b3d..1d2c807 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json
@@ -218,8 +218,7 @@
"worker.heartbeat.frequency.secs": "1",
"zmq.hwm": "0",
"storm.zookeeper.connection.timeout": "15000",
- "storm.thrift.transport": "{{storm_thrift_transport}}",
- "_storm.thrift.secure.transport": "SECURED_TRANSPORT_CLASS",
+ "storm.thrift.transport": "SECURED_TRANSPORT_CLASS",
"storm.messaging.netty.server_worker_threads": "1",
"supervisor.worker.start.timeout.secs": "120",
"zmq.threads": "1",
@@ -235,8 +234,7 @@
"nimbus.monitor.freq.secs": "10",
"storm.cluster.mode": "distributed",
"dev.zookeeper.path": "/tmp/dev-storm-zookeeper",
- "drpc.invocations.port": "3773",
- "_storm.thrift.nonsecure.transport": "NON_SECURED_TRANSPORT_CLASS",
+ "drpc.invocations.port": "3773",
"storm.zookeeper.root": "/storm",
"logviewer.childopts": "-Xmx128m",
"transactional.zookeeper.port": "null",
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/secured.json b/ambari-server/src/test/python/stacks/2.1/configs/secured.json
index faf9d08..71e76c2 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/secured.json
@@ -835,7 +835,9 @@
"storm_ui_keytab" : "/etc/security/keytabs/spnego.service.keytab",
"storm_ui_principal_name" : "HTTP/_HOST",
"nimbus_seeds_supported" : "false",
- "storm_logs_supported": "false"
+ "storm_logs_supported": "false",
+ "nimbus_principal_name": "nimbus/_HOST",
+ "nimbus_keytab": "/etc/security/keytabs/nimbus.service.keytab"
},
"falcon-env": {
"falcon_port": "15000",