You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by lp...@apache.org on 2017/09/14 14:38:46 UTC
[10/38] ambari git commit: AMBARI-21893 : NameNode Heap Usage (Daily)
metric alert status flips to UNKNOWN intermittently when AMS HTTPS is
enabled. (avijayan)
AMBARI-21893 : NameNode Heap Usage (Daily) metric alert status flips to UNKNOWN intermittently when AMS HTTPS is enabled. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7fa7a6c1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7fa7a6c1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7fa7a6c1
Branch: refs/heads/feature-branch-AMBARI-21307
Commit: 7fa7a6c17607341e8a6888f3dd8b938dce7425ff
Parents: bb4645f
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Tue Sep 12 17:01:51 2017 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Tue Sep 12 17:01:51 2017 -0700
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog260.java | 30 +++++++++
.../0.1.0/configuration/ams-ssl-client.xml | 9 ---
.../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 10 +--
.../0.1.0/package/scripts/params.py | 10 ++-
.../server/upgrade/UpgradeCatalog260Test.java | 70 +++++++++++++++++++-
5 files changed, 110 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7fa7a6c1/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java
index d05f39a..2de85fc 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java
@@ -21,6 +21,7 @@ import static org.apache.ambari.server.view.ViewContextImpl.CORE_SITE;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -29,6 +30,7 @@ import javax.persistence.EntityManager;
import javax.persistence.Query;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.orm.DBAccessor;
import org.apache.ambari.server.orm.dao.ArtifactDAO;
import org.apache.ambari.server.orm.entities.ArtifactEntity;
@@ -122,6 +124,9 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog {
public static final String HOST_COMPONENT_DESIRED_STATE = "hostcomponentdesiredstate";
public static final String HOST_COMPONENT_STATE = "hostcomponentstate";
+ public static final String AMS_SSL_CLIENT = "ams-ssl-client";
+ public static final String METRIC_TRUSTSTORE_ALIAS = "ssl.client.truststore.alias";
+
/**
* Logger.
*/
@@ -395,6 +400,7 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog {
removeSupersetFromDruid();
ensureZeppelinProxyUserConfigs();
updateKerberosDescriptorArtifacts();
+ updateAmsConfigs();
}
public int getCurrentVersionID() throws AmbariException, SQLException {
@@ -535,4 +541,28 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog {
}
}
}
+
+ protected void updateAmsConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+
+
+ Config amsSslClient = cluster.getDesiredConfigByType(AMS_SSL_CLIENT);
+ if (amsSslClient != null) {
+ Map<String, String> amsSslClientProperties = amsSslClient.getProperties();
+
+ if (amsSslClientProperties.containsKey(METRIC_TRUSTSTORE_ALIAS)) {
+ LOG.info("Removing " + METRIC_TRUSTSTORE_ALIAS + " from " + AMS_SSL_CLIENT);
+ removeConfigurationPropertiesFromCluster(cluster, AMS_SSL_CLIENT, Collections.singleton(METRIC_TRUSTSTORE_ALIAS));
+ }
+
+ }
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7fa7a6c1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml
index cac39de..d75bba2 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml
@@ -39,13 +39,4 @@
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
- <property>
- <name>ssl.client.truststore.alias</name>
- <value></value>
- <description>Alias used to create certificate for AMS. (Default is hostname)</description>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7fa7a6c1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
index 9a31ade..51e0756 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
@@ -515,10 +515,12 @@ def export_ca_certs(dir_path):
truststore_p12 = os.path.join(tmpdir,'truststore.p12')
if (params.metric_truststore_type.lower() == 'jks'):
- # Convert truststore from JKS to PKCS12
- cmd = format("{sudo} {java64_home}/bin/keytool -importkeystore -srckeystore {metric_truststore_path} -destkeystore {truststore_p12} -srcalias {metric_truststore_alias} -deststoretype PKCS12 -srcstorepass {metric_truststore_password} -deststorepass {metric_truststore_password}")
- Execute(cmd,
- )
+ if not params.metric_truststore_alias:
+ for alias in params.metric_truststore_alias_list:
+ # Convert truststore from JKS to PKCS12
+ cmd = format("{sudo} {java64_home}/bin/keytool -importkeystore -srckeystore {metric_truststore_path} -destkeystore {truststore_p12} -srcalias " + alias + " -deststoretype PKCS12 -srcstorepass {metric_truststore_password} -deststorepass {metric_truststore_password}")
+ Execute(cmd,
+ )
truststore = truststore_p12
# Export all CA certificates from the truststore to the conf directory
http://git-wip-us.apache.org/repos/asf/ambari/blob/7fa7a6c1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 756da26..0242b73 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -100,11 +100,15 @@ else:
metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
-metric_truststore_alias = default("/configurations/ams-ssl-client/ssl.client.truststore.alias", None)
-if not metric_truststore_alias:
- metric_truststore_alias = metric_collector_host
metric_truststore_ca_certs='ca.pem'
+metric_truststore_alias_list = []
+for host in ams_collector_hosts.split(","):
+ metric_truststore_alias = default("/configurations/ams-ssl-client/{host}.ssl.client.truststore.alias", None)
+ if not metric_truststore_alias:
+ metric_truststore_alias = host
+ metric_truststore_alias_list.append(metric_truststore_alias)
+
agent_cache_dir = config['hostLevelParams']['agentCacheDir']
service_package_folder = config['commandParams']['service_package_folder']
stack_name = default("/hostLevelParams/stack_name", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/7fa7a6c1/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java
index 33c29bc..b70f37b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java
@@ -23,6 +23,7 @@ import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.capture;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.createMockBuilder;
+import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
@@ -30,6 +31,7 @@ import static org.easymock.EasyMock.newCapture;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.reset;
import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertTrue;
import java.io.File;
import java.net.URL;
@@ -40,16 +42,19 @@ import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.persistence.EntityManager;
+import com.google.common.collect.Maps;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.ActionManager;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.AmbariManagementControllerImpl;
import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.controller.MaintenanceStateHelper;
import org.apache.ambari.server.orm.DBAccessor;
@@ -67,7 +72,9 @@ import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
import org.apache.ambari.server.state.stack.OsFamily;
import org.easymock.Capture;
+import org.easymock.EasyMock;
import org.easymock.EasyMockRunner;
+import org.easymock.EasyMockSupport;
import org.easymock.Mock;
import org.easymock.MockType;
import org.junit.After;
@@ -480,8 +487,8 @@ public class UpgradeCatalog260Test {
}
public void verifyGetCurrentVersionID(Capture<String[]> scdcaptureKey, Capture<String[]> scdcaptureValue) {
- Assert.assertTrue(Arrays.equals(scdcaptureKey.getValue(), new String[]{UpgradeCatalog260.STATE_COLUMN}));
- Assert.assertTrue(Arrays.equals(scdcaptureValue.getValue(), new String[]{UpgradeCatalog260.CURRENT}));
+ assertTrue(Arrays.equals(scdcaptureKey.getValue(), new String[]{UpgradeCatalog260.STATE_COLUMN}));
+ assertTrue(Arrays.equals(scdcaptureValue.getValue(), new String[]{UpgradeCatalog260.CURRENT}));
}
public void expectUpdateServiceComponentDesiredStateTable(Capture<DBColumnInfo> scdstadd1, Capture<DBColumnInfo> scdstalter1, Capture<DBColumnInfo> scdstadd2, Capture<DBColumnInfo> scdstalter2) throws SQLException {
@@ -625,7 +632,7 @@ public class UpgradeCatalog260Test {
verify(clusters, cluster, zeppelinEnvConf, coreSiteConf, coreSiteConfNew, controller);
- Assert.assertTrue(captureCoreSiteConfProperties.hasCaptured());
+ assertTrue(captureCoreSiteConfProperties.hasCaptured());
Assert.assertEquals("existing_value", captureCoreSiteConfProperties.getValue().get("hadoop.proxyuser.zeppelin_user.hosts"));
Assert.assertEquals("*", captureCoreSiteConfProperties.getValue().get("hadoop.proxyuser.zeppelin_user.groups"));
}
@@ -673,4 +680,61 @@ public class UpgradeCatalog260Test {
Assert.assertNull(kerberosDescriptorUpdated.getService("RANGER_KMS").getComponent("RANGER_KMS_SERVER").getIdentity("/smokeuser"));
}
+
+ @Test
+ public void testUpdateAmsConfigs() throws Exception{
+
+ Map<String, String> oldProperties = new HashMap<String, String>() {
+ {
+ put("ssl.client.truststore.location", "/some/location");
+ put("ssl.client.truststore.alias", "test_alias");
+ }
+ };
+ Map<String, String> newProperties = new HashMap<String, String>() {
+ {
+ put("ssl.client.truststore.location", "/some/location");
+ }
+ };
+
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
+ final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
+ Config mockAmsSslClient = easyMockSupport.createNiceMock(Config.class);
+
+ expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", cluster);
+ }}).once();
+ expect(cluster.getDesiredConfigByType("ams-ssl-client")).andReturn(mockAmsSslClient).atLeastOnce();
+ expect(mockAmsSslClient.getProperties()).andReturn(oldProperties).anyTimes();
+
+ Injector injector = easyMockSupport.createNiceMock(Injector.class);
+ expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes();
+ expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes();
+
+ replay(injector, clusters, mockAmsSslClient, cluster);
+
+ AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
+ .addMockedMethod("createConfiguration")
+ .addMockedMethod("getClusters", new Class[] { })
+ .addMockedMethod("createConfig")
+ .withConstructor(createNiceMock(ActionManager.class), clusters, injector)
+ .createNiceMock();
+
+ Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
+ Capture<Map> propertiesCapture = EasyMock.newCapture();
+
+ expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
+ expect(controller.getClusters()).andReturn(clusters).anyTimes();
+ expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(propertiesCapture), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+ replay(controller, injector2);
+ new UpgradeCatalog260(injector2).updateAmsConfigs();
+ easyMockSupport.verifyAll();
+
+ Map<String, String> updatedProperties = propertiesCapture.getValue();
+ assertTrue(Maps.difference(newProperties, updatedProperties).areEqual());
+ }
+
}