You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2016/09/14 22:03:49 UTC
[3/5] ambari git commit: AMBARI-18132 : Remove FIFO and Normalizer in
ams-env config. (avijayan)
AMBARI-18132 : Remove FIFO and Normalizer in ams-env config. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0e343585
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0e343585
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0e343585
Branch: refs/heads/trunk
Commit: 0e3435856a2aee7951f765e90e55e027b6455c1a
Parents: 9455b52
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Tue Sep 13 20:32:07 2016 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Wed Sep 14 15:03:34 2016 -0700
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog250.java | 62 ++++++++++-
.../0.1.0/configuration/ams-env.xml | 6 -
.../0.1.0/package/scripts/params.py | 2 -
.../server/upgrade/UpgradeCatalog250Test.java | 110 +++++++++++++++++++
4 files changed, 171 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e343585/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index 43f489b..35c773a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -18,9 +18,18 @@
package org.apache.ambari.server.upgrade;
import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.orm.dao.DaoUtils;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,7 +42,7 @@ import com.google.inject.Injector;
public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
protected static final String HOST_VERSION_TABLE = "host_version";
-
+ private static final String AMS_ENV = "ams-env";
/**
* Logger.
*/
@@ -97,6 +106,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
*/
@Override
protected void executeDMLUpdates() throws AmbariException, SQLException {
+ updateAMSConfigs();
}
protected void updateHostVersionTable() throws SQLException {
@@ -106,6 +116,56 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
dbAccessor.addUniqueConstraint(HOST_VERSION_TABLE, "UQ_host_repo", "repo_version_id", "host_id");
}
+ protected void updateAMSConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+
+ Config amsEnv = cluster.getDesiredConfigByType(AMS_ENV);
+ if (amsEnv != null) {
+ Map<String, String> amsEnvProperties = amsEnv.getProperties();
+ String content = amsEnvProperties.get("content");
+ Map<String, String> newProperties = new HashMap<>();
+ newProperties.put("content", updateAmsEnvContent(content));
+ updateConfigurationPropertiesForCluster(cluster, AMS_ENV, newProperties, true, true);
+ }
+
+ }
+ }
+ }
+ }
+
+
+ protected String updateAmsEnvContent(String content) {
+ if (content == null) {
+ return null;
+ }
+
+ List<String> toReplaceList = new ArrayList<>();
+ toReplaceList.add("\n# HBase normalizer enabled\n");
+ toReplaceList.add("\n# HBase compaction policy enabled\n");
+ toReplaceList.add("export AMS_HBASE_NORMALIZER_ENABLED={{ams_hbase_normalizer_enabled}}\n");
+ toReplaceList.add("export AMS_HBASE_FIFO_COMPACTION_ENABLED={{ams_hbase_fifo_compaction_enabled}}\n");
+
+ //Because of AMBARI-15331 : AMS HBase FIFO compaction policy and Normalizer settings are not handled correctly
+ toReplaceList.add("export HBASE_NORMALIZATION_ENABLED={{ams_hbase_normalizer_enabled}}\n");
+ toReplaceList.add("export HBASE_FIFO_COMPACTION_POLICY_ENABLED={{ams_hbase_fifo_compaction_policy_enabled}}\n");
+
+
+ for (String toReplace : toReplaceList) {
+ if (content.contains(toReplace)) {
+ content = content.replace(toReplace, StringUtils.EMPTY);
+ }
+ }
+
+ return content;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e343585/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
index a52cb82..48232af 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
@@ -130,12 +130,6 @@ export AMS_HBASE_PID_DIR={{hbase_pid_dir}}
# AMS Collector heapsize
export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}
-# HBase normalizer enabled
-export AMS_HBASE_NORMALIZER_ENABLED={{ams_hbase_normalizer_enabled}}
-
-# HBase compaction policy enabled
-export AMS_HBASE_FIFO_COMPACTION_ENABLED={{ams_hbase_fifo_compaction_enabled}}
-
# HBase Tables Initialization check enabled
export AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e343585/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 61b89f8..22024bb 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -159,8 +159,6 @@ ams_grafana_cert_key = default("/configurations/ams-grafana-ini/cert_key", '/etc
ams_hbase_home_dir = "/usr/lib/ams-hbase/"
-ams_hbase_normalizer_enabled = default("/configurations/ams-hbase-site/hbase.normalizer.enabled", None)
-ams_hbase_fifo_compaction_enabled = default("/configurations/ams-site/timeline.metrics.hbase.fifo.compaction.enabled", None)
ams_hbase_init_check_enabled = default("/configurations/ams-site/timeline.metrics.hbase.init.check.enabled", True)
#hadoop params
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e343585/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
index 6dadb22..c4e0a7c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
@@ -28,8 +28,21 @@ import static org.easymock.EasyMock.verify;
import javax.persistence.EntityManager;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import org.apache.ambari.server.actionmanager.ActionManager;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.AmbariManagementControllerImpl;
+import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.controller.MaintenanceStateHelper;
import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.stack.OsFamily;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.easymock.EasyMockSupport;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -40,6 +53,15 @@ import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.Provider;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createMockBuilder;
+import static org.junit.Assert.assertTrue;
/**
* {@link UpgradeCatalog250} unit tests.
*/
@@ -86,4 +108,92 @@ public class UpgradeCatalog250Test {
verify(dbAccessor);
}
+ @Test
+ public void testExecuteDMLUpdates() throws Exception {
+ Method updateAmsConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAMSConfigs");
+
+ UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class)
+ .addMockedMethod(updateAmsConfigs)
+ .createMock();
+
+ upgradeCatalog250.updateAMSConfigs();
+ expectLastCall().once();
+
+ replay(upgradeCatalog250);
+
+ upgradeCatalog250.executeDMLUpdates();
+
+ verify(upgradeCatalog250);
+ }
+
+ @Test
+ public void testAmsEnvUpdateConfigs() throws Exception{
+
+ Map<String, String> oldPropertiesAmsEnv = new HashMap<String, String>() {
+ {
+ put("content", "\n" +
+ "# AMS Collector heapsize\n" +
+ "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}\n" +
+ "\n" +
+ "# HBase normalizer enabled\n" +
+ "export AMS_HBASE_NORMALIZER_ENABLED={{ams_hbase_normalizer_enabled}}\n" +
+ "\n" +
+ "# HBase compaction policy enabled\n" +
+ "export HBASE_FIFO_COMPACTION_POLICY_ENABLED={{ams_hbase_fifo_compaction_policy_enabled}}\n" +
+ "\n" +
+ "# HBase Tables Initialization check enabled\n" +
+ "export AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}}\n");
+ }
+ };
+ Map<String, String> newPropertiesAmsEnv = new HashMap<String, String>() {
+ {
+ put("content", "\n" +
+ "# AMS Collector heapsize\n" +
+ "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}\n" +
+ "\n" +
+ "# HBase Tables Initialization check enabled\n" +
+ "export AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}}\n");
+ }
+ };
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
+ final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
+ Config mockAmsEnv = easyMockSupport.createNiceMock(Config.class);
+
+ expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", cluster);
+ }}).once();
+ expect(cluster.getDesiredConfigByType("ams-env")).andReturn(mockAmsEnv).atLeastOnce();
+ expect(mockAmsEnv.getProperties()).andReturn(oldPropertiesAmsEnv).anyTimes();
+
+ Injector injector = easyMockSupport.createNiceMock(Injector.class);
+ expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes();
+ expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes();
+ expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class)).anyTimes();
+
+ replay(injector, clusters, mockAmsEnv, cluster);
+
+ AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
+ .addMockedMethod("createConfiguration")
+ .addMockedMethod("getClusters", new Class[] { })
+ .addMockedMethod("createConfig")
+ .withConstructor(createNiceMock(ActionManager.class), clusters, injector)
+ .createNiceMock();
+
+ Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
+ Capture<Map> propertiesCapture = EasyMock.newCapture();
+
+ expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
+ expect(controller.getClusters()).andReturn(clusters).anyTimes();
+ expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(propertiesCapture), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+ replay(controller, injector2);
+ new UpgradeCatalog250(injector2).updateAMSConfigs();
+ easyMockSupport.verifyAll();
+
+ Map<String, String> updatedProperties = propertiesCapture.getValue();
+ assertTrue(Maps.difference(newPropertiesAmsEnv, updatedProperties).areEqual());
+ }
}