You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2015/10/13 13:50:27 UTC
ambari git commit: AMBARI-13388. Update hbase-env config without
removing custom changes.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 cbd3f97af -> ba40d1024
AMBARI-13388. Update hbase-env config without removing custom changes.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ba40d102
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ba40d102
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ba40d102
Branch: refs/heads/branch-2.1
Commit: ba40d102454573147d0860ac4065534c1dfe274c
Parents: cbd3f97
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Tue Oct 13 14:49:13 2015 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Tue Oct 13 14:49:13 2015 +0300
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog213.java | 33 +++++++++--
.../services/HBASE/configuration/hbase-env.xml | 7 ++-
.../services/HBASE/configuration/hbase-env.xml | 9 ++-
.../server/upgrade/UpgradeCatalog213Test.java | 59 ++++++++++++++++++--
4 files changed, 94 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba40d102/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
index 1455f05..b60404e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
@@ -52,6 +52,7 @@ import org.apache.ambari.server.state.alert.SourceType;
import org.apache.ambari.server.state.stack.upgrade.Direction;
import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
+import org.apache.ambari.server.utils.VersionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,6 +62,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.text.MessageFormat;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
@@ -73,6 +75,8 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
private static final String STORM_SITE = "storm-site";
private static final String AMS_ENV = "ams-env";
private static final String AMS_HBASE_ENV = "ams-hbase-env";
+ private static final String HBASE_ENV_CONFIG = "hbase-env";
+ private static final String CONTENT_PROPERTY = "content";
public static final String UPGRADE_PACKAGE_COL = "upgrade_package";
public static final String UPGRADE_TYPE_COL = "upgrade_type";
@@ -147,8 +151,10 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
executeStackUpgradeDDLUpdates();
bootstrapRepoVersionForHDP21();
- addMissingConfigs();
+ addNewConfigurationsFromXml();
+ updateStormConfigs();
updateAMSConfigs();
+ updateHbaseEnvConfig();
updateAlertDefinitions();
}
@@ -546,10 +552,6 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
return rootJson.toString();
}
- protected void addMissingConfigs() throws AmbariException {
- updateStormConfigs();
- }
-
protected void updateStormConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Clusters clusters = ambariManagementController.getClusters();
@@ -575,6 +577,27 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
}
}
+ protected void updateHbaseEnvConfig() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+
+ for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) {
+ StackId stackId = cluster.getCurrentStackVersion();
+ if (stackId != null && stackId.getStackName().equals("HDP") &&
+ VersionUtils.compareVersions(stackId.getStackVersion(), "2.2") >= 0) {
+ Config hbaseEnvConfig = cluster.getDesiredConfigByType(HBASE_ENV_CONFIG);
+ if (hbaseEnvConfig != null) {
+ String content = hbaseEnvConfig.getProperties().get(CONTENT_PROPERTY);
+ if (content != null && content.indexOf("MaxDirectMemorySize={{hbase_max_direct_memory_size}}m") < 0) {
+ String newPartOfContent = "\n\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}\"\n\n";
+ content += newPartOfContent;
+ Map<String, String> updates = Collections.singletonMap(CONTENT_PROPERTY, content);
+ updateConfigurationPropertiesForCluster(cluster, HBASE_ENV_CONFIG, updates, true, false);
+ }
+ }
+ }
+ }
+ }
+
protected void updateAMSConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Clusters clusters = ambariManagementController.getClusters();
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba40d102/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-env.xml
index 5f6b1bd..a68295e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-env.xml
@@ -162,12 +162,15 @@ export HBASE_MANAGES_ZK=false
{% if security_enabled %}
export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}}"
export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}"
-export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}"
{% else %}
export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log"
export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}"
-export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
{% endif %}
+
+# HBase off-heap MaxDirectMemorySize
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}"
</value>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba40d102/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/configuration/hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/configuration/hbase-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/configuration/hbase-env.xml
index 80348dd..1ecad34 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/configuration/hbase-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/configuration/hbase-env.xml
@@ -86,18 +86,21 @@ export HBASE_MANAGES_ZK=false
{% if java_version < 8 %}
JDK_DEPENDED_OPTS="-XX:PermSize=128m -XX:MaxPermSize=128m"
-{% endif %}
+{% endif %}
{% if security_enabled %}
export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}}"
export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}} $JDK_DEPENDED_OPTS"
-export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %} -Djava.security.auth.login.config={{regionserver_jaas_config_file}} $JDK_DEPENDED_OPTS"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}} $JDK_DEPENDED_OPTS"
export PHOENIX_QUERYSERVER_OPTS="$PHOENIX_QUERYSERVER_OPTS -Djava.security.auth.login.config={{queryserver_jaas_config_file}}"
{% else %}
export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log"
export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} $JDK_DEPENDED_OPTS"
-export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %} $JDK_DEPENDED_OPTS"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} $JDK_DEPENDED_OPTS"
{% endif %}
+
+# HBase off-heap MaxDirectMemorySize
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}"
</value>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba40d102/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
index 1be1446..d04e4c9 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
@@ -167,25 +167,34 @@ public class UpgradeCatalog213Test {
f.set(upgradeCatalog, configuration);
*/
- Method addMissingConfigs = UpgradeCatalog213.class.getDeclaredMethod("addMissingConfigs");
- Method updateAMSConfigs = UpgradeCatalog213.class.getDeclaredMethod("updateAMSConfigs");
Method updateAlertDefinitions = UpgradeCatalog213.class.getDeclaredMethod("updateAlertDefinitions");
Method executeStackUpgradeDDLUpdates = UpgradeCatalog213.class.getDeclaredMethod("executeStackUpgradeDDLUpdates");
Method bootstrapRepoVersionForHDP21 = UpgradeCatalog213.class.getDeclaredMethod("bootstrapRepoVersionForHDP21");
+ Method updateStormConfigs = UpgradeCatalog213.class.getDeclaredMethod("updateStormConfigs");
+ Method updateAMSConfigs = UpgradeCatalog213.class.getDeclaredMethod("updateAMSConfigs");
+ Method updateHbaseEnvConfig = UpgradeCatalog213.class.getDeclaredMethod("updateHbaseEnvConfig");
+ Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml");
+
UpgradeCatalog213 upgradeCatalog213 = createMockBuilder(UpgradeCatalog213.class)
- .addMockedMethod(addMissingConfigs)
+ .addMockedMethod(addNewConfigurationsFromXml)
.addMockedMethod(updateAMSConfigs)
.addMockedMethod(updateAlertDefinitions)
.addMockedMethod(executeStackUpgradeDDLUpdates)
.addMockedMethod(bootstrapRepoVersionForHDP21)
+ .addMockedMethod(updateStormConfigs)
+ .addMockedMethod(updateHbaseEnvConfig)
.createMock();
+ upgradeCatalog213.addNewConfigurationsFromXml();
+ expectLastCall().once();
upgradeCatalog213.executeStackUpgradeDDLUpdates();
expectLastCall().once();
upgradeCatalog213.bootstrapRepoVersionForHDP21();
expectLastCall().once();
- upgradeCatalog213.addMissingConfigs();
+ upgradeCatalog213.updateStormConfigs();
+ expectLastCall().once();
+ upgradeCatalog213.updateHbaseEnvConfig();
expectLastCall().once();
upgradeCatalog213.updateAMSConfigs();
expectLastCall().once();
@@ -204,6 +213,48 @@ public class UpgradeCatalog213Test {
//upgradeSectionDDL.verify(dbAccessor);
}
+ @Test
+ public void testUpdateHbaseEnvConfig() throws AmbariException {
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+ final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class);
+ final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class);
+ final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class);
+ final Map<String, String> propertiesHbaseEnv = new HashMap<String, String>() {
+ {
+ put("content", "test");
+ }
+ };
+
+ final Config mockHbaseEnv = easyMockSupport.createNiceMock(Config.class);
+ expect(mockHbaseEnv.getProperties()).andReturn(propertiesHbaseEnv).once();
+
+ final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(AmbariManagementController.class).toInstance(mockAmbariManagementController);
+ bind(Clusters.class).toInstance(mockClusters);
+ bind(EntityManager.class).toInstance(entityManager);
+
+ bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+ bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+ }
+ });
+
+ expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once();
+ expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", mockClusterExpected);
+ }}).atLeastOnce();
+ expect(mockClusterExpected.getCurrentStackVersion()).andReturn(new StackId("HDP","2.2"));
+
+ expect(mockClusterExpected.getDesiredConfigByType("hbase-env")).andReturn(mockHbaseEnv).atLeastOnce();
+ expect(mockHbaseEnv.getProperties()).andReturn(propertiesHbaseEnv).atLeastOnce();
+
+ easyMockSupport.replayAll();
+ mockInjector.getInstance(UpgradeCatalog213.class).updateHbaseEnvConfig();
+ easyMockSupport.verifyAll();
+
+ }
+
/**
* Verify that when bootstrapping HDP 2.1, records get inserted into the
* repo_version, cluster_version, and host_version tables.