You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rz...@apache.org on 2016/08/26 20:48:12 UTC
[41/50] ambari git commit: AMBARI-18238. HBase Master doesn't start
after upgrading from HDP 2.3.6 using Ambari (dgrinenko via dlisnichenko)
AMBARI-18238. HBase Master doesn't start after upgrading from HDP 2.3.6 using Ambari (dgrinenko via dlisnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2979f45e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2979f45e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2979f45e
Branch: refs/heads/2.4.0-maint
Commit: 2979f45edf977703f35c7e3ad59d3c6d2bb945a7
Parents: e472ede
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Tue Aug 23 19:26:09 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Tue Aug 23 19:27:56 2016 +0300
----------------------------------------------------------------------
.../HBaseEnvMaxDirectMemorySizeAction.java | 87 +++++++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 4 +
.../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 4 +
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 9 +
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 9 +
.../HBaseEnvMaxDirectMemorySizeActionTest.java | 194 +++++++++++++++++++
6 files changed, 307 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2979f45e/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java
new file mode 100644
index 0000000..b238bca
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import com.google.inject.Inject;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Computes HBase Env content property.
+ * This class is only used when moving from HDP-2.3 to HDP-2.4 and HDP-2.3 to HDP-2.5
+ */
+public class HBaseEnvMaxDirectMemorySizeAction extends AbstractServerAction {
+ private static final String SOURCE_CONFIG_TYPE = "hbase-env";
+ private static final String CONTENT_NAME = "content";
+ private static final String APPEND_CONTENT_LINE = "export HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}\"";
+ private static final String CHECK_REGEX = "^.*\\s*(HBASE_MASTER_OPTS)\\s*=.*(XX:MaxDirectMemorySize).*$";
+ private static final Pattern REGEX = Pattern.compile(CHECK_REGEX, Pattern.MULTILINE);
+
+ @Inject
+ private Clusters clusters;
+
+ @Override
+ public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+
+ String clusterName = getExecutionCommand().getClusterName();
+ Cluster cluster = clusters.getCluster(clusterName);
+ Config config = cluster.getDesiredConfigByType(SOURCE_CONFIG_TYPE);
+
+ if (config == null) {
+ return createCommandReport(0, HostRoleStatus.FAILED,"{}",
+ String.format("Source type %s not found", SOURCE_CONFIG_TYPE), "");
+ }
+
+ Map<String, String> properties = config.getProperties();
+ String content = properties.get(CONTENT_NAME);
+
+
+ if (content == null) {
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ String.format("The %s/%s property was not found. The -XX:MaxDirectMemorySize will not be added", SOURCE_CONFIG_TYPE, CONTENT_NAME), "");
+ }
+
+ Matcher m = REGEX.matcher(content);
+
+ if (m.find()){
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ String.format("The %s/%s property already contains an entry for %s and will not be modified", SOURCE_CONFIG_TYPE, CONTENT_NAME, APPEND_CONTENT_LINE), "");
+ }
+
+ String appendedContent = content + "\n" + APPEND_CONTENT_LINE;
+ properties.put(CONTENT_NAME, appendedContent);
+
+ config.setProperties(properties);
+ config.persist(false);
+
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ String.format("The %s/%s property was appended with %s", SOURCE_CONFIG_TYPE, CONTENT_NAME, APPEND_CONTENT_LINE),"");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/2979f45e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index 450e350..eb6f333 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -273,6 +273,10 @@
<task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/>
</execute-stage>
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
+ <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/>
+ </execute-stage>
+
<!-- TEZ -->
<execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
<task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2979f45e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index b9067c7..f5f06c0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -299,6 +299,10 @@
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/>
</execute-stage>
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
+ <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/>
+ </execute-stage>
+
<!-- TEZ -->
<execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
<task xsi:type="configure" id="hdp_2_5_0_0_tez_client_adjust_tez_lib_uris_property"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2979f45e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 0e566c8..5ac0540 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -133,6 +133,15 @@
</service>
</group>
+ <group name="HBASE" title="Update HBase Configuration">
+ <skippable>true</skippable>
+
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
+ <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/>
+ </execute-stage>
+
+ </group>
+
<group name="KAFKA" title="Kafka">
<skippable>true</skippable>
<service name="KAFKA">
http://git-wip-us.apache.org/repos/asf/ambari/blob/2979f45e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 7237a6a..09bd2ac 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -150,6 +150,15 @@
</service>
</group>
+ <group name="HBASE" title="Update HBase Configuration">
+ <skippable>true</skippable>
+
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
+ <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/>
+ </execute-stage>
+
+ </group>
+
<!-- This group should exist for all RUs that cross a major stack version. -->
<group xsi:type="cluster" name="UPDATE_CLIENT_CONFIGS" title="Update Client Configs">
<direction>UPGRADE</direction>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2979f45e/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java
new file mode 100644
index 0000000..4c1d7a3
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import com.google.inject.Injector;
+import junit.framework.Assert;
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigImpl;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests HiveEnvClasspathAction logic
+ */
+public class HBaseEnvMaxDirectMemorySizeActionTest {
+ private Injector injector;
+ private Clusters clusters;
+ private Field m_clusterField;
+
+ @Before
+ public void setup() throws Exception {
+ injector = EasyMock.createMock(Injector.class);
+ clusters = EasyMock.createMock(Clusters.class);
+ Cluster cluster = EasyMock.createMock(Cluster.class);
+
+ Config hbaseEnv = new ConfigImpl("hbase-env") {
+ Map<String, String> mockProperties = new HashMap<String, String>() {{
+ put("content","# Set environment variables here.\n" +
+ "\n" +
+ "# The java implementation to use. Java 1.6 required.\n" +
+ "export JAVA_HOME={{java64_home}}\n" +
+ "\n" +
+ "# HBase Configuration directory\n" +
+ "export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n" +
+ "\n" +
+ "# Extra Java CLASSPATH elements. Optional.\n" +
+ "export HBASE_CLASSPATH=${HBASE_CLASSPATH}\n" +
+ "\n" +
+ "# The maximum amount of heap to use, in MB. Default is 1000.\n" +
+ "# export HBASE_HEAPSIZE=1000\n" +
+ "\n" +
+ "# Extra Java runtime options.\n" +
+ "# Below are what we set by default. May only work with SUN JVM.\n" +
+ "# For more on why as well as other possible settings,\n" +
+ "# see http://wiki.apache.org/hadoop/PerformanceTuning\n" +
+ "export SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n" +
+ "# Uncomment below to enable java garbage collection logging.\n" +
+ "# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log -Djava.io.tmpdir={{java_io_tmpdir}}\"\n" +
+ "\n" +
+ "# Uncomment and adjust to enable JMX exporting\n" +
+ "# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.\n" +
+ "# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n" +
+ "#\n" +
+ "# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n" +
+ "# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size\n" +
+ "# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n" +
+ "# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n" +
+ "\n" +
+ "# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\n" +
+ "export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n" +
+ "\n" +
+ "# Extra ssh options. Empty by default.\n" +
+ "# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n" +
+ "\n" +
+ "# Where log files are stored. $HBASE_HOME/logs by default.\n" +
+ "export HBASE_LOG_DIR={{log_dir}}\n" +
+ "\n" +
+ "# A string representing this instance of hbase. $USER by default.\n" +
+ "# export HBASE_IDENT_STRING=$USER\n" +
+ "\n" +
+ "# The scheduling priority for daemon processes. See 'man nice'.\n" +
+ "# export HBASE_NICENESS=10\n" +
+ "\n" +
+ "# The directory where pid files are stored. /tmp by default.\n" +
+ "export HBASE_PID_DIR={{pid_dir}}\n" +
+ "\n" +
+ "# Seconds to sleep between slave commands. Unset by default. This\n" +
+ "# can be useful in large clusters, where, e.g., slave rsyncs can\n" +
+ "# otherwise arrive faster than the master can service them.\n" +
+ "# export HBASE_SLAVE_SLEEP=0.1\n" +
+ "\n" +
+ "# Tell HBase whether it should manage it's own instance of Zookeeper or not.\n" +
+ "export HBASE_MANAGES_ZK=false\n" +
+ "\n" +
+ "{% if security_enabled %}\n" +
+ "export HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}} -Djava.io.tmpdir={{java_io_tmpdir}}\"\n" +
+ "export HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}\"\n" +
+ "export HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}\"\n" +
+ "{% else %}\n" +
+ "export HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.io.tmpdir={{java_io_tmpdir}}\"\n" +
+ "export HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}\"\n" +
+ "export HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n" +
+ "{% endif %}");
+ }};
+ @Override
+ public Map<String, String> getProperties() {
+ return mockProperties;
+ }
+
+ @Override
+ public void setProperties(Map<String, String> properties) {
+ mockProperties.putAll(properties);
+ }
+
+ @Override
+ public void persist(boolean newConfig) {
+ // no-op
+ }
+ };
+
+
+ expect(cluster.getDesiredConfigByType("hbase-env")).andReturn(hbaseEnv).atLeastOnce();
+
+ expect(clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
+ expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce();
+
+ replay(injector, clusters, cluster);
+
+ m_clusterField = HBaseEnvMaxDirectMemorySizeAction.class.getDeclaredField("clusters");
+ m_clusterField.setAccessible(true);
+ }
+
+
+ @Test
+ public void testAction() throws Exception {
+ Pattern regex = Pattern.compile("^.*\\s*(HBASE_MASTER_OPTS)\\s*=.*(XX:MaxDirectMemorySize).*$", Pattern.MULTILINE);
+ Map<String, String> commandParams = new HashMap<String, String>();
+ commandParams.put("clusterName", "c1");
+
+ ExecutionCommand executionCommand = new ExecutionCommand();
+ executionCommand.setCommandParams(commandParams);
+ executionCommand.setClusterName("c1");
+
+ HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+ expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+ expect(hrc.getStageId()).andReturn(2L).anyTimes();
+ expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+ replay(hrc);
+
+ HBaseEnvMaxDirectMemorySizeAction action = new HBaseEnvMaxDirectMemorySizeAction();
+
+ m_clusterField.set(action, clusters);
+
+ action.setExecutionCommand(executionCommand);
+ action.setHostRoleCommand(hrc);
+
+ CommandReport report = action.execute(null);
+ assertNotNull(report);
+
+ Cluster c = clusters.getCluster("c1");
+ Config config = c.getDesiredConfigByType("hbase-env");
+ Map<String, String> map = config.getProperties();
+
+ Assert.assertTrue(map.containsKey("content"));
+ String content = map.get("content");
+ Matcher m = regex.matcher(content);
+
+ assertEquals(true, m.find());
+ }
+
+}