You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/11/11 15:09:24 UTC
ambari git commit: AMBARI-13817 HDP-2.3 does not have the ability to
set heapsize for hive components (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 1ca59255e -> d8bf840ce
AMBARI-13817 HDP-2.3 does not have the ability to set heapsize for hive components (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d8bf840c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d8bf840c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d8bf840c
Branch: refs/heads/trunk
Commit: d8bf840ce13be2ff8d9444ee4f2751e5d5fc2db3
Parents: 1ca5925
Author: Dmytro Sen <ds...@apache.org>
Authored: Wed Nov 11 16:09:15 2015 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Wed Nov 11 16:09:15 2015 +0200
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog213.java | 66 ++++++++++++++++-
.../HIVE/0.12.0.2.0/configuration/hive-env.xml | 6 +-
.../services/HIVE/configuration/hive-env.xml | 8 +-
.../services/HIVE/configuration/hive-env.xml | 10 ++-
.../server/upgrade/UpgradeCatalog213Test.java | 78 ++++++++++++++++++++
5 files changed, 160 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bf840c/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
index f2b79a6..8df27c1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.regex.Matcher;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
@@ -92,6 +93,7 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
private static final String HBASE_ENV_CONFIG = "hbase-env";
private static final String FLUME_ENV_CONFIG = "flume-env";
private static final String HIVE_SITE_CONFIG = "hive-site";
+ private static final String HIVE_ENV_CONFIG = "hive-env";
private static final String RANGER_ENV_CONFIG = "ranger-env";
private static final String ZOOKEEPER_LOG4J_CONFIG = "zookeeper-log4j";
private static final String NIMBS_MONITOR_FREQ_SECS_PROPERTY = "nimbus.monitor.freq.secs";
@@ -225,7 +227,7 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
*/
protected void executeStageDDLUpdates() throws SQLException {
dbAccessor.addColumn(STAGE_TABLE,
- new DBAccessor.DBColumnInfo("supports_auto_skip_failure", Integer.class, 1, 0, false));
+ new DBAccessor.DBColumnInfo("supports_auto_skip_failure", Integer.class, 1, 0, false));
}
/**
@@ -793,6 +795,27 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
updateConfigurationPropertiesForCluster(cluster, HIVE_SITE_CONFIG, updates, true, false);
}
}
+ StackId stackId = cluster.getCurrentStackVersion();
+ boolean isStackNotLess23 = (stackId != null && stackId.getStackName().equals("HDP") &&
+ VersionUtils.compareVersions(stackId.getStackVersion(), "2.3") >= 0);
+
+ Config hiveEnvConfig = cluster.getDesiredConfigByType(HIVE_ENV_CONFIG);
+ if (hiveEnvConfig != null) {
+ Map<String, String> hiveEnvProps = new HashMap<String, String>();
+ String content = hiveEnvConfig.getProperties().get(CONTENT_PROPERTY);
+ // For HDP-2.3 we need to add hive heap size management to content,
+ // for others we need to update content
+ if(content != null) {
+ if(isStackNotLess23) {
+ content = updateHiveEnvContentHDP23(content);
+ } else {
+ content = updateHiveEnvContent(content);
+ }
+ hiveEnvProps.put(CONTENT_PROPERTY, content);
+ updateConfigurationPropertiesForCluster(cluster, HIVE_ENV_CONFIG, hiveEnvProps, true, true);
+ }
+ }
+
}
}
@@ -1033,6 +1056,47 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
return content;
}
+ protected String updateHiveEnvContent(String hiveEnvContent) {
+ if(hiveEnvContent == null) {
+ return null;
+ }
+ // There are two cases here
+ // We do not have "export HADOOP_CLIENT_OPTS" and we need to add it
+ // We have "export HADOOP_CLIENT_OPTS" with wrong order
+ String exportHadoopClientOpts = "(?s).*export\\s*HADOOP_CLIENT_OPTS.*";
+ if (hiveEnvContent.matches(exportHadoopClientOpts)) {
+ String oldHeapSizeRegex = "export\\s*HADOOP_CLIENT_OPTS=\"-Xmx\\$\\{HADOOP_HEAPSIZE\\}m\\s*\\$HADOOP_CLIENT_OPTS\"";
+ String newHeapSizeRegex = "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\"";
+ return hiveEnvContent.replaceAll(oldHeapSizeRegex, Matcher.quoteReplacement(newHeapSizeRegex));
+ } else {
+ String oldHeapSizeRegex = "export\\s*HADOOP_HEAPSIZE\\s*=\\s*\"\\{\\{hive_heapsize\\}\\}\"\\.*\\n\\s*fi\\s*\\n";
+ String newHeapSizeRegex = "export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client\n" +
+ "fi\n" +
+ "\n" +
+ "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\"";
+ return hiveEnvContent.replaceAll(oldHeapSizeRegex, Matcher.quoteReplacement(newHeapSizeRegex));
+ }
+ }
+
+ protected String updateHiveEnvContentHDP23(String hiveEnvContent) {
+ if(hiveEnvContent == null) {
+ return null;
+ }
+ String oldHeapSizeRegex = "# The heap size of the jvm stared by hive shell script can be controlled via:\\s*\\n";
+ String newHeapSizeRegex = "# The heap size of the jvm stared by hive shell script can be controlled via:\n" +
+ "\n" +
+ "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+ " export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore\n" +
+ "else\n" +
+ " export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client\n" +
+ "fi\n" +
+ "\n" +
+ "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\"\n" +
+ "\n";
+ return hiveEnvContent.replaceFirst(oldHeapSizeRegex, Matcher.quoteReplacement(newHeapSizeRegex));
+ }
+
+
private String memoryToIntMb(String memorySize) {
if (memorySize == null) {
return "0";
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bf840c/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
index 447993d..86165be 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
@@ -182,12 +182,12 @@
# The heap size of the jvm stared by hive shell script can be controlled via:
if [ "$SERVICE" = "metastore" ]; then
- export HADOOP_HEAPSIZE="{{hive_metastore_heapsize}}"
+ export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
else
- export HADOOP_HEAPSIZE="{{hive_heapsize}}"
+ export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
fi
-export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS"
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m"
# Larger heap size may be required when running queries over large number of files or partitions.
# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bf840c/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
index c331802..f84bc01 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
@@ -50,13 +50,15 @@
fi
# The heap size of the jvm stared by hive shell script can be controlled via:
-export SERVICE=$SERVICE
+
if [ "$SERVICE" = "metastore" ]; then
- export HADOOP_HEAPSIZE="{{hive_metastore_heapsize}}"
+ export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
else
- export HADOOP_HEAPSIZE="{{hive_heapsize}}"
+ export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
fi
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m"
+
# Larger heap size may be required when running queries over large number of files or partitions.
# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be
# appropriate for hive server (hwi etc).
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bf840c/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
index 30141c6..77de518 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
@@ -37,6 +37,14 @@
# The heap size of the jvm stared by hive shell script can be controlled via:
+if [ "$SERVICE" = "metastore" ]; then
+ export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
+else
+ export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
+fi
+
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m"
+
# Larger heap size may be required when running queries over large number of files or partitions.
# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be
# appropriate for hive server (hwi etc).
@@ -68,4 +76,4 @@ export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}"
</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bf840c/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
index b0a6c3a..f30bf82 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
@@ -1139,7 +1139,11 @@ public class UpgradeCatalog213Test {
final Map<String, String> propertiesHiveSiteExpected = new HashMap<String, String>() {{
put("hive.server2.logging.operation.log.location", "/tmp/hive/operation_logs");
}};
+ final Map<String, String> propertiesHiveEnv = new HashMap<String, String>() {{
+ put("content", "test content");
+ }};
final Config hiveSiteConf = easyMockSupport.createNiceMock(Config.class);
+ final Config hiveEnvConf = easyMockSupport.createNiceMock(Config.class);
final Injector mockInjector = Guice.createInjector(new AbstractModule() {
@Override
protected void configure() {
@@ -1156,8 +1160,10 @@ public class UpgradeCatalog213Test {
put("normal", mockClusterExpected);
}}).atLeastOnce();
expect(mockClusterExpected.getDesiredConfigByType("hive-site")).andReturn(hiveSiteConf).atLeastOnce();
+ expect(mockClusterExpected.getDesiredConfigByType("hive-env")).andReturn(hiveEnvConf).atLeastOnce();
expect(hiveSiteConf.getProperties()).andReturn(propertiesHiveSite).once();
+ expect(hiveEnvConf.getProperties()).andReturn(propertiesHiveEnv).once();
UpgradeCatalog213 upgradeCatalog213 = createMockBuilder(UpgradeCatalog213.class)
.withConstructor(Injector.class)
@@ -1168,6 +1174,9 @@ public class UpgradeCatalog213Test {
upgradeCatalog213.updateConfigurationPropertiesForCluster(mockClusterExpected,
"hive-site", propertiesHiveSiteExpected, true, false);
expectLastCall().once();
+ upgradeCatalog213.updateConfigurationPropertiesForCluster(mockClusterExpected,
+ "hive-env", propertiesHiveEnv, true, true);
+ expectLastCall().once();
easyMockSupport.replayAll();
replay(upgradeCatalog213);
@@ -1176,4 +1185,73 @@ public class UpgradeCatalog213Test {
}
+ @Test
+ public void testUpdateHiveEnvContentHDP23() throws Exception {
+ UpgradeCatalog213 upgradeCatalog213 = new UpgradeCatalog213(injector);
+ String testContent = "# The heap size of the jvm stared by hive shell script can be controlled via:\n" +
+ "\n" +
+ "# Larger heap size may be required when running queries over large number of files or partitions.\n";
+ String expectedResult = "# The heap size of the jvm stared by hive shell script can be controlled via:\n" +
+ "\n" +
+ "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+ " export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore\n" +
+ "else\n" +
+ " export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client\n" +
+ "fi\n" +
+ "\n" +
+ "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\"\n" +
+ "\n" +
+ "# Larger heap size may be required when running queries over large number of files or partitions.\n";
+ Assert.assertEquals(expectedResult, upgradeCatalog213.updateHiveEnvContentHDP23(testContent));
+ }
+
+
+ @Test
+ public void testUpdateHiveEnvContent() throws Exception {
+ UpgradeCatalog213 upgradeCatalog213 = new UpgradeCatalog213(injector);
+ // Test first case
+ String testContent = "# The heap size of the jvm stared by hive shell script can be controlled via:\n" +
+ "\n" +
+ "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+ " export HADOOP_HEAPSIZE=\"{{hive_metastore_heapsize}}\"\n" +
+ "else\n" +
+ " export HADOOP_HEAPSIZE=\"{{hive_heapsize}}\"\n" +
+ "fi\n" +
+ "\n" +
+ "export HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n" +
+ "\n" +
+ "# Larger heap size may be required when running queries over large number of files or partitions.\n";
+ String expectedResult = "# The heap size of the jvm stared by hive shell script can be controlled via:\n" +
+ "\n" +
+ "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+ " export HADOOP_HEAPSIZE=\"{{hive_metastore_heapsize}}\"\n" +
+ "else\n" +
+ " export HADOOP_HEAPSIZE=\"{{hive_heapsize}}\"\n" +
+ "fi\n" +
+ "\n" +
+ "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\"\n" +
+ "\n" +
+ "# Larger heap size may be required when running queries over large number of files or partitions.\n";
+ Assert.assertEquals(expectedResult, upgradeCatalog213.updateHiveEnvContent(testContent));
+ // Test second case
+ testContent = "# The heap size of the jvm stared by hive shell script can be controlled via:\n" +
+ "export SERVICE=$SERVICE\n" +
+ "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+ " export HADOOP_HEAPSIZE=\"{{hive_metastore_heapsize}}\"\n" +
+ "else\n" +
+ " export HADOOP_HEAPSIZE=\"{{hive_heapsize}}\"\n" +
+ "fi\n" +
+ "\n";
+ expectedResult = "# The heap size of the jvm stared by hive shell script can be controlled via:\n" +
+ "export SERVICE=$SERVICE\n" +
+ "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+ " export HADOOP_HEAPSIZE=\"{{hive_metastore_heapsize}}\"\n" +
+ "else\n" +
+ " export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client\n" +
+ "fi\n" +
+ "\n" +
+ "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\"";
+ Assert.assertEquals(expectedResult, upgradeCatalog213.updateHiveEnvContent(testContent));
+ }
+
}