You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2016/12/12 06:02:12 UTC
[3/5] ambari git commit: AMBARI-18901. LLAP integration enhancements
(Swapan Sridhar via smohanty)
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml
new file mode 100644
index 0000000..4bd1a19
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>tez.task.launch.cmd-opts</name>
+ <value>-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{{heap_dump_opts}}</value>
+ <description>Java options for tasks. The Xmx value is derived based on tez.task.resource.memory.mb and is 80% of this value by default.
+ Used only if the value is not specified explicitly by the DAG definition.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 562444b..9072dd0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -18,6 +18,7 @@ limitations under the License.
"""
from resource_management.core.logger import Logger
import json
+import re
from resource_management.libraries.functions import format
@@ -30,7 +31,8 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict()
childRecommendConfDict = {
"DRUID": self.recommendDruidConfigurations,
- "ATLAS": self.recommendAtlasConfigurations
+ "ATLAS": self.recommendAtlasConfigurations,
+ "TEZ": self.recommendTezConfigurations
}
parentRecommendConfDict.update(childRecommendConfDict)
return parentRecommendConfDict
@@ -225,3 +227,24 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
"druid.processing.numThreads")}
]
return self.toConfigurationValidationProblems(validationItems, "druid-broker")
+
+ def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
+ super(HDP26StackAdvisor, self).recommendTezConfigurations(configurations, clusterData, services, hosts)
+ putTezProperty = self.putProperty(configurations, "tez-site")
+
+ # TEZ JVM options
+ jvmGCParams = "-XX:+UseParallelGC"
+ if "ambari-server-properties" in services and "java.home" in services["ambari-server-properties"]:
+ # JDK8 needs different parameters
+ match = re.match(".*\/jdk(1\.\d+)[\-\_\.][^/]*$", services["ambari-server-properties"]["java.home"])
+ if match and len(match.groups()) > 0:
+ # Is version >= 1.8
+ versionSplits = re.split("\.", match.group(1))
+ if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7:
+ jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB"
+ tez_jvm_opts = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA "
+ # Append 'jvmGCParams' and 'Heap Dump related option' (({{heap_dump_opts}}) Expanded while writing the
+ # configurations at start/restart time).
+ tez_jvm_updated_opts = tez_jvm_opts + jvmGCParams + "{{heap_dump_opts}}"
+ putTezProperty('tez.task.launch.cmd-opts', tez_jvm_updated_opts)
+ Logger.info("Updated 'tez-site' config 'tez.task.launch.cmd-opts' as : {0}".format(tez_jvm_updated_opts))
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
index 16da70c..4e42d2d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
@@ -18,31 +18,14 @@
package org.apache.ambari.server.upgrade;
-import javax.persistence.EntityManager;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.google.inject.Provider;
import junit.framework.Assert;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.anyString;
-import static org.easymock.EasyMock.capture;
-import static org.easymock.EasyMock.createMockBuilder;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.createStrictMock;
-import static org.easymock.EasyMock.eq;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.newCapture;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
-import static org.easymock.EasyMock.verify;
-import static org.junit.Assert.assertTrue;
-
-import java.lang.reflect.Method;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.Statement;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import org.apache.ambari.server.actionmanager.ActionManager;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
@@ -62,13 +45,29 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import com.google.common.collect.Maps;
-import com.google.gson.Gson;
-import com.google.inject.Binder;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.google.inject.Provider;
+import javax.persistence.EntityManager;
+import java.lang.reflect.Method;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createMockBuilder;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.createStrictMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.newCapture;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertTrue;
/**
* {@link UpgradeCatalog250} unit tests.
@@ -213,18 +212,22 @@ public class UpgradeCatalog250Test {
Method updateAmsConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAMSConfigs");
Method updateKafkaConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateKafkaConfigs");
Method updateHiveLlapConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHiveLlapConfigs");
+ Method updateHIVEInteractiveConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHIVEInteractiveConfigs");
+ Method updateTEZInteractiveConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateTEZInteractiveConfigs");
Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml");
Method updateTablesForZeppelinViewRemoval = UpgradeCatalog250.class.getDeclaredMethod("updateTablesForZeppelinViewRemoval");
Method updateAtlasConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAtlasConfigs");
UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class)
- .addMockedMethod(updateAmsConfigs)
- .addMockedMethod(updateKafkaConfigs)
- .addMockedMethod(updateHiveLlapConfigs)
- .addMockedMethod(addNewConfigurationsFromXml)
- .addMockedMethod(updateTablesForZeppelinViewRemoval)
- .addMockedMethod(updateAtlasConfigs)
- .createMock();
+ .addMockedMethod(updateAmsConfigs)
+ .addMockedMethod(updateKafkaConfigs)
+ .addMockedMethod(updateHiveLlapConfigs)
+ .addMockedMethod(addNewConfigurationsFromXml)
+ .addMockedMethod(updateHIVEInteractiveConfigs)
+ .addMockedMethod(updateTEZInteractiveConfigs)
+ .addMockedMethod(updateTablesForZeppelinViewRemoval)
+ .addMockedMethod(updateAtlasConfigs)
+ .createMock();
upgradeCatalog250.updateAMSConfigs();
expectLastCall().once();
@@ -235,6 +238,12 @@ public class UpgradeCatalog250Test {
upgradeCatalog250.updateKafkaConfigs();
expectLastCall().once();
+ upgradeCatalog250.updateHIVEInteractiveConfigs();
+ expectLastCall().once();
+
+ upgradeCatalog250.updateTEZInteractiveConfigs();
+ expectLastCall().once();
+
upgradeCatalog250.updateHiveLlapConfigs();
expectLastCall().once();
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index c8da075..70ce79e 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1548,12 +1548,11 @@ class TestHDP22StackAdvisor(TestCase):
expected["hive-site"]["properties"]["hive.stats.fetch.column.stats"]="false"
expected["hive-site"]["properties"]["hive.security.authorization.enabled"]="true"
expected["hive-site"]["properties"]["hive.server2.enable.doAs"]="false"
- expected["hive-site"]["properties"]["hive.security.metastore.authorization.manager"]=\
- "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly"
+ expected["hive-site"]["properties"]["hive.security.metastore.authorization.manager"]="org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly"
expected["hiveserver2-site"]["properties"]["hive.security.authorization.enabled"]="true"
expected["hiveserver2-site"]["properties"]["hive.security.authorization.manager"]="org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"
expected["hiveserver2-site"]["properties"]["hive.security.authenticator.manager"]="org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator"
- expected["hiveserver2-site"]["properties"]["hive.conf.restricted.list"]="hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role"
+ expected["hiveserver2-site"]["properties"]["hive.conf.restricted.list"]="hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager,hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled"
self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
@@ -1646,7 +1645,7 @@ class TestHDP22StackAdvisor(TestCase):
expected["hiveserver2-site"]["properties"]["hive.security.authenticator.manager"] = "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator"
expected["hiveserver2-site"]["properties"]["hive.security.authorization.manager"] = "com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory"
expected["hiveserver2-site"]["properties"]["hive.security.authorization.enabled"] = "true"
- expected["hiveserver2-site"]["properties"]["hive.conf.restricted.list"]="hive.security.authorization.enabled,hive.security.authorization.manager,hive.security.authenticator.manager"
+ expected["hiveserver2-site"]["properties"]["hive.conf.restricted.list"]="hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager,hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled"
self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations['hiveserver2-site'], expected["hiveserver2-site"])
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
index 3221d5d..e00668a 100644
--- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
+++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
@@ -384,6 +384,8 @@ class TestHiveServerInteractive(RMFTestCase):
hive_site_conf['hive.exec.post.hooks'] = 'a,b,org.apache.hadoop.hive.ql.hooks.ATSHook'
del hive_site_conf['hive.enforce.bucketing']
del hive_site_conf['hive.enforce.sorting']
+ del hive_site_conf['hive.llap.io.memory.size']
+ hive_site_conf['hive.llap.io.memory.size'] = 357564416L
hiveserver2_site_conf = {}
hiveserver2_site_conf.update(self.getConfig()['configurations']['hiveserver2-site'])