You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/08/19 19:44:10 UTC
ambari git commit: AMBARI-12817 YARN ownership is not set correctly
on container-executor (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 3705d2968 -> 73a1db1ac
AMBARI-12817 YARN ownership is not set correctly on container-executor (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/73a1db1a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/73a1db1a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/73a1db1a
Branch: refs/heads/trunk
Commit: 73a1db1ac6f71d2ed50be7130de52c85bfde03c2
Parents: 3705d29
Author: Dmytro Sen <ds...@apache.org>
Authored: Wed Aug 19 20:39:35 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Wed Aug 19 20:39:35 2015 +0300
----------------------------------------------------------------------
...ackLevelConfigurationResourceDefinition.java | 11 +++----
.../server/api/services/AmbariMetaInfo.java | 5 +++-
.../apache/ambari/server/stack/StackModule.java | 31 ++++++++++++++------
.../YARN/2.1.0.2.0/configuration/yarn-site.xml | 6 ++++
.../src/main/resources/key_properties.json | 1 +
.../stacks/HDP/2.0.6/services/stack_advisor.py | 6 ++++
.../services/YARN/configuration/yarn-site.xml | 4 +++
.../stacks/2.0.6/common/test_stack_advisor.py | 5 +++-
.../stacks/2.2/common/test_stack_advisor.py | 22 ++++++++++++--
.../app/controllers/wizard/step7_controller.js | 12 ++++++--
ambari-web/app/mixins/common/serverValidator.js | 10 +++++++
11 files changed, 92 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java
index ce6a5b3..5a6ff7e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java
@@ -27,7 +27,7 @@ import org.apache.ambari.server.controller.spi.Resource.Type;
public class StackLevelConfigurationResourceDefinition extends BaseResourceDefinition {
public StackLevelConfigurationResourceDefinition(Type resourceType) {
- super(Resource.Type.StackLevelConfiguration);
+ super(resourceType);
}
public StackLevelConfigurationResourceDefinition() {
@@ -43,11 +43,12 @@ public class StackLevelConfigurationResourceDefinition extends BaseResourceDefin
public String getSingularName() {
return "configuration";
}
- /*
+
@Override
public Set<SubResourceDefinition> getSubResourceDefinitions() {
- Set<SubResourceDefinition> setChildren = new HashSet<SubResourceDefinition>();
- return setChildren;
+ Set<SubResourceDefinition> subs = new HashSet<SubResourceDefinition>();
+ subs.add(new SubResourceDefinition(Resource.Type.StackConfigurationDependency));
+
+ return subs;
}
- */
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
index 4afa9b0..561b3f4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
@@ -631,7 +631,10 @@ public class AmbariMetaInfo {
public Set<PropertyInfo> getPropertiesByName(String stackName, String version, String serviceName, String propertyName)
throws AmbariException {
- Set<PropertyInfo> properties = getServiceProperties(stackName, version, serviceName);
+
+ Set<PropertyInfo> properties = serviceName == null ?
+ getStackProperties(stackName, version)
+ : getServiceProperties(stackName, version, serviceName);
if (properties.size() == 0) {
throw new StackAccessException("stackName=" + stackName
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index 4b88aff..2df6e04 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@ -590,7 +590,8 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
// Stack-definition has 'depends-on' relationship specified.
// We have a map to construct the 'depended-by' relationship.
- Map<PropertyDependencyInfo, Set<PropertyDependencyInfo>> dependedByMap = new HashMap<PropertyDependencyInfo, Set<PropertyDependencyInfo>>();
+ Map<PropertyDependencyInfo, Set<PropertyDependencyInfo>> dependedByMap =
+ new HashMap<PropertyDependencyInfo, Set<PropertyDependencyInfo>>();
// Go through all service-configs and gather the reversed 'depended-by'
// relationship into map. Since we do not have the reverse {@link PropertyInfo},
@@ -616,14 +617,26 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
// Go through all service-configs again and set their 'depended-by' if necessary.
for (ServiceModule serviceModule : serviceModules.values()) {
- for (PropertyInfo pi : serviceModule.getModuleInfo().getProperties()) {
- String type = ConfigHelper.fileNameToConfigType(pi.getFilename());
- String name = pi.getName();
- Set<PropertyDependencyInfo> set =
- dependedByMap.remove(new PropertyDependencyInfo(type, name));
- if (set != null) {
- pi.getDependedByProperties().addAll(set);
- }
+ addDependedByProperties(dependedByMap, serviceModule.getModuleInfo().getProperties());
+ }
+ // Go through all stack-configs again and set their 'depended-by' if necessary.
+ addDependedByProperties(dependedByMap, stackInfo.getProperties());
+ }
+
+ /**
+ * Add dependendByProperties to property info's
+ * @param dependedByMap Map containing the 'depended-by' relationships
+ * @param properties properties to check against dependedByMap
+ */
+ private void addDependedByProperties(Map<PropertyDependencyInfo, Set<PropertyDependencyInfo>> dependedByMap,
+ Collection<PropertyInfo> properties) {
+ for (PropertyInfo pi : properties) {
+ String type = ConfigHelper.fileNameToConfigType(pi.getFilename());
+ String name = pi.getName();
+ Set<PropertyDependencyInfo> set =
+ dependedByMap.remove(new PropertyDependencyInfo(type, name));
+ if (set != null) {
+ pi.getDependedByProperties().addAll(set);
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
index 686e40c..26a74df 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
@@ -178,6 +178,12 @@
<name>yarn.nodemanager.linux-container-executor.group</name>
<value>hadoop</value>
<description>Unix group of the NodeManager</description>
+ <depends-on>
+ <property>
+ <type>cluster-env</type>
+ <name>user_group</name>
+ </property>
+ </depends-on>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-server/src/main/resources/key_properties.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/key_properties.json b/ambari-server/src/main/resources/key_properties.json
index ff20b8b..9d71981 100644
--- a/ambari-server/src/main/resources/key_properties.json
+++ b/ambari-server/src/main/resources/key_properties.json
@@ -70,6 +70,7 @@
"StackVersion": "StackConfigurationDependency/stack_version",
"StackService": "StackConfigurationDependency/service_name",
"StackConfiguration": "StackConfigurationDependency/property_name",
+ "StackLevelConfiguration": "StackConfigurationDependency/property_name",
"StackConfigurationDependency": "StackConfigurationDependency/dependency_name"
},
"StackServiceComponent": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 3fd010a..5fa9f41 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -130,6 +130,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer']))
putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]))
putYarnEnvProperty('min_user_id', self.get_system_min_uid())
+ containerExecutorGroup = 'hadoop'
+ if 'cluster-env' in services['configurations'] and 'user_group' in services['configurations']['cluster-env']['properties']:
+ containerExecutorGroup = services['configurations']['cluster-env']['properties']['user_group']
+ putYarnProperty("yarn.nodemanager.linux-container-executor.group", containerExecutorGroup)
def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts):
putMapredProperty = self.putProperty(configurations, "mapred-site", services)
@@ -722,8 +726,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return self.toConfigurationValidationProblems(validationItems, "mapred-site")
def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ clusterEnv = getSiteProperties(configurations, "cluster-env")
validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
{"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
+ {"config-name": 'yarn.nodemanager.linux-container-executor.group', "item": self.validatorEqualsPropertyItem(properties, "yarn.nodemanager.linux-container-executor.group", clusterEnv, "user_group")},
{"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
return self.toConfigurationValidationProblems(validationItems, "yarn-site")
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
index 049eeb2..c69fd39 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
@@ -452,6 +452,10 @@
<type>yarn-env</type>
<name>yarn_cgroups_enabled</name>
</property>
+ <property>
+ <type>cluster-env</type>
+ <name>user_group</name>
+ </property>
</depends-on>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index b478cc0..887e172 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -202,6 +202,7 @@ class TestHDP206StackAdvisor(TestCase):
expectedItems = [
{"message": "Value is less than the recommended default of 512", "level": "WARN"},
+ {'message': 'Value should be set for yarn.nodemanager.linux-container-executor.group', 'level': 'ERROR'},
{"message": "Value should be integer", "level": "ERROR"},
{"message": "Value should be set", "level": "ERROR"}
]
@@ -466,6 +467,7 @@ class TestHDP206StackAdvisor(TestCase):
def test_recommendYARNConfigurations(self):
configurations = {}
+ services = {"configurations": configurations}
clusterData = {
"containers" : 5,
"ramPerContainer": 256
@@ -478,6 +480,7 @@ class TestHDP206StackAdvisor(TestCase):
},
"yarn-site": {
"properties": {
+ "yarn.nodemanager.linux-container-executor.group": "hadoop",
"yarn.nodemanager.resource.memory-mb": "1280",
"yarn.scheduler.minimum-allocation-mb": "256",
"yarn.scheduler.maximum-allocation-mb": "1280"
@@ -485,7 +488,7 @@ class TestHDP206StackAdvisor(TestCase):
}
}
- self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, None, None)
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None)
self.assertEquals(configurations, expected)
def test_recommendMapReduce2Configurations_mapMemoryLessThan2560(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index e764739..851c8b5 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -690,6 +690,7 @@ class TestHDP22StackAdvisor(TestCase):
def test_recommendYARNConfigurations(self):
configurations = {}
+ services = {"configurations": configurations}
clusterData = {
"cpu": 4,
"containers" : 5,
@@ -703,6 +704,7 @@ class TestHDP22StackAdvisor(TestCase):
},
"yarn-site": {
"properties": {
+ "yarn.nodemanager.linux-container-executor.group": "hadoop",
"yarn.nodemanager.resource.memory-mb": "1280",
"yarn.scheduler.minimum-allocation-mb": "256",
"yarn.scheduler.maximum-allocation-mb": "1280",
@@ -713,7 +715,7 @@ class TestHDP22StackAdvisor(TestCase):
}
}
- self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, None, None)
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None)
self.assertEquals(configurations, expected)
def test_recommendYARNConfigurationAttributes(self):
@@ -745,6 +747,7 @@ class TestHDP22StackAdvisor(TestCase):
},
"yarn-site": {
"properties": {
+ "yarn.nodemanager.linux-container-executor.group": "hadoop",
"yarn.nodemanager.resource.memory-mb": "1280",
"yarn.scheduler.minimum-allocation-mb": "256",
"yarn.scheduler.maximum-allocation-vcores": "2",
@@ -1303,7 +1306,12 @@ class TestHDP22StackAdvisor(TestCase):
"yarn.scheduler.maximum-allocation-mb": "2048",
"yarn.nodemanager.resource.cpu-vcores": "2"
},
- }
+ },
+ "cluster-env": {
+ "properties": {
+ "user_group": "hadoopcustom",
+ }
+ }
}
clusterData = {
"cpu": 4,
@@ -1312,6 +1320,11 @@ class TestHDP22StackAdvisor(TestCase):
"totalAvailableRam": 4096,
}
expected = {
+ "cluster-env": {
+ "properties": {
+ "user_group": "hadoopcustom"
+ }
+ },
"yarn-env": {
"properties": {
"min_user_id": "500"
@@ -1339,6 +1352,7 @@ class TestHDP22StackAdvisor(TestCase):
},
"yarn-site": {
"properties": {
+ "yarn.nodemanager.linux-container-executor.group": "hadoopcustom",
"yarn.nodemanager.resource.memory-mb": "1792",
"yarn.scheduler.minimum-allocation-mb": "100",
"yarn.scheduler.maximum-allocation-vcores": "1",
@@ -1597,6 +1611,7 @@ class TestHDP22StackAdvisor(TestCase):
},
"yarn-site": {
"properties": {
+ "yarn.nodemanager.linux-container-executor.group": "hadoop",
"yarn.nodemanager.resource.memory-mb": "1280",
"yarn.scheduler.minimum-allocation-mb": "100",
"yarn.scheduler.maximum-allocation-vcores": "1",
@@ -1807,6 +1822,7 @@ class TestHDP22StackAdvisor(TestCase):
},
"yarn-site": {
"properties": {
+ "yarn.nodemanager.linux-container-executor.group": "hadoop",
"yarn.nodemanager.resource.memory-mb": "1280",
"yarn.scheduler.minimum-allocation-mb": "700",
"yarn.scheduler.maximum-allocation-vcores": "1",
@@ -2687,6 +2703,7 @@ class TestHDP22StackAdvisor(TestCase):
},
"yarn-site": {
"properties": {
+ "yarn.nodemanager.linux-container-executor.group": "hadoop",
"yarn.nodemanager.container-executor.group": "hadoop",
"yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor",
"yarn.nodemanager.linux-container-executor.cgroups.mount-path": "/cgroup",
@@ -2743,6 +2760,7 @@ class TestHDP22StackAdvisor(TestCase):
"yarn.nodemanager.container-executor.group": "hadoop",
"yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
"yarn.nodemanager.linux-container-executor.cgroups.mount-path": "/cgroup",
+ "yarn.nodemanager.linux-container-executor.group": "hadoop",
"yarn.nodemanager.container-executor.cgroups.mount": "true",
"yarn.nodemanager.resource.memory-mb": "39424",
"yarn.scheduler.minimum-allocation-mb": "3584",
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-web/app/controllers/wizard/step7_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/wizard/step7_controller.js b/ambari-web/app/controllers/wizard/step7_controller.js
index 4d13b5b..5bc939d 100644
--- a/ambari-web/app/controllers/wizard/step7_controller.js
+++ b/ambari-web/app/controllers/wizard/step7_controller.js
@@ -883,7 +883,7 @@ App.WizardStep7Controller = Em.Controller.extend(App.ServerValidatorMixin, App.E
var serviceConfigTags = [];
for (var site in data.Clusters.desired_configs) {
if (data.Clusters.desired_configs.hasOwnProperty(site)) {
- if (installedServiceSites.contains(site)) {
+ if (installedServiceSites.contains(site) || site == 'cluster-env') {
serviceConfigTags.push({
siteName: site,
tagName: data.Clusters.desired_configs[site].tag,
@@ -1183,8 +1183,14 @@ App.WizardStep7Controller = Em.Controller.extend(App.ServerValidatorMixin, App.E
var serviceToShow = this.get('selectedServiceNames').concat('MISC');
var miscConfigs = this.get('stepConfigs').findProperty('serviceName', 'MISC').configs;
if (this.get('wizardController.name') == "addServiceController") {
- miscConfigs.findProperty('name', 'smokeuser').set('value', this.get('content.smokeuser')).set('isEditable', false);
- miscConfigs.findProperty('name', 'user_group').set('value', this.get('content.group')).set('isEditable', false);
+ miscConfigs.findProperty('name', 'smokeuser').set('isEditable', false);
+ miscConfigs.findProperty('name', 'user_group').set('isEditable', false);
+ if (this.get('content.smokeuser')) {
+ miscConfigs.findProperty('name', 'smokeuser').set('value', this.get('content.smokeuser'));
+ }
+ if (this.get('content.group')) {
+ miscConfigs.findProperty('name', 'user_group').set('value', this.get('content.group'));
+ }
}
App.config.miscConfigVisibleProperty(miscConfigs, serviceToShow);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/73a1db1a/ambari-web/app/mixins/common/serverValidator.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/serverValidator.js b/ambari-web/app/mixins/common/serverValidator.js
index b0c0772..9d696c2 100644
--- a/ambari-web/app/mixins/common/serverValidator.js
+++ b/ambari-web/app/mixins/common/serverValidator.js
@@ -129,7 +129,17 @@ App.ServerValidatorMixin = Em.Mixin.create({
recommendations.blueprint.configurations = blueprintUtils.buildConfigsJSON(this.get('services'), this.get('stepConfigs').filter(function(serviceConfigs) {
return self.get('installedServiceNames').contains(serviceConfigs.get('serviceName'));
}));
+ // include cluster-env site to recommendations call
+ var miscService = this.get('services').findProperty('serviceName', 'MISC');
+ if (miscService) {
+ var miscConfigs = blueprintUtils.buildConfigsJSON([miscService], [this.get('stepConfigs').findProperty('serviceName', 'MISC')]);
+ var clusterEnv = App.permit(miscConfigs, 'cluster-env');
+ if (!App.isEmptyObject(clusterEnv)) {
+ $.extend(recommendations.blueprint.configurations, clusterEnv);
+ }
+ }
}
+
return App.ajax.send({
'name': 'config.recommendations',
'sender': this,