You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2013/11/04 10:22:08 UTC
git commit: AMBARI-3675: Default value of 'Default virtual memory for
a job's map-task' is not valid. (jaimin)
Updated Branches:
refs/heads/trunk 95d63f0ed -> c13df4d7d
AMBARI-3675: Default value of 'Default virtual memory for a job's map-task' is not valid. (jaimin)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/c13df4d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/c13df4d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/c13df4d7
Branch: refs/heads/trunk
Commit: c13df4d7dc3d65dfd580fccadb278913f47f843a
Parents: 95d63f0
Author: Jaimin Jetly <ja...@hortonworks.com>
Authored: Mon Nov 4 01:19:01 2013 -0800
Committer: Jaimin Jetly <ja...@hortonworks.com>
Committed: Mon Nov 4 01:21:04 2013 -0800
----------------------------------------------------------------------
.../app/controllers/wizard/step7_controller.js | 2 +-
ambari-web/app/utils/config.js | 7 +-
.../yarn_defaults_provider.js | 192 ++++++++++---------
ambari-web/app/utils/helper.js | 3 +
ambari-web/app/utils/string_utils.js | 2 +-
5 files changed, 114 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/c13df4d7/ambari-web/app/controllers/wizard/step7_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/wizard/step7_controller.js b/ambari-web/app/controllers/wizard/step7_controller.js
index edee875..273b2b1 100644
--- a/ambari-web/app/controllers/wizard/step7_controller.js
+++ b/ambari-web/app/controllers/wizard/step7_controller.js
@@ -98,7 +98,7 @@ App.WizardStep7Controller = Em.Controller.extend({
slaveComponentHosts: this.get('wizardController').getDBProperty('slaveComponentHosts')
};
//STEP 6: Distribute configs by service and wrap each one in App.ServiceConfigProperty (configs -> serviceConfigs)
- var serviceConfigs = App.config.renderConfigs(configs, storedConfigs, this.get('allInstalledServiceNames'), this.get('selectedServiceNames'), localDB);
+ var serviceConfigs = App.config.renderConfigs(configs, storedConfigs, this.get('allInstalledServiceNames'), this.get('selectedServiceNames'), localDB, storedConfigs);
this.set('stepConfigs', serviceConfigs);
this.activateSpecialConfigs();
this.set('selectedService', this.get('stepConfigs').filterProperty('showConfig', true).objectAt(0));
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/c13df4d7/ambari-web/app/utils/config.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/config.js b/ambari-web/app/utils/config.js
index f03554f..c032474 100644
--- a/ambari-web/app/utils/config.js
+++ b/ambari-web/app/utils/config.js
@@ -365,6 +365,7 @@ App.config = Em.Object.create({
configData.filename = stored.filename;
configData.description = stored.description;
} else if (!preDefined && stored) {
+
configData = {
id: stored.id,
name: stored.name,
@@ -465,9 +466,10 @@ App.config = Em.Object.create({
* @param allInstalledServiceNames
* @param selectedServiceNames
* @param localDB
+ * @param storedConfigs
* @return {Array}
*/
- renderConfigs: function (configs, storedConfigs, allInstalledServiceNames, selectedServiceNames, localDB) {
+ renderConfigs: function (configs, storedConfigs, allInstalledServiceNames, selectedServiceNames, localDB, storedConfigs) {
var renderedServiceConfigs = [];
var services = [];
@@ -500,7 +502,8 @@ App.config = Em.Object.create({
serviceConfig.set('showConfig', service.showConfig);
// Use calculated default values for some configs
- if (service.defaultsProviders) {
+
+ if (!storedConfigs && service.defaultsProviders) {
service.defaultsProviders.forEach(function(defaultsProvider) {
var defaults = defaultsProvider.getDefaults(localDB);
for(var name in defaults) {
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/c13df4d7/ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js b/ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js
index 0ae3e70..80732cb 100644
--- a/ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js
+++ b/ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js
@@ -31,6 +31,7 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({
'mapreduce.reduce.memory.mb': null,
'mapreduce.map.java.opts': null,
'mapreduce.reduce.java.opts': null,
+ 'mapreduce.task.io.sort.mb': null,
'yarn.app.mapreduce.am.resource.mb': null,
'yarn.app.mapreduce.am.command-opts': null
},
@@ -61,110 +62,118 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({
*/
hBaseRam: null,
+ GB: 1024,
/**
* Minimum container size (in RAM).
* This value is dependent on the amount of RAM available, as in smaller memory nodes the minimum container size should also be smaller
*
* Value in MB!
*/
- recommendedMinimumContainerSize: function() {
+ recommendedMinimumContainerSize: function () {
if (!this.clusterDataIsValid()) return null;
var ram = this.get('clusterData.ram');
- switch(true) {
- case (ram < 4*1024): return 256;
- case (ram >= 4*1024 && ram < 8*1024): return 512;
- case (ram >= 8*1024 && ram < 24*1024): return 1024;
- case (ram >= 24*1024):
- default: return 2048;
+ switch (true) {
+ case (ram <=4 ):
+ return 256;
+ case (ram <= 8):
+ return 512;
+ case (ram <= 24):
+ return 1024;
+ default:
+ return 2048;
}
}.property('clusterData.ram'),
/**
* Maximum number of containers allowed per node
+ * max (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE)))
* min (2*CORES, 1.8*DISKS, (Total available RAM) / MIN_CONTAINER_SIZE)
*/
- containers: function() {
+ containers: function () {
if (!this.clusterDataIsValid()) return null;
var cpu = this.get('clusterData.cpu');
var disk = this.get('clusterData.disk');
var ram = this.get('clusterData.ram');
var containerSize = this.get('recommendedMinimumContainerSize');
cpu *= 2;
- disk *= 1.8;
+ disk = Math.ceil(disk * 1.8);
ram = (ram - this.get('reservedRam'));
if (this.get('clusterData.hBaseInstalled')) {
ram -= this.get('hBaseRam')
}
+ ram *= this.get('GB');
ram /= containerSize;
- if (cpu < disk) {
- if (cpu < ram) {
- return cpu;
- }
- return ram;
- }
- else {
- if (disk < ram) {
- return parseInt(disk);
- }
- return ram;
- }
- }.property('clusterData.cpu', 'clusterData.ram', 'clusterData.disk', 'recommendedMinimumContainerSize'),
+ return Math.round(Math.max(cpu, Math.min(disk, ram)));
+ }.property('clusterData.cpu', 'clusterData.ram', 'clusterData.hBaseInstalled', 'clusterData.disk', 'reservedRam', 'hBaseRam', 'recommendedMinimumContainerSize'),
/**
* amount of RAM per container
- * RAM-per-container = max(MIN_CONTAINER_SIZE, (Total Available RAM) / containers))
+ * RAM-per-container = abs(MIN_CONTAINER_SIZE, (Total Available RAM) / containers))
*
* Value in MB!
*/
- ramPerContainer: function() {
- var containerSize = this.get('recommendedMinimumContainerSize');
+ ramPerContainer: function () {
var containers = this.get('containers');
- if (!containerSize || !containers) {
- return null;
- }
- var s = this.get('clusterData.ram') - this.get('reservedRam');
+ var ram = this.get('clusterData.ram');
+ ram = (ram - this.get('reservedRam'));
if (this.get('clusterData.hBaseInstalled')) {
- s -= this.get('hBaseRam');
+ ram -= this.get('hBaseRam')
}
- s /= containers;
- return (containerSize > s) ? containerSize : s;
- }.property('recommendedMinimumContainerSize', 'containers'),
+ ram *= this.get('GB');
+ var container_ram = Math.abs(ram / containers);
+ return container_ram > this.get('GB') ? container_ram / (512 * 512) : container_ram;
+ }.property('recommendedMinimumContainerSize', 'containers', 'clusterData.ram', 'clusterData.hBaseInstalled', 'hBaseRam', 'reservedRam'),
+
+ mapMemory: function () {
+ return this.get('ramPerContainer');
+ }.property('ramPerContainer'),
+
+ reduceMemory: function () {
+ var ramPerContainer = this.get('ramPerContainer');
+ return ramPerContainer <= 2048 ? 2 * ramPerContainer : ramPerContainer;
+ }.property('ramPerContainer'),
+
+ amMemeory: function () {
+ return Math.max(this.get('mapMemory'), this.get('reduceMemory'));
+ }.property('mapMemory', 'reduceMemory'),
/**
* Reserved for HBase and system memory is based on total available memory
*/
- reservedMemoryRecommendations: function() {
- var table = [
- {os:1,hbase:1},
- {os:2,hbase:1},
- {os:2,hbase:2},
- {os:4,hbase:4},
- {os:6,hbase:8},
- {os:8,hbase:8},
- {os:8,hbase:8},
- {os:12,hbase:16},
- {os:24,hbase:24},
- {os:32,hbase:32},
- {os:64,hbase:64}
- ];
- var ram = this.get('clusterData.ram') / 1024;
- var index = 0;
- switch (true) {
- case (ram <= 4): index = 0; break;
- case (ram > 4 && ram <= 8): index = 1; break;
- case (ram > 8 && ram <= 16): index = 2; break;
- case (ram > 16 && ram <= 24): index = 3; break;
- case (ram > 24 && ram <= 48): index = 4; break;
- case (ram > 48 && ram <= 64): index = 5; break;
- case (ram > 64 && ram <= 72): index = 6; break;
- case (ram > 72 && ram <= 96): index = 7; break;
- case (ram > 96 && ram <= 128): index = 8; break;
- case (ram > 128 && ram <= 256): index = 9; break;
- case (ram > 256 && ram <= 512): index = 10; break;
- default: index = 10; break;
+
+
+
+ reservedStackRecommendations: function () {
+ var memory = this.get('clusterData.ram');
+ var reservedStack = { 4: 1, 8: 2, 16: 2, 24: 4, 48: 6, 64: 8, 72: 8, 96: 12,
+ 128: 24, 256: 32, 512: 64};
+
+ if (memory in reservedStack) {
+ this.set('reservedRam', reservedStack[memory]);
}
- this.set('reservedRam', table[index].os * 1024);
- this.set('hBaseRam', table[index].hbase * 1024);
+ if (memory <= 4)
+ this.set('reservedRam', 1);
+ else if (memory >= 512)
+ this.set('reservedRam', 64);
+ else
+ this.set('reservedRam', 1);
+ }.observes('clusterData.ram'),
+
+ hbaseMemRecommendations: function () {
+ var memory = this.get('clusterData.ram');
+ var reservedHBase = {4:1, 8:1, 16:2, 24:4, 48:8, 64:8, 72:8, 96:16,
+ 128:24, 256:32, 512:64};
+
+ if (memory in reservedHBase) {
+ this.set('reservedRam', reservedHBase[memory]);
+ }
+ if (memory <= 4)
+ this.set('hBaseRam', 1);
+ else if (memory >= 512)
+ this.set('hBaseRam', 64);
+ else
+ this.set('hBaseRam', 2);
+
}.observes('clusterData.ram'),
/**
@@ -198,20 +207,21 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({
* </code>
* @return {object}
*/
- getDefaults: function(localDB) {
+ getDefaults: function (localDB) {
this._super();
this.getClusterData(localDB);
var configs = {};
jQuery.extend(configs, this.get('configsTemplate'));
- configs['yarn.nodemanager.resource.memory-mb'] = this.get('containers') * this.get('ramPerContainer');
- configs['yarn.scheduler.minimum-allocation-mb'] = this.get('ramPerContainer');
- configs['yarn.scheduler.maximum-allocation-mb'] = this.get('containers') * this.get('ramPerContainer');
- configs['mapreduce.map.memory.mb'] = this.get('ramPerContainer');
- configs['mapreduce.reduce.memory.mb'] = 2 * this.get('ramPerContainer');
- configs['mapreduce.map.java.opts'] = Math.round(0.8 * this.get('ramPerContainer'));
- configs['mapreduce.reduce.java.opts'] = Math.round(0.8 *2 * this.get('ramPerContainer'));
- configs['yarn.app.mapreduce.am.resource.mb'] = 2 * this.get('ramPerContainer');
- configs['yarn.app.mapreduce.am.command-opts'] = Math.round(0.8 * 2 * this.get('ramPerContainer'));
+ configs['yarn.nodemanager.resource.memory-mb'] = Math.round(this.get('containers') * this.get('ramPerContainer'));
+ configs['yarn.scheduler.minimum-allocation-mb'] = Math.round(this.get('ramPerContainer'));
+ configs['yarn.scheduler.maximum-allocation-mb'] = Math.round(this.get('containers') * this.get('ramPerContainer'));
+ configs['yarn.app.mapreduce.am.resource.mb'] = Math.round(this.get('amMemory'));
+ configs['yarn.app.mapreduce.am.command-opts'] = Math.round(0.8 * this.get('amMemory'));
+ configs['mapreduce.map.memory.mb'] = Math.round(this.get('mapMemory'));
+ configs['mapreduce.reduce.memory.mb'] = Math.round(this.get('reduceMemory'));
+ configs['mapreduce.map.java.opts'] = Math.round(0.8 * this.get('mapMemory'));
+ configs['mapreduce.reduce.java.opts'] = Math.round(0.8 * this.get('reduceMemory'));
+ configs['mapreduce.task.io.sort.mb'] = Math.round(0.4 * this.get('mapMemory'));
return configs;
},
@@ -219,13 +229,13 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({
* Calculate needed cluster data (like disk count, cpu count, ram (in MB!) and hbase availability)
* @param {object} localDB Object with information about hosts and master/slave components
*/
- getClusterData: function(localDB) {
+ getClusterData: function (localDB) {
this._super();
- var components = ['RESOURCEMANAGER', 'NODEMANAGER'];
+ var components = ['NODEMANAGER'];
var hosts = [];
if (!localDB.hosts || !(localDB.masterComponentHosts || localDB.slaveComponentHosts)) return;
var hBaseInstalled = !!localDB.masterComponentHosts.filterProperty('component', 'HBASE_MASTER').length;
- components.forEach(function(component) {
+ components.forEach(function (component) {
var mc = localDB.masterComponentHosts.findProperty('component', component);
if (mc) {
if (!hosts.contains(mc.hostName)) {
@@ -235,7 +245,7 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({
else {
var sc = localDB.slaveComponentHosts.findProperty('componentName', component);
if (sc) {
- sc.hosts.map(function(host) {
+ sc.hosts.map(function (host) {
if (!hosts.contains(host.hostName)) {
hosts.push(host.hostName);
}
@@ -249,23 +259,29 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({
ram: 0,
hBaseInstalled: hBaseInstalled
};
- hosts.forEach(function(hostName) {
- var host = localDB.hosts[hostName];
- if (host) {
- clusterData.cpu += parseInt(host.cpu);
- clusterData.disk += host.disk_info.length;
- clusterData.ram += Math.round(parseFloat(host.memory) / 1024);
- }
- });
+ var host = hosts[0] && localDB.hosts[hosts[0]];
+ if (host) {
+ clusterData.cpu = parseInt(host.cpu);
+ invalidMountPoints= ['/homes/','/dev/','/tmp/'];
+ var length = 0;
+ host.disk_info.forEach(function(disk) {
+ //invalid mountpoints
+ if (!(disk.mountpoint.startsWith('/homes/') || disk.mountpoint.startsWith('/dev/') || disk.mountpoint.startsWith('/tmp/'))) {
+ length++;
+ }
+ },this);
+ clusterData.disk = length;
+ clusterData.ram = Math.round(parseFloat(host.memory) / (1024 * 1024));
+ }
this.set('clusterData', clusterData);
},
/**
* Verify <code>clusterData</code> - check if all properties are defined
*/
- clusterDataIsValid: function() {
+ clusterDataIsValid: function () {
if (!this.get('clusterData')) return false;
- if (this.get('clusterData.ram') == null || this.get('clusterData.cpu') == null || this.get('clusterData.disk') == null || this.get('clusterData.hBaseInstalled') == null) return false;
+ if (this.get('clusterData.ram') == null || this.get('clusterData.cpu') == null || this.get('clusterData.disk') == null || this.get('clusterData.hBaseInstalled') == null) return false;
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/c13df4d7/ambari-web/app/utils/helper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/helper.js b/ambari-web/app/utils/helper.js
index 60961dc..f86a2be 100644
--- a/ambari-web/app/utils/helper.js
+++ b/ambari-web/app/utils/helper.js
@@ -23,6 +23,9 @@ String.prototype.trim = function () {
String.prototype.endsWith = function(suffix) {
return this.indexOf(suffix, this.length - suffix.length) !== -1;
};
+String.prototype.startsWith = function (prefix){
+ return this.indexOf(prefix) == 0;
+};
/**
* convert ip address string to long int
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/c13df4d7/ambari-web/app/utils/string_utils.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/string_utils.js b/ambari-web/app/utils/string_utils.js
index 85068ad..92cb75f 100644
--- a/ambari-web/app/utils/string_utils.js
+++ b/ambari-web/app/utils/string_utils.js
@@ -107,7 +107,7 @@ module.exports = {
},
isSingleLine: function(string){
- return string.trim().indexOf("\n") == -1;
+ return String(string).trim().indexOf("\n") == -1;
},
/**
* transform array of objects into CSV format content