You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/12/06 16:34:38 UTC
ambari git commit: AMBARI-14234 If a non-hdfs path is specified (such
as wasb) as the hbase.rootdir then stack advisor throws error (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk c1b3692d7 -> c4dea5a04
AMBARI-14234 If a non-hdfs path is specified (such as wasb) as the hbase.rootdir then stack advisor throws error (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c4dea5a0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c4dea5a0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c4dea5a0
Branch: refs/heads/trunk
Commit: c4dea5a04c57712d2cc22f25a01f1656ab22c4f5
Parents: c1b3692
Author: Dmytro Sen <ds...@apache.org>
Authored: Sun Dec 6 17:34:23 2015 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Sun Dec 6 17:34:23 2015 +0200
----------------------------------------------------------------------
.../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 2 +-
.../0.1.0/package/scripts/params.py | 1 +
.../stacks/HDP/2.0.6/services/stack_advisor.py | 30 +++++++-------
.../stacks/2.0.6/common/test_stack_advisor.py | 42 ++++++++++++++++++--
4 files changed, 57 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4dea5a0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
index 615e93a..733fcc0 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
@@ -285,7 +285,7 @@ def ams(name=None):
)
pass
- if not params.is_local_fs_rootdir:
+ if not params.is_local_fs_rootdir and params.is_ams_distributed:
# Configuration needed to support NN HA
XmlConfig("hdfs-site.xml",
conf_dir=params.ams_collector_conf_dir,
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4dea5a0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index ec5e848..9125a01 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -90,6 +90,7 @@ hbase_pid_dir = status_params.hbase_pid_dir
is_hbase_distributed = config['configurations']['ams-hbase-site']['hbase.cluster.distributed']
is_local_fs_rootdir = hbase_root_dir.startswith('file://')
+is_ams_distributed = config['configurations']['ams-site']['timeline.metrics.service.operation.mode'] == 'distributed'
# security is disabled for embedded mode, when HBase is backed by file
security_enabled = False if not is_hbase_distributed else config['configurations']['cluster-env']['security_enabled']
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4dea5a0/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 9d9e0dd..f3e6f8b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -476,11 +476,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if host["Hosts"]["host_name"] == collectorHostName:
mountpoints = self.getPreferredMountPoints(host["Hosts"])
break
- if not rootDir.startswith("hdfs://"):
+ isLocalRootDir = rootDir.startswith("file://")
+ if isLocalRootDir:
rootDir = re.sub("^file:///|/", "", rootDir, count=1)
rootDir = "file://" + os.path.join(mountpoints[0], rootDir)
tmpDir = re.sub("^file:///|/", "", tmpDir, count=1)
- if len(mountpoints) > 1 and not rootDir.startswith("hdfs://"):
+ if len(mountpoints) > 1 and isLocalRootDir:
tmpDir = os.path.join(mountpoints[1], tmpDir)
else:
tmpDir = os.path.join(mountpoints[0], tmpDir)
@@ -840,7 +841,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
op_mode = ams_site.get("timeline.metrics.service.operation.mode")
hbase_rootdir = properties.get("hbase.rootdir")
hbase_tmpdir = properties.get("hbase.tmp.dir")
- if op_mode == "distributed" and not hbase_rootdir.startswith("hdfs://"):
+ if op_mode == "distributed" and hbase_rootdir.startswith("file://"):
rootdir_item = self.getWarnItem("In distributed mode hbase.rootdir should point to HDFS.")
pass
@@ -855,9 +856,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
for collectorHostName in amsCollectorHosts:
for host in hosts["items"]:
if host["Hosts"]["host_name"] == collectorHostName:
- validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorEnoughDiskSpace(properties, 'hbase.rootdir', host["Hosts"], recommendedDiskSpace)}])
- validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties, recommendedDefaults, 'hbase.rootdir', host["Hosts"])}])
- validationItems.extend([{"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties, recommendedDefaults, 'hbase.tmp.dir', host["Hosts"])}])
+ if op_mode == 'embedded':
+ validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorEnoughDiskSpace(properties, 'hbase.rootdir', host["Hosts"], recommendedDiskSpace)}])
+ validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties, recommendedDefaults, 'hbase.rootdir', host["Hosts"])}])
+ validationItems.extend([{"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties, recommendedDefaults, 'hbase.tmp.dir', host["Hosts"])}])
dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE")
if not hbase_rootdir.startswith("hdfs"):
@@ -951,35 +953,35 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
maxMasterXmn = 0.2 * hbase_master_heapsize
if hbase_master_xmn_size < minMasterXmn:
masterXmnItem = self.getWarnItem("Value is lesser than the recommended minimum Xmn size of {0} "
- "(12% of hbase_master_heapsize)".format(int(math.ceil(minMasterXmn))))
+ "(12% of hbase_master_heapsize)".format(int(ceil(minMasterXmn))))
if hbase_master_xmn_size > maxMasterXmn:
masterXmnItem = self.getWarnItem("Value is greater than the recommended maximum Xmn size of {0} "
- "(20% of hbase_master_heapsize)".format(int(math.floor(maxMasterXmn))))
+ "(20% of hbase_master_heapsize)".format(int(floor(maxMasterXmn))))
minRegionServerXmn = 0.12 * hbase_regionserver_heapsize
maxRegionServerXmn = 0.2 * hbase_regionserver_heapsize
if hbase_regionserver_xmn_size < minRegionServerXmn:
regionServerXmnItem = self.getWarnItem("Value is lesser than the recommended minimum Xmn size of {0} "
"(12% of hbase_regionserver_heapsize)"
- .format(int(math.ceil(minRegionServerXmn))))
+ .format(int(ceil(minRegionServerXmn))))
if hbase_regionserver_xmn_size > maxRegionServerXmn:
regionServerXmnItem = self.getWarnItem("Value is greater than the recommended maximum Xmn size of {0} "
"(20% of hbase_regionserver_heapsize)"
- .format(int(math.floor(maxRegionServerXmn))))
+ .format(int(floor(maxRegionServerXmn))))
else:
minMasterXmn = 0.12 * (hbase_master_heapsize + hbase_regionserver_heapsize)
maxMasterXmn = 0.2 * (hbase_master_heapsize + hbase_regionserver_heapsize)
if hbase_master_xmn_size < minMasterXmn:
masterXmnItem = self.getWarnItem("Value is lesser than the recommended minimum Xmn size of {0} "
"(12% of hbase_master_heapsize + hbase_regionserver_heapsize)"
- .format(int(math.ceil(minMasterXmn))))
+ .format(int(ceil(minMasterXmn))))
if hbase_master_xmn_size > maxMasterXmn:
masterXmnItem = self.getWarnItem("Value is greater than the recommended maximum Xmn size of {0} "
"(20% of hbase_master_heapsize + hbase_regionserver_heapsize)"
- .format(int(math.floor(maxMasterXmn))))
+ .format(int(floor(maxMasterXmn))))
if masterXmnItem:
validationItems.extend([{"config-name": "hbase_master_xmn_size", "item": masterXmnItem}])
@@ -1098,7 +1100,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if not propertyName in properties:
return self.getErrorItem("Value should be set")
dir = properties[propertyName]
- if dir.startswith("hdfs://") or dir == recommendedDefaults.get(propertyName):
+ if not dir.startswith("file://") or dir == recommendedDefaults.get(propertyName):
return None
dir = re.sub("^file://", "", dir, count=1)
@@ -1116,7 +1118,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if not propertyName in properties:
return self.getErrorItem("Value should be set")
dir = properties[propertyName]
- if dir.startswith("hdfs://"):
+ if not dir.startswith("file://"):
return None
dir = re.sub("^file://", "", dir, count=1)
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4dea5a0/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 439a769..d79b2ad 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -1726,8 +1726,8 @@ class TestHDP206StackAdvisor(TestCase):
"mountpoint" : "/"
}
]}
- properties = {"property1": "/var/dir"}
- recommendedDefaults = {"property1": "/var/dir"}
+ properties = {"property1": "file:///var/dir"}
+ recommendedDefaults = {"property1": "file:///var/dir"}
# only / mountpoint - no warning
self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults, 'property1', hostInfo) == None)
# More preferable /grid/0 mountpoint - warning
@@ -1738,7 +1738,7 @@ class TestHDP206StackAdvisor(TestCase):
"mountpoint" : "/grid/0"
}
)
- recommendedDefaults = {"property1": "/grid/0/var/dir"}
+ recommendedDefaults = {"property1": "file:///grid/0/var/dir"}
warn = self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults, 'property1', hostInfo)
self.assertFalse(warn == None)
self.assertEquals({'message': 'It is not recommended to use root partition for property1', 'level': 'WARN'}, warn)
@@ -1753,6 +1753,42 @@ class TestHDP206StackAdvisor(TestCase):
)
self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults, 'property1', hostInfo) == None)
+ def test_validatorEnoughDiskSpace(self):
+ reqiuredDiskSpace = 1048576
+ errorMsg = "Ambari Metrics disk space requirements not met. \n" \
+ "Recommended disk space for partition / is 1G"
+
+ # local FS, enough space
+ hostInfo = {"disk_info": [
+ {
+ "available" : "1048578",
+ "type" : "ext4",
+ "mountpoint" : "/"
+ }
+ ]}
+ properties = {"property1": "file:///var/dir"}
+ self.assertTrue(self.stackAdvisor.validatorEnoughDiskSpace(properties, 'property1', hostInfo, reqiuredDiskSpace) == None)
+
+ # local FS, no enough space
+ hostInfo = {"disk_info": [
+ {
+ "available" : "1",
+ "type" : "ext4",
+ "mountpoint" : "/"
+ }
+ ]}
+ warn = self.stackAdvisor.validatorEnoughDiskSpace(properties, 'property1', hostInfo, reqiuredDiskSpace)
+ self.assertTrue(warn != None)
+ self.assertEquals({'message': errorMsg, 'level': 'WARN'}, warn)
+
+ # non-local FS, HDFS
+ properties = {"property1": "hdfs://h1"}
+ self.assertTrue(self.stackAdvisor.validatorEnoughDiskSpace(properties, 'property1', hostInfo, reqiuredDiskSpace) == None)
+
+ # non-local FS, WASB
+ properties = {"property1": "wasb://h1"}
+ self.assertTrue(self.stackAdvisor.validatorEnoughDiskSpace(properties, 'property1', hostInfo, reqiuredDiskSpace) == None)
+
def test_round_to_n(self):
self.assertEquals(self.stack_advisor_impl.round_to_n(0), 0)
self.assertEquals(self.stack_advisor_impl.round_to_n(1000), 1024)