You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/11/10 15:42:05 UTC
[2/2] ambari git commit: AMBARI-13800 It is not possible to change
"Metrics Collector log dir" (dsen)
AMBARI-13800 It is not possible to change "Metrics Collector log dir" (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0fdd0894
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0fdd0894
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0fdd0894
Branch: refs/heads/trunk
Commit: 0fdd089424e98478df87f4a7720d893fa94cfee6
Parents: 9b8701c
Author: Dmytro Sen <ds...@apache.org>
Authored: Tue Nov 10 16:41:40 2015 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Tue Nov 10 16:41:40 2015 +0200
----------------------------------------------------------------------
.../conf/unix/ambari-metrics-collector | 4 +-
.../0.1.0/configuration/ams-hbase-env.xml | 2 +-
.../0.1.0/package/scripts/ams_service.py | 3 +-
.../AMBARI_METRICS/test_metrics_collector.py | 211 ++--
.../python/stacks/2.0.6/configs/default.json | 2 +-
.../2.0.6/configs/default_ams_embedded.json | 1108 ++++++++++++++++++
6 files changed, 1233 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0fdd0894/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
index b8d6602..0018bc2 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector
@@ -34,7 +34,7 @@ HBASE_CMD=${HBASE_DIR}/bin/hbase
METRIC_TABLES=(METRIC_AGGREGATE_DAILY METRIC_AGGREGATE_HOURLY METRIC_AGGREGATE METRIC_RECORD METRIC_RECORD_DAILY METRIC_RECORD_HOURLY METRIC_RECORD_MINUTE)
METRIC_COLLECTOR=ambari-metrics-collector
-AMS_LOG_DIR=/var/log/ambari-metrics-collector
+AMS_COLLECTOR_LOG_DIR=/var/log/ambari-metrics-collector
STOP_TIMEOUT=5
@@ -218,7 +218,7 @@ function start()
echo "Collector successfully started."
echo "Initializing Ambari Metrics data model"
# Wait until METRIC_* tables created
- for retry in {1..10}
+ for retry in {1..5}
do
echo 'list' | ${HBASE_CMD} --config ${HBASE_CONF_DIR} shell | grep ^${METRIC_TABLES[0]} > /dev/null 2>&1
if [ $? -eq 0 ]; then
http://git-wip-us.apache.org/repos/asf/ambari/blob/0fdd0894/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml
index 0b899aa..8a5a0f1 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml
@@ -124,7 +124,7 @@
<value>
# Set environment variables here.
-# The java implementation to use. Java 1.6 required.
+# The java implementation to use. Java 1.6+ required.
export JAVA_HOME={{java64_home}}
# HBase Configuration directory
http://git-wip-us.apache.org/repos/asf/ambari/blob/0fdd0894/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_service.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_service.py
index 1afe1d8..2b475dd 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_service.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_service.py
@@ -55,7 +55,8 @@ def ams_service(name, action):
)
if not params.is_hbase_distributed and os.path.exists(format("{zookeeper_data_dir}")):
- Execute(format("{sudo} rm -rf {zookeeper_data_dir}/*")
+ Directory(params.zookeeper_data_dir,
+ action='delete'
)
daemon_cmd = format("{cmd} start")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0fdd0894/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
index 4beffba..b04e1d5 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
@@ -20,13 +20,13 @@ limitations under the License.
from mock.mock import MagicMock, patch
from stacks.utils.RMFTestCase import *
-
+@patch("os.path.exists", new = MagicMock(return_value=True))
@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
class TestOozieClient(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "AMBARI_METRICS/0.1.0/package"
STACK_VERSION = "2.0.6"
- def test_start_default(self):
+ def test_start_default_distributed(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metrics_collector.py",
classname = "AmsCollector",
command = "start",
@@ -35,9 +35,9 @@ class TestOozieClient(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.maxDiff=None
- self.assert_hbase_configure('master')
- self.assert_hbase_configure('regionserver')
- self.assert_ams('collector')
+ self.assert_hbase_configure('master', distributed=True)
+ self.assert_hbase_configure('regionserver', distributed=True)
+ self.assert_ams('collector', distributed=True)
self.assertResourceCalled('Execute', '/usr/lib/ams-hbase/bin/hbase-daemon.sh --config /etc/ams-hbase/conf start zookeeper',
not_if = 'ls /var/run/ambari-metrics-collector//hbase-ams-zookeeper.pid >/dev/null 2>&1 && ps `cat /var/run/ambari-metrics-collector//hbase-ams-zookeeper.pid` >/dev/null 2>&1',
user = 'ams'
@@ -57,7 +57,29 @@ class TestOozieClient(RMFTestCase):
)
self.assertNoMoreResources()
- def assert_ams(self, name=None):
+ def test_start_default_embedded(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metrics_collector.py",
+ classname = "AmsCollector",
+ command = "start",
+ config_file="default_ams_embedded.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.maxDiff=None
+ self.assert_hbase_configure('master')
+ self.assert_hbase_configure('regionserver')
+ self.assert_ams('collector')
+ self.assertResourceCalled('Execute', 'ambari-sudo.sh rm -rf /var/lib/ambari-metrics-collector/hbase-tmp/*.tmp',
+ )
+ self.assertResourceCalled('Directory', '/var/lib/ambari-metrics-collector/hbase-tmp/zookeeper',
+ action = ['delete']
+ )
+ self.assertResourceCalled('Execute', '/usr/sbin/ambari-metrics-collector --config /etc/ambari-metrics-collector/conf start',
+ user = 'ams'
+ )
+ self.assertNoMoreResources()
+
+ def assert_ams(self, name=None, distributed=False):
self.assertResourceCalled('Directory', '/etc/ambari-metrics-collector/conf',
owner = 'ams',
group = 'hadoop',
@@ -126,40 +148,41 @@ class TestOozieClient(RMFTestCase):
mode=0644,
content=Template("ams.conf.j2")
)
- self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
- owner = 'ams',
- group = 'hadoop',
- mode=0644,
- conf_dir = '/etc/ambari-metrics-collector/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'],
- configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
- )
- self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
- owner = 'ams',
- group = 'hadoop',
- mode=0644,
- conf_dir = '/etc/ams-hbase/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'],
- configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
- )
- self.assertResourceCalled('XmlConfig', 'core-site.xml',
- owner = 'ams',
- group = 'hadoop',
- mode=0644,
- conf_dir = '/etc/ambari-metrics-collector/conf',
- configurations = self.getConfig()['configurations']['core-site'],
- configuration_attributes = self.getConfig()['configuration_attributes']['core-site']
- )
- self.assertResourceCalled('XmlConfig', 'core-site.xml',
- owner = 'ams',
- group = 'hadoop',
- mode=0644,
- conf_dir = '/etc/ams-hbase/conf',
- configurations = self.getConfig()['configurations']['core-site'],
- configuration_attributes = self.getConfig()['configuration_attributes']['core-site']
- )
+ if distributed:
+ self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
+ owner = 'ams',
+ group = 'hadoop',
+ mode=0644,
+ conf_dir = '/etc/ambari-metrics-collector/conf',
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
+ )
+ self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
+ owner = 'ams',
+ group = 'hadoop',
+ mode=0644,
+ conf_dir = '/etc/ams-hbase/conf',
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
+ )
+ self.assertResourceCalled('XmlConfig', 'core-site.xml',
+ owner = 'ams',
+ group = 'hadoop',
+ mode=0644,
+ conf_dir = '/etc/ambari-metrics-collector/conf',
+ configurations = self.getConfig()['configurations']['core-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['core-site']
+ )
+ self.assertResourceCalled('XmlConfig', 'core-site.xml',
+ owner = 'ams',
+ group = 'hadoop',
+ mode=0644,
+ conf_dir = '/etc/ams-hbase/conf',
+ configurations = self.getConfig()['configurations']['core-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['core-site']
+ )
- def assert_hbase_configure(self, name=None):
+ def assert_hbase_configure(self, name=None, distributed=False):
self.assertResourceCalled('Directory', '/etc/ams-hbase/conf',
owner = 'ams',
group = 'hadoop',
@@ -184,13 +207,6 @@ class TestOozieClient(RMFTestCase):
configurations = self.getConfig()['configurations']['ams-hbase-site'],
configuration_attributes = self.getConfig()['configuration_attributes']['ams-hbase-site']
)
- self.assertResourceCalled('Directory', '/var/lib/ambari-metrics-collector/hbase-tmp/phoenix-spool',
- owner = 'ams',
- cd_access = 'a',
- group = 'hadoop',
- mode = 0755,
- recursive = True
- )
self.assertResourceCalled('XmlConfig', 'hbase-policy.xml',
owner = 'ams',
group = 'hadoop',
@@ -221,52 +237,63 @@ class TestOozieClient(RMFTestCase):
)
if name == 'master':
- self.assertResourceCalled('HdfsResource', 'hdfs://localhost:8020/apps/hbase/data',
- security_enabled = False,
- hadoop_bin_dir = '/usr/bin',
- keytab = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- user = 'hdfs',
- dfs_type = '',
- owner = 'ams',
- mode = 0775,
- hadoop_conf_dir = '/etc/hadoop/conf',
- type = 'directory',
- action = ['create_on_execute'],
- hdfs_site=self.getConfig()['configurations']['hdfs-site'],
- principal_name=UnknownConfigurationMock(),
- default_fs='hdfs://c6401.ambari.apache.org:8020',
- )
- self.assertResourceCalled('HdfsResource', '/amshbase/staging',
- security_enabled = False,
- hadoop_bin_dir = '/usr/bin',
- keytab = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- user = 'hdfs',
- dfs_type = '',
- owner = 'ams',
- mode = 0711,
- hadoop_conf_dir = '/etc/hadoop/conf',
- type = 'directory',
- action = ['create_on_execute'],
- hdfs_site=self.getConfig()['configurations']['hdfs-site'],
- principal_name=UnknownConfigurationMock(),
- default_fs='hdfs://c6401.ambari.apache.org:8020',
- )
- self.assertResourceCalled('HdfsResource', None,
- security_enabled = False,
- hadoop_bin_dir = '/usr/bin',
- keytab = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- user = 'hdfs',
- hadoop_conf_dir = '/etc/hadoop/conf',
- action = ['execute'],
- hdfs_site=self.getConfig()['configurations']['hdfs-site'],
- principal_name=UnknownConfigurationMock(),
- default_fs='hdfs://c6401.ambari.apache.org:8020',
- )
- self.assertResourceCalled('File', '/var/run/ambari-metrics-collector//distributed_mode', action=["create"],
- mode=0644, owner='ams')
+ if distributed:
+ self.assertResourceCalled('HdfsResource', 'hdfs://localhost:8020/apps/hbase/data',
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/bin',
+ keytab = UnknownConfigurationMock(),
+ kinit_path_local = '/usr/bin/kinit',
+ user = 'hdfs',
+ dfs_type = '',
+ owner = 'ams',
+ mode = 0775,
+ hadoop_conf_dir = '/etc/hadoop/conf',
+ type = 'directory',
+ action = ['create_on_execute'],
+ hdfs_site=self.getConfig()['configurations']['hdfs-site'],
+ principal_name=UnknownConfigurationMock(),
+ default_fs='hdfs://c6401.ambari.apache.org:8020',
+ )
+ self.assertResourceCalled('HdfsResource', '/amshbase/staging',
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/bin',
+ keytab = UnknownConfigurationMock(),
+ kinit_path_local = '/usr/bin/kinit',
+ user = 'hdfs',
+ dfs_type = '',
+ owner = 'ams',
+ mode = 0711,
+ hadoop_conf_dir = '/etc/hadoop/conf',
+ type = 'directory',
+ action = ['create_on_execute'],
+ hdfs_site=self.getConfig()['configurations']['hdfs-site'],
+ principal_name=UnknownConfigurationMock(),
+ default_fs='hdfs://c6401.ambari.apache.org:8020',
+ )
+ self.assertResourceCalled('HdfsResource', None,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/bin',
+ keytab = UnknownConfigurationMock(),
+ kinit_path_local = '/usr/bin/kinit',
+ user = 'hdfs',
+ hadoop_conf_dir = '/etc/hadoop/conf',
+ action = ['execute'],
+ hdfs_site=self.getConfig()['configurations']['hdfs-site'],
+ principal_name=UnknownConfigurationMock(),
+ default_fs='hdfs://c6401.ambari.apache.org:8020',
+ )
+ self.assertResourceCalled('File', '/var/run/ambari-metrics-collector//distributed_mode', action=["create"],
+ mode=0644, owner='ams')
+ else:
+ self.assertResourceCalled('Directory', '/var/lib/ambari-metrics-collector/hbase',
+ owner = 'ams',
+ cd_access="a",
+ recursive = True
+ )
+ self.assertResourceCalled('File', '/var/run/ambari-metrics-collector//distributed_mode',
+ owner = 'ams',
+ action = ['delete']
+ )
self.assertResourceCalled('File', '/etc/ams-hbase/conf/log4j.properties',
owner = 'ams',
group = 'hadoop',
http://git-wip-us.apache.org/repos/asf/ambari/blob/0fdd0894/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index 2cdcb19..bc40657 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -788,7 +788,7 @@
"hbase.regionserver.thread.compaction.large": "2",
"phoenix.query.timeoutMs": "1200000",
"hbase.local.dir": "${hbase.tmp.dir}/local",
- "hbase.cluster.distributed": "false",
+ "hbase.cluster.distributed": "true",
"zookeeper.session.timeout.localHBaseCluster": "20000",
"hbase.client.scanner.caching": "10000",
"phoenix.sequence.saltBuckets": "2",