You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2014/12/01 21:03:40 UTC
[12/22] ambari git commit: AMBARI-5707. Renaming a module. (swagle)
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/NODEMANAGER.dat
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/NODEMANAGER.dat b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/NODEMANAGER.dat
deleted file mode 100644
index 239b3d4..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/NODEMANAGER.dat
+++ /dev/null
@@ -1,67 +0,0 @@
-jvm.JvmMetrics.GcCount
-jvm.JvmMetrics.GcCountCopy
-jvm.JvmMetrics.GcCountMarkSweepCompact
-jvm.JvmMetrics.GcTimeMillis
-jvm.JvmMetrics.GcTimeMillisCopy
-jvm.JvmMetrics.GcTimeMillisMarkSweepCompact
-jvm.JvmMetrics.LogError
-jvm.JvmMetrics.LogFatal
-jvm.JvmMetrics.LogInfo
-jvm.JvmMetrics.LogWarn
-jvm.JvmMetrics.MemHeapCommittedM
-jvm.JvmMetrics.MemHeapMaxM
-jvm.JvmMetrics.MemHeapUsedM
-jvm.JvmMetrics.MemMaxM
-jvm.JvmMetrics.MemNonHeapCommittedM
-jvm.JvmMetrics.MemNonHeapMaxM
-jvm.JvmMetrics.MemNonHeapUsedM
-jvm.JvmMetrics.ThreadsBlocked
-jvm.JvmMetrics.ThreadsNew
-jvm.JvmMetrics.ThreadsRunnable
-jvm.JvmMetrics.ThreadsTerminated
-jvm.JvmMetrics.ThreadsTimedWaiting
-jvm.JvmMetrics.ThreadsWaiting
-mapred.ShuffleMetrics.ShuffleConnections
-mapred.ShuffleMetrics.ShuffleOutputBytes
-mapred.ShuffleMetrics.ShuffleOutputsFailed
-mapred.ShuffleMetrics.ShuffleOutputsOK
-metricssystem.MetricsSystem.DroppedPubAll
-metricssystem.MetricsSystem.NumActiveSinks
-metricssystem.MetricsSystem.NumActiveSources
-metricssystem.MetricsSystem.NumAllSinks
-metricssystem.MetricsSystem.NumAllSources
-metricssystem.MetricsSystem.PublishAvgTime
-metricssystem.MetricsSystem.PublishNumOps
-metricssystem.MetricsSystem.Sink_timelineAvgTime
-metricssystem.MetricsSystem.Sink_timelineDropped
-metricssystem.MetricsSystem.Sink_timelineNumOps
-metricssystem.MetricsSystem.Sink_timelineQsize
-metricssystem.MetricsSystem.SnapshotAvgTime
-metricssystem.MetricsSystem.SnapshotNumOps
-rpc.rpc.CallQueueLength
-rpc.rpc.NumOpenConnections
-rpc.rpc.ReceivedBytes
-rpc.rpc.RpcAuthenticationFailures
-rpc.rpc.RpcAuthenticationSuccesses
-rpc.rpc.RpcAuthorizationFailures
-rpc.rpc.RpcAuthorizationSuccesses
-rpc.rpc.RpcProcessingTimeAvgTime
-rpc.rpc.RpcProcessingTimeNumOps
-rpc.rpc.RpcQueueTimeAvgTime
-rpc.rpc.RpcQueueTimeNumOps
-rpc.rpc.SentBytes
-ugi.UgiMetrics.GetGroupsAvgTime
-ugi.UgiMetrics.GetGroupsNumOps
-ugi.UgiMetrics.LoginFailureAvgTime
-ugi.UgiMetrics.LoginFailureNumOps
-ugi.UgiMetrics.LoginSuccessAvgTime
-ugi.UgiMetrics.LoginSuccessNumOps
-yarn.NodeManagerMetrics.AllocatedContainers
-yarn.NodeManagerMetrics.AllocatedGB
-yarn.NodeManagerMetrics.AvailableGB
-yarn.NodeManagerMetrics.ContainersCompleted
-yarn.NodeManagerMetrics.ContainersFailed
-yarn.NodeManagerMetrics.ContainersIniting
-yarn.NodeManagerMetrics.ContainersKilled
-yarn.NodeManagerMetrics.ContainersLaunched
-yarn.NodeManagerMetrics.ContainersRunning
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/RESOURCEMANAGER.dat
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/RESOURCEMANAGER.dat b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/RESOURCEMANAGER.dat
deleted file mode 100644
index ec698db..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/RESOURCEMANAGER.dat
+++ /dev/null
@@ -1,88 +0,0 @@
-jvm.JvmMetrics.GcCount
-jvm.JvmMetrics.GcCountPS MarkSweep
-jvm.JvmMetrics.GcCountPS Scavenge
-jvm.JvmMetrics.GcTimeMillis
-jvm.JvmMetrics.GcTimeMillisPS MarkSweep
-jvm.JvmMetrics.GcTimeMillisPS Scavenge
-jvm.JvmMetrics.LogError
-jvm.JvmMetrics.LogFatal
-jvm.JvmMetrics.LogInfo
-jvm.JvmMetrics.LogWarn
-jvm.JvmMetrics.MemHeapCommittedM
-jvm.JvmMetrics.MemHeapMaxM
-jvm.JvmMetrics.MemHeapUsedM
-jvm.JvmMetrics.MemMaxM
-jvm.JvmMetrics.MemNonHeapCommittedM
-jvm.JvmMetrics.MemNonHeapMaxM
-jvm.JvmMetrics.MemNonHeapUsedM
-jvm.JvmMetrics.ThreadsBlocked
-jvm.JvmMetrics.ThreadsNew
-jvm.JvmMetrics.ThreadsRunnable
-jvm.JvmMetrics.ThreadsTerminated
-jvm.JvmMetrics.ThreadsTimedWaiting
-jvm.JvmMetrics.ThreadsWaiting
-metricssystem.MetricsSystem.DroppedPubAll
-metricssystem.MetricsSystem.NumActiveSinks
-metricssystem.MetricsSystem.NumActiveSources
-metricssystem.MetricsSystem.NumAllSinks
-metricssystem.MetricsSystem.NumAllSources
-metricssystem.MetricsSystem.PublishAvgTime
-metricssystem.MetricsSystem.PublishNumOps
-metricssystem.MetricsSystem.Sink_timelineAvgTime
-metricssystem.MetricsSystem.Sink_timelineDropped
-metricssystem.MetricsSystem.Sink_timelineNumOps
-metricssystem.MetricsSystem.Sink_timelineQsize
-metricssystem.MetricsSystem.SnapshotAvgTime
-metricssystem.MetricsSystem.SnapshotNumOps
-rpc.rpc.CallQueueLength
-rpc.rpc.NumOpenConnections
-rpc.rpc.ReceivedBytes
-rpc.rpc.RpcAuthenticationFailures
-rpc.rpc.RpcAuthenticationSuccesses
-rpc.rpc.RpcAuthorizationFailures
-rpc.rpc.RpcAuthorizationSuccesses
-rpc.rpc.RpcProcessingTimeAvgTime
-rpc.rpc.RpcProcessingTimeNumOps
-rpc.rpc.RpcQueueTimeAvgTime
-rpc.rpc.RpcQueueTimeNumOps
-rpc.rpc.SentBytes
-rpcdetailed.rpcdetailed.NodeHeartbeatAvgTime
-rpcdetailed.rpcdetailed.NodeHeartbeatNumOps
-rpcdetailed.rpcdetailed.RegisterNodeManagerAvgTime
-rpcdetailed.rpcdetailed.RegisterNodeManagerNumOps
-ugi.UgiMetrics.GetGroupsAvgTime
-ugi.UgiMetrics.GetGroupsNumOps
-ugi.UgiMetrics.LoginFailureAvgTime
-ugi.UgiMetrics.LoginFailureNumOps
-ugi.UgiMetrics.LoginSuccessAvgTime
-ugi.UgiMetrics.LoginSuccessNumOps
-yarn.ClusterMetrics.NumActiveNMs
-yarn.ClusterMetrics.NumDecommissionedNMs
-yarn.ClusterMetrics.NumLostNMs
-yarn.ClusterMetrics.NumRebootedNMs
-yarn.ClusterMetrics.NumUnhealthyNMs
-yarn.QueueMetrics.ActiveApplications
-yarn.QueueMetrics.ActiveUsers
-yarn.QueueMetrics.AggregateContainersAllocated
-yarn.QueueMetrics.AggregateContainersReleased
-yarn.QueueMetrics.AllocatedContainers
-yarn.QueueMetrics.AllocatedMB
-yarn.QueueMetrics.AllocatedVCores
-yarn.QueueMetrics.AppsCompleted
-yarn.QueueMetrics.AppsFailed
-yarn.QueueMetrics.AppsKilled
-yarn.QueueMetrics.AppsPending
-yarn.QueueMetrics.AppsRunning
-yarn.QueueMetrics.AppsSubmitted
-yarn.QueueMetrics.AvailableMB
-yarn.QueueMetrics.AvailableVCores
-yarn.QueueMetrics.PendingContainers
-yarn.QueueMetrics.PendingMB
-yarn.QueueMetrics.PendingVCores
-yarn.QueueMetrics.ReservedContainers
-yarn.QueueMetrics.ReservedMB
-yarn.QueueMetrics.ReservedVCores
-yarn.QueueMetrics.running_0
-yarn.QueueMetrics.running_1440
-yarn.QueueMetrics.running_300
-yarn.QueueMetrics.running_60
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/SLAVE_HBASE.dat
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/SLAVE_HBASE.dat b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/SLAVE_HBASE.dat
deleted file mode 100644
index 38b870f..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/metrics_def/SLAVE_HBASE.dat
+++ /dev/null
@@ -1,178 +0,0 @@
-ipc.IPC.authenticationFailures
-ipc.IPC.authenticationSuccesses
-ipc.IPC.authorizationFailures
-ipc.IPC.authorizationSuccesses
-ipc.IPC.numCallsInGeneralQueue
-ipc.IPC.numCallsInPriorityQueue
-ipc.IPC.numCallsInReplicationQueue
-ipc.IPC.numOpenConnections
-ipc.IPC.ProcessCallTime_75th_percentile
-ipc.IPC.ProcessCallTime_95th_percentile
-ipc.IPC.ProcessCallTime_99th_percentile
-ipc.IPC.ProcessCallTime_max
-ipc.IPC.ProcessCallTime_mean
-ipc.IPC.ProcessCallTime_median
-ipc.IPC.ProcessCallTime_min
-ipc.IPC.ProcessCallTime_num_ops
-ipc.IPC.QueueCallTime_75th_percentile
-ipc.IPC.QueueCallTime_95th_percentile
-ipc.IPC.QueueCallTime_99th_percentile
-ipc.IPC.QueueCallTime_max
-ipc.IPC.QueueCallTime_mean
-ipc.IPC.QueueCallTime_median
-ipc.IPC.QueueCallTime_min
-ipc.IPC.QueueCallTime_num_ops
-ipc.IPC.queueSize
-ipc.IPC.receivedBytes
-ipc.IPC.sentBytes
-jvm.JvmMetrics.GcCount
-jvm.JvmMetrics.GcCountConcurrentMarkSweep
-jvm.JvmMetrics.GcCountCopy
-jvm.JvmMetrics.GcTimeMillis
-jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep
-jvm.JvmMetrics.GcTimeMillisCopy
-jvm.JvmMetrics.LogError
-jvm.JvmMetrics.LogFatal
-jvm.JvmMetrics.LogInfo
-jvm.JvmMetrics.LogWarn
-jvm.JvmMetrics.MemHeapCommittedM
-jvm.JvmMetrics.MemHeapMaxM
-jvm.JvmMetrics.MemHeapUsedM
-jvm.JvmMetrics.MemMaxM
-jvm.JvmMetrics.MemNonHeapCommittedM
-jvm.JvmMetrics.MemNonHeapMaxM
-jvm.JvmMetrics.MemNonHeapUsedM
-jvm.JvmMetrics.ThreadsBlocked
-jvm.JvmMetrics.ThreadsNew
-jvm.JvmMetrics.ThreadsRunnable
-jvm.JvmMetrics.ThreadsTerminated
-jvm.JvmMetrics.ThreadsTimedWaiting
-jvm.JvmMetrics.ThreadsWaiting
-metricssystem.MetricsSystem.DroppedPubAll
-metricssystem.MetricsSystem.NumActiveSinks
-metricssystem.MetricsSystem.NumActiveSources
-metricssystem.MetricsSystem.NumAllSinks
-metricssystem.MetricsSystem.NumAllSources
-metricssystem.MetricsSystem.PublishAvgTime
-metricssystem.MetricsSystem.PublishNumOps
-metricssystem.MetricsSystem.Sink_timelineAvgTime
-metricssystem.MetricsSystem.Sink_timelineDropped
-metricssystem.MetricsSystem.Sink_timelineNumOps
-metricssystem.MetricsSystem.Sink_timelineQsize
-metricssystem.MetricsSystem.SnapshotAvgTime
-metricssystem.MetricsSystem.SnapshotNumOps
-regionserver.Server.Append_75th_percentile
-regionserver.Server.Append_95th_percentile
-regionserver.Server.Append_99th_percentile
-regionserver.Server.Append_max
-regionserver.Server.Append_mean
-regionserver.Server.Append_median
-regionserver.Server.Append_min
-regionserver.Server.Append_num_ops
-regionserver.Server.blockCacheCount
-regionserver.Server.blockCacheEvictionCount
-regionserver.Server.blockCacheExpressHitPercent
-regionserver.Server.blockCacheFreeSize
-regionserver.Server.blockCacheHitCount
-regionserver.Server.blockCacheMissCount
-regionserver.Server.blockCacheSize
-regionserver.Server.blockCountHitPercent
-regionserver.Server.checkMutateFailedCount
-regionserver.Server.checkMutatePassedCount
-regionserver.Server.compactionQueueLength
-regionserver.Server.Delete_75th_percentile
-regionserver.Server.Delete_95th_percentile
-regionserver.Server.Delete_99th_percentile
-regionserver.Server.Delete_max
-regionserver.Server.Delete_mean
-regionserver.Server.Delete_median
-regionserver.Server.Delete_min
-regionserver.Server.Delete_num_ops
-regionserver.Server.flushQueueLength
-regionserver.Server.Get_75th_percentile
-regionserver.Server.Get_95th_percentile
-regionserver.Server.Get_99th_percentile
-regionserver.Server.Get_max
-regionserver.Server.Get_mean
-regionserver.Server.Get_median
-regionserver.Server.Get_min
-regionserver.Server.Get_num_ops
-regionserver.Server.hlogFileCount
-regionserver.Server.hlogFileSize
-regionserver.Server.Increment_75th_percentile
-regionserver.Server.Increment_95th_percentile
-regionserver.Server.Increment_99th_percentile
-regionserver.Server.Increment_max
-regionserver.Server.Increment_mean
-regionserver.Server.Increment_median
-regionserver.Server.Increment_min
-regionserver.Server.Increment_num_ops
-regionserver.Server.memStoreSize
-regionserver.Server.Mutate_75th_percentile
-regionserver.Server.Mutate_95th_percentile
-regionserver.Server.Mutate_99th_percentile
-regionserver.Server.Mutate_max
-regionserver.Server.Mutate_mean
-regionserver.Server.Mutate_median
-regionserver.Server.Mutate_min
-regionserver.Server.Mutate_num_ops
-regionserver.Server.mutationsWithoutWALCount
-regionserver.Server.mutationsWithoutWALSize
-regionserver.Server.percentFilesLocal
-regionserver.Server.readRequestCount
-regionserver.Server.regionCount
-regionserver.Server.regionServerStartTime
-regionserver.Server.Replay_75th_percentile
-regionserver.Server.Replay_95th_percentile
-regionserver.Server.Replay_99th_percentile
-regionserver.Server.Replay_max
-regionserver.Server.Replay_mean
-regionserver.Server.Replay_median
-regionserver.Server.Replay_min
-regionserver.Server.Replay_num_ops
-regionserver.Server.slowAppendCount
-regionserver.Server.slowDeleteCount
-regionserver.Server.slowGetCount
-regionserver.Server.slowIncrementCount
-regionserver.Server.slowPutCount
-regionserver.Server.staticBloomSize
-regionserver.Server.staticIndexSize
-regionserver.Server.storeCount
-regionserver.Server.storeFileCount
-regionserver.Server.storeFileIndexSize
-regionserver.Server.storeFileSize
-regionserver.Server.totalRequestCount
-regionserver.Server.updatesBlockedTime
-regionserver.Server.writeRequestCount
-regionserver.WAL.appendCount
-regionserver.WAL.AppendSize_75th_percentile
-regionserver.WAL.AppendSize_95th_percentile
-regionserver.WAL.AppendSize_99th_percentile
-regionserver.WAL.AppendSize_max
-regionserver.WAL.AppendSize_mean
-regionserver.WAL.AppendSize_median
-regionserver.WAL.AppendSize_min
-regionserver.WAL.AppendSize_num_ops
-regionserver.WAL.AppendTime_75th_percentile
-regionserver.WAL.AppendTime_95th_percentile
-regionserver.WAL.AppendTime_99th_percentile
-regionserver.WAL.AppendTime_max
-regionserver.WAL.AppendTime_mean
-regionserver.WAL.AppendTime_median
-regionserver.WAL.AppendTime_min
-regionserver.WAL.AppendTime_num_ops
-regionserver.WAL.slowAppendCount
-regionserver.WAL.SyncTime_75th_percentile
-regionserver.WAL.SyncTime_95th_percentile
-regionserver.WAL.SyncTime_99th_percentile
-regionserver.WAL.SyncTime_max
-regionserver.WAL.SyncTime_mean
-regionserver.WAL.SyncTime_median
-regionserver.WAL.SyncTime_min
-regionserver.WAL.SyncTime_num_ops
-ugi.UgiMetrics.GetGroupsAvgTime
-ugi.UgiMetrics.GetGroupsNumOps
-ugi.UgiMetrics.LoginFailureAvgTime
-ugi.UgiMetrics.LoginFailureNumOps
-ugi.UgiMetrics.LoginSuccessAvgTime
-ugi.UgiMetrics.LoginSuccessNumOps
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/ams_query.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/ams_query.py b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/ams_query.py
deleted file mode 100644
index d51357a..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/ams_query.py
+++ /dev/null
@@ -1,209 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import urllib2
-import signal
-import sys
-import optparse
-import time
-
-# http://162.216.148.45:8188/ws/v1/timeline/metrics?
-# metricNames=rpc.rpc.RpcAuthenticationSuccesses
-# &appId=nodemanager&hostname=local.0&startTime=1414152029&endTime=1414155629
-
-AMS_URL = "http://{0}:8188/ws/v1/timeline/metrics?metricNames={1}&appid={" \
- "2}&hostname={3}"
-
-# in fact it can be list automatically generated from ambari
-# UI queries
-host_metrics = {
- 'cpu': ['cpu_user', 'cpu_wio', 'cpu_nice', 'cpu_aidle', 'cpu_system', 'cpu_idle'],
- 'disk': ['disk_total', 'disk_free'],
- 'load': ['load_one', 'load_fifteen', 'load_five'],
- 'mem': ['swap_free', 'mem_shared', 'mem_free', 'mem_cached', 'mem_buffers'],
- 'network': ['bytes_in', 'bytes_out', 'pkts_in', 'pkts_out'],
- 'process': ['proc_total', 'proc_run']
-}
-
-# HDFS_SERVICE
-namenode_metrics = {
- 'dfs.Capacity': ['dfs.FSNamesystem.CapacityRemainingGB',
- 'dfs.FSNamesystem.CapacityUsedGB',
- 'dfs.FSNamesystem.CapacityTotalGB'],
- 'dfs.Replication': ['dfs.FSNamesystem.PendingReplicationBlocks',
- 'dfs.FSNamesystem.UnderReplicatedBlocks'],
- 'dfs.File': ['dfs.namenode.FileInfoOps', 'dfs.namenode.CreateFileOps'],
- 'jvm.gc': ['jvm.JvmMetrics.GcTimeMillis'],
- 'jvm.mem': ['jvm.JvmMetrics.MemNonHeapUsedM',
- 'jvm.JvmMetrics.MemNonHeapCommittedM',
- 'jvm.JvmMetrics.MemHeapUsedM',
- 'jvm.JvmMetrics.MemHeapCommittedM'],
- 'jvm.thread': ['jvm.JvmMetrics.ThreadsRunnable',
- 'jvm.JvmMetrics.ThreadsBlocked',
- 'jvm.JvmMetrics.ThreadsWaiting',
- 'jvm.JvmMetrics.ThreadsTimedWaiting'],
- 'rpc': ['rpc.rpc.RpcQueueTimeAvgTime']
-}
-
-all_metrics = {
- 'HOST': host_metrics,
- 'namenode': namenode_metrics
-}
-
-all_metrics_times = {}
-
-
-# hostnames = ['EPPLKRAW0101.0'] # 'local.0'
-# metrics_test_host = '162.216.150.247' # metricstest-100
-# metrics_test_host = '162.216.148.45' # br-3
-# start_time = int(time.time()) # 1414425208
-
-
-def main(argv=None):
- # Allow Ctrl-C
- signal.signal(signal.SIGINT, signal_handler)
-
- parser = optparse.OptionParser()
-
- parser.add_option("-H", "--host", dest="host",
- help="AMS host")
- parser.add_option("-t", "--starttime", dest="start_time_secs",
- default=int(time.time()),
- help="start time in seconds, default value is current time")
- parser.add_option("-n", "--nodes", dest="node_names",
- help="nodes from cluster, used as a param to query for")
- (options, args) = parser.parse_args()
-
- if options.host is None:
- print "AMS host name is required (--host or -h)"
- exit(-1)
-
- if options.node_names is None:
- print "cluster nodes are required (--nodes or -n)"
- exit(3)
-
- global start_time_secs, metrics_test_host, hostnames
-
- metrics_test_host = options.host
- start_time_secs = int(options.start_time_secs)
- hostnames = [options.node_names]
-
- while True:
- run()
- time.sleep(15)
- start_time_secs += 15
-
-
-def signal_handler(signal, frame):
- print('Exiting, Ctrl+C press detected!')
- print_all_metrics(all_metrics_times)
- sys.exit(0)
-
-
-def run():
- hostname = ','.join(hostnames)
- qs = QuerySender(metrics_test_host, True)
- for metric_name in all_metrics:
- print
- print 'Querying for ' + metric_name + ' metrics'
- current_time_secs = start_time_secs
- qs.query_all_app_metrics(hostname, metric_name,
- all_metrics[metric_name],
- current_time_secs)
-
-
-def add_query_metrics_for_app_id(app_id, metric_timing):
- if not app_id in all_metrics_times:
- all_metrics_times[app_id] = []
- all_metrics_times[app_id].append(metric_timing)
-
-
-def print_all_metrics(metrics):
- print 'Metrics Summary'
- for app_id in sorted(metrics):
- first = True
- for single_query_metrics in metrics[app_id]:
- print_app_metrics(app_id, single_query_metrics, first)
- first = False
-
-
-def print_app_metrics(app_id, metric_timing, header=False):
- #header
- if header:
- print app_id + ': ' + ','.join(sorted(metric_timing.keys()))
- #vals
- print app_id + ':',
- for key in sorted(metric_timing):
- print '%.3f,' % metric_timing[key],
- print
-
-
-class QuerySender:
- def __init__(self, metrics_address, print_responses=False):
- self.metrics_address = metrics_address
- self.print_responses = print_responses
-
- def query_all_app_metrics(self, hostname, app_id, metrics, current_time_secs):
- metric_timing = {}
- for key in metrics:
- print 'Getting metrics for', key
- query_time = time.time()
-
- metric_names = ','.join(metrics[key])
- self.query(hostname, app_id, metric_names, current_time_secs)
- query_time_elapsed = time.time() - query_time
-
- print 'Query for "%s" took %s' % (key, query_time_elapsed)
- metric_timing[key] = query_time_elapsed
-
- add_query_metrics_for_app_id(app_id, metric_timing)
- if self.print_responses:
- print_app_metrics(app_id, metric_timing)
-
- def query(self, hostname, app_id, metric_names, current_time_secs):
- url = self.create_url(hostname, metric_names, app_id, current_time_secs)
- print url
- response = self.send(url)
- if self.print_responses:
- print response
- pass
-
- def send(self, url):
- request = urllib2.Request(url)
- try:
- response = urllib2.urlopen(request, timeout=int(30))
- response = response.read()
- return response
-
- except urllib2.URLError as e:
- print e.reason
-
- def create_url(self, hostname, metric_names, app_id, current_time_secs):
- server = AMS_URL.format(self.metrics_address,
- metric_names,
- app_id,
- hostname)
- t = current_time_secs
- server += '&startTime=%s&endTime=%s' % (t, t + 3600)
- return server
-
-
-if __name__ == '__main__':
- main()
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/start.sh
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/start.sh b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/start.sh
deleted file mode 100644
index 0507af1..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/start.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# set -x
-cd "$(dirname "$0")"
-
-METRICS_HOST=$1
-HOST_COUNT=$2
-
-echo $$ > sim_pid
-cat sim_pid
-#HOMEDIR
-exec java -jar ../lib/ambari-metrics/ambari-metrics-hadoop-timelineservice-simulator*.jar -h `hostname -f` -n ${HOST_COUNT} -m ${METRICS_HOST} -c 15000 -s 60000
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/start_slaves.sh
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/start_slaves.sh b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/start_slaves.sh
deleted file mode 100644
index e1e51c8..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/start_slaves.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-## set -x
-
-SLAVES=$1
-METRICS_HOST=$2
-HOST_COUNT=$3
-LOADSIM_HOMEDIR="LoadSimulator-1.0-SNAPSHOT"
-
-pdsh -w ${SLAVES} "$LOADSIM_HOMEDIR/start.sh ${METRICS_HOST} ${HOST_COUNT}'"
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/status_slaves.sh
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/status_slaves.sh b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/status_slaves.sh
deleted file mode 100644
index 79787fd..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/status_slaves.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-## set -x
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/stop.sh
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/stop.sh b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/stop.sh
deleted file mode 100644
index 2220861..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/stop.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# set -x
-cd "$(dirname "$0")"
-read PID <./sim_pid
-
-FOUND_PID=`ps aux | grep simulator | grep $PID`
-if [ -z "${FOUND_PID}" ]
-then
- echo "process simulator not fund under pid $PID"
-else
- echo "process simulator running as $PID, killing"
- kill ${PID}
-fi
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/stop_slaves.sh
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/stop_slaves.sh b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/stop_slaves.sh
deleted file mode 100644
index 7cb567a..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/resources/scripts/stop_slaves.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# set -x
-
-SLAVES=$1
-LOADSIM_HOMEDIR="LoadSimulator-1.0-SNAPSHOT"
-
-pdsh -w ${SLAVES} "$LOADSIM_HOMEDIR/stop.sh"
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/ams-site.xml b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/ams-site.xml
deleted file mode 100644
index b08609d..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/ams-site.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<!-- Empty site, all defaults should be initialized in the code -->
-<configuration>
- <property>
- <name>test</name>
- <value>testReady</value>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/hadoop-policy.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/hadoop-policy.xml b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/hadoop-policy.xml
deleted file mode 100644
index 41bde16..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/hadoop-policy.xml
+++ /dev/null
@@ -1,134 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration supports_final="true">
- <property>
- <name>security.client.protocol.acl</name>
- <value>*</value>
- <description>ACL for ClientProtocol, which is used by user code
- via the DistributedFileSystem.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.client.datanode.protocol.acl</name>
- <value>*</value>
- <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
- for block recovery.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.datanode.protocol.acl</name>
- <value>*</value>
- <description>ACL for DatanodeProtocol, which is used by datanodes to
- communicate with the namenode.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.inter.datanode.protocol.acl</name>
- <value>*</value>
- <description>ACL for InterDatanodeProtocol, the inter-datanode protocol
- for updating generation timestamp.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.namenode.protocol.acl</name>
- <value>*</value>
- <description>ACL for NamenodeProtocol, the protocol used by the secondary
- namenode to communicate with the namenode.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.inter.tracker.protocol.acl</name>
- <value>*</value>
- <description>ACL for InterTrackerProtocol, used by the tasktrackers to
- communicate with the jobtracker.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.job.client.protocol.acl</name>
- <value>*</value>
- <description>ACL for JobSubmissionProtocol, used by job clients to
- communciate with the jobtracker for job submission, querying job status etc.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.job.task.protocol.acl</name>
- <value>*</value>
- <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
- tasks to communicate with the parent tasktracker.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.admin.operations.protocol.acl</name>
- <value>hadoop</value>
- <description>ACL for AdminOperationsProtocol. Used for admin commands.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.refresh.usertogroups.mappings.protocol.acl</name>
- <value>hadoop</value>
- <description>ACL for RefreshUserMappingsProtocol. Used to refresh
- users mappings. The ACL is a comma-separated list of user and
- group names. The user and group list is separated by a blank. For
- e.g. "alice,bob users,wheel". A special value of "*" means all
- users are allowed.</description>
- </property>
-
-<property>
- <name>security.refresh.policy.protocol.acl</name>
- <value>hadoop</value>
- <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
- dfsadmin and mradmin commands to refresh the security policy in-effect.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/hbase-site.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/hbase-site.xml b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/hbase-site.xml
deleted file mode 100644
index c453900..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/conf/hbase-site.xml
+++ /dev/null
@@ -1,230 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
- <property>
- <name>hbase.defaults.for.version.skip</name>
- <value>true</value>
- </property>
- <property>
- <name>hbase.rootdir</name>
- <value>file:///var/lib/hbase</value>
- <description>
- AMS service uses HBase as default storage backend. Set the rootdir for
- HBase to either local filesystem path if using AMS in embedded mode or
- to a HDFS dir, example: hdfs://namenode.example.org:9000/hbase. By
- default HBase writes into /tmp. Change this configuration else all data
- will be lost on machine restart.
- </description>
- </property>
- <property>
- <name>hbase.tmp.dir</name>
- <value>/tmp</value>
- <description>
- Temporary directory on the local filesystem.
- Change this setting to point to a location more permanent
- than '/tmp' (The '/tmp' directory is often cleared on
- machine restart).
- </description>
- </property>
- <property>
- <name>hbase.local.dir</name>
- <value>${hbase.tmp.dir}/local</value>
- <description>Directory on the local filesystem to be used as a local storage
- </description>
- </property>
- <property>
- <name>hbase.cluster.distributed</name>
- <value>true</value>
- <description>
- The mode the cluster will be in. Possible values are false for
- standalone mode and true for distributed mode. If false, startup will run
- all HBase and ZooKeeper daemons together in the one JVM.
- </description>
- </property>
- <property>
- <name>hbase.master.wait.on.regionservers.mintostart</name>
- <value>1</value>
- <description>
- Ensure that HBase Master waits for # many region server to start.
- </description>
- </property>
- <property>
- <name>hbase.zookeeper.quorum</name>
- <value>localhost</value>
- <description>Comma separated list of servers in the ZooKeeper Quorum.
- For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
- By default this is set to localhost for local and pseudo-distributed modes
- of operation. For a fully-distributed setup, this should be set to a full
- list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
- this is the list of servers which we will start/stop ZooKeeper on.
- </description>
- </property>
- <property>
- <name>hbase.master.info.bindAddress</name>
- <value>0.0.0.0</value>
- <description>The bind address for the HBase Master web UI</description>
- </property>
- <property>
- <name>hbase.master.info.port</name>
- <value>90010</value>
- <description>The port for the HBase Master web UI.</description>
- </property>
- <property>
- <name>hbase.regionserver.info.port</name>
- <value>90030</value>
- <description>The port for the HBase RegionServer web UI.</description>
- </property>
- <property>
- <name>hbase.hregion.majorcompaction</name>
- <value>0</value>
- <description>
- The time (in milliseconds) between 'major' compactions of all
- HStoreFiles in a region.
- 0 to disable automated major compactions.
- </description>
- </property>
- <property>
- <name>phoenix.query.spoolThresholdBytes</name>
- <value>12582912</value>
- <description>
- Threshold size in bytes after which results from parallelly executed
- query results are spooled to disk. Default is 20 mb.
- </description>
- </property>
- <property>
- <name>hbase.zookeeper.property.dataDir</name>
- <value>${hbase.tmp.dir}/zookeeper</value>
- <description>
- Property from ZooKeeper's config zoo.cfg.
- The directory where the snapshot is stored.
- </description>
- </property>
- <property>
- <name>hbase.client.scanner.caching</name>
- <value>10000</value>
- <description>
- Number of rows that will be fetched when calling next on a scanner
- if it is not served from (local, client) memory.
- </description>
- </property>
- <property>
- <name>hfile.block.cache.size</name>
- <value>0.3</value>
- <description>
- Percentage of maximum heap (-Xmx setting) to allocate to block cache
- used by a StoreFile. Default of 0.4 means allocate 40%.
- </description>
- </property>
- <property>
- <name>hbase.regionserver.global.memstore.upperLimit</name>
- <value>0.5</value>
- <description>
- Maximum size of all memstores in a region server before new
- updates are blocked and flushes are forced. Defaults to 40% of heap
- </description>
- </property>
- <property>
- <name>hbase.regionserver.global.memstore.lowerLimit</name>
- <value>0.4</value>
- <description>
- When memstores are being forced to flush to make room in
- memory, keep flushing until we hit this mark. Defaults to 35% of heap.
- This value equal to hbase.regionserver.global.memstore.upperLimit causes
- the minimum possible flushing to occur when updates are blocked due to
- memstore limiting.
- </description>
- </property>
- <property>
- <name>phoenix.groupby.maxCacheSize</name>
- <value>307200000</value>
- <description>
- Size in bytes of pages cached during GROUP BY spilling. Default is 100Mb.
- </description>
- </property>
- <property>
- <name>hbase.hregion.memstore.block.multiplier</name>
- <value>4</value>
- <description>
- Block updates if memstore has hbase.hregion.memstore.block.multiplier
- times hbase.hregion.memstore.flush.size bytes. Useful preventing runaway
- memstore during spikes in update traffic.
- </description>
- </property>
- <property>
- <name>hbase.hstore.flusher.count</name>
- <value>2</value>
- <description>
- The number of flush threads. With fewer threads, the MemStore flushes
- will be queued. With more threads, the flushes will be executed in parallel,
- increasing the load on HDFS, and potentially causing more compactions.
- </description>
- </property>
- <property>
- <name>phoenix.query.timeoutMs</name>
- <value>1200000</value>
- <description>
- Number of milliseconds after which a query will timeout on the client.
- Default is 10 min.
- </description>
- </property>
- <property>
- <name>hbase.client.scanner.timeout.period</name>
- <value>900000</value>
- <description>
- Client scanner lease period in milliseconds.
- </description>
- </property>
- <property>
- <name>hbase.regionserver.thread.compaction.large</name>
- <value>2</value>
- <description>
- Configuration key for the large compaction threads.
- </description>
- </property>
- <property>
- <name>hbase.regionserver.thread.compaction.small</name>
- <value>3</value>
- <description>
- Configuration key for the small compaction threads.
- </description>
- </property>
- <property>
- <name>hbase.hstore.blockingStoreFiles</name>
- <value>200</value>
- <description>
- If more than this number of StoreFiles exist in any one Store
- (one StoreFile is written per flush of MemStore), updates are blocked for
- this region until a compaction is completed, or until
- hbase.hstore.blockingWaitTime has been exceeded.
- </description>
- </property>
- <property>
- <name>hbase.hregion.memstore.flush.size</name>
- <value>134217728</value>
- <description>
- Memstore will be flushed to disk if size of the memstore exceeds this
- number of bytes. Value is checked by a thread that runs every
- hbase.server.thread.wakefrequency.
- </description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
deleted file mode 100644
index c41b8a7..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice;
-
-import java.io.IOException;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
-
-public class ApplicationHistoryStoreTestUtils {
-
- protected ApplicationHistoryStore store;
-
- protected void writeApplicationStartData(ApplicationId appId)
- throws IOException {
- store.applicationStarted(ApplicationStartData.newInstance(appId,
- appId.toString(), "test type", "test queue", "test user", 0, 0));
- }
-
- protected void writeApplicationFinishData(ApplicationId appId)
- throws IOException {
- store.applicationFinished(ApplicationFinishData.newInstance(appId, 0,
- appId.toString(), FinalApplicationStatus.UNDEFINED,
- YarnApplicationState.FINISHED));
- }
-
- protected void writeApplicationAttemptStartData(
- ApplicationAttemptId appAttemptId) throws IOException {
- store.applicationAttemptStarted(ApplicationAttemptStartData.newInstance(
- appAttemptId, appAttemptId.toString(), 0,
- ContainerId.newInstance(appAttemptId, 1)));
- }
-
- protected void writeApplicationAttemptFinishData(
- ApplicationAttemptId appAttemptId) throws IOException {
- store.applicationAttemptFinished(ApplicationAttemptFinishData.newInstance(
- appAttemptId, appAttemptId.toString(), "test tracking url",
- FinalApplicationStatus.UNDEFINED, YarnApplicationAttemptState.FINISHED));
- }
-
- protected void writeContainerStartData(ContainerId containerId)
- throws IOException {
- store.containerStarted(ContainerStartData.newInstance(containerId,
- Resource.newInstance(0, 0), NodeId.newInstance("localhost", 0),
- Priority.newInstance(containerId.getId()), 0));
- }
-
- protected void writeContainerFinishData(ContainerId containerId)
- throws IOException {
- store.containerFinished(ContainerFinishData.newInstance(containerId, 0,
- containerId.toString(), 0, ContainerState.COMPLETE));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
deleted file mode 100644
index 6b06918..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice;
-
-import java.io.IOException;
-import java.util.List;
-
-import junit.framework.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerReport;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-// Timeline service client support is not enabled for AMS
-@Ignore
-public class TestApplicationHistoryClientService extends
- ApplicationHistoryStoreTestUtils {
-
- ApplicationHistoryServer historyServer = null;
- String expectedLogUrl = null;
-
- @Before
- public void setup() {
- historyServer = new ApplicationHistoryServer();
- Configuration config = new YarnConfiguration();
- expectedLogUrl = WebAppUtils.getHttpSchemePrefix(config) +
- WebAppUtils.getAHSWebAppURLWithoutScheme(config) +
- "/applicationhistory/logs/localhost:0/container_0_0001_01_000001/" +
- "container_0_0001_01_000001/test user";
- config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
- MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
- historyServer.init(config);
- historyServer.start();
- store =
- ((ApplicationHistoryManagerImpl) historyServer.getApplicationHistory())
- .getHistoryStore();
- }
-
- @After
- public void tearDown() throws Exception {
- historyServer.stop();
- }
-
- @Test
- public void testApplicationReport() throws IOException, YarnException {
- ApplicationId appId = null;
- appId = ApplicationId.newInstance(0, 1);
- writeApplicationStartData(appId);
- writeApplicationFinishData(appId);
- GetApplicationReportRequest request =
- GetApplicationReportRequest.newInstance(appId);
- GetApplicationReportResponse response =
- historyServer.getClientService().getClientHandler()
- .getApplicationReport(request);
- ApplicationReport appReport = response.getApplicationReport();
- Assert.assertNotNull(appReport);
- Assert.assertEquals("application_0_0001", appReport.getApplicationId()
- .toString());
- Assert.assertEquals("test type", appReport.getApplicationType().toString());
- Assert.assertEquals("test queue", appReport.getQueue().toString());
- }
-
- @Test
- public void testApplications() throws IOException, YarnException {
- ApplicationId appId = null;
- appId = ApplicationId.newInstance(0, 1);
- writeApplicationStartData(appId);
- writeApplicationFinishData(appId);
- ApplicationId appId1 = ApplicationId.newInstance(0, 2);
- writeApplicationStartData(appId1);
- writeApplicationFinishData(appId1);
- GetApplicationsRequest request = GetApplicationsRequest.newInstance();
- GetApplicationsResponse response =
- historyServer.getClientService().getClientHandler()
- .getApplications(request);
- List<ApplicationReport> appReport = response.getApplicationList();
- Assert.assertNotNull(appReport);
- Assert.assertEquals(appId, appReport.get(0).getApplicationId());
- Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
- }
-
- @Test
- public void testApplicationAttemptReport() throws IOException, YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 1);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- writeApplicationAttemptStartData(appAttemptId);
- writeApplicationAttemptFinishData(appAttemptId);
- GetApplicationAttemptReportRequest request =
- GetApplicationAttemptReportRequest.newInstance(appAttemptId);
- GetApplicationAttemptReportResponse response =
- historyServer.getClientService().getClientHandler()
- .getApplicationAttemptReport(request);
- ApplicationAttemptReport attemptReport =
- response.getApplicationAttemptReport();
- Assert.assertNotNull(attemptReport);
- Assert.assertEquals("appattempt_0_0001_000001", attemptReport
- .getApplicationAttemptId().toString());
- }
-
- @Test
- public void testApplicationAttempts() throws IOException, YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 1);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- ApplicationAttemptId appAttemptId1 =
- ApplicationAttemptId.newInstance(appId, 2);
- writeApplicationAttemptStartData(appAttemptId);
- writeApplicationAttemptFinishData(appAttemptId);
- writeApplicationAttemptStartData(appAttemptId1);
- writeApplicationAttemptFinishData(appAttemptId1);
- GetApplicationAttemptsRequest request =
- GetApplicationAttemptsRequest.newInstance(appId);
- GetApplicationAttemptsResponse response =
- historyServer.getClientService().getClientHandler()
- .getApplicationAttempts(request);
- List<ApplicationAttemptReport> attemptReports =
- response.getApplicationAttemptList();
- Assert.assertNotNull(attemptReports);
- Assert.assertEquals(appAttemptId, attemptReports.get(0)
- .getApplicationAttemptId());
- Assert.assertEquals(appAttemptId1, attemptReports.get(1)
- .getApplicationAttemptId());
- }
-
- @Test
- public void testContainerReport() throws IOException, YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 1);
- writeApplicationStartData(appId);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
- writeContainerStartData(containerId);
- writeContainerFinishData(containerId);
- writeApplicationFinishData(appId);
- GetContainerReportRequest request =
- GetContainerReportRequest.newInstance(containerId);
- GetContainerReportResponse response =
- historyServer.getClientService().getClientHandler()
- .getContainerReport(request);
- ContainerReport container = response.getContainerReport();
- Assert.assertNotNull(container);
- Assert.assertEquals(containerId, container.getContainerId());
- Assert.assertEquals(expectedLogUrl, container.getLogUrl());
- }
-
- @Test
- public void testContainers() throws IOException, YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 1);
- writeApplicationStartData(appId);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
- ContainerId containerId1 = ContainerId.newInstance(appAttemptId, 2);
- writeContainerStartData(containerId);
- writeContainerFinishData(containerId);
- writeContainerStartData(containerId1);
- writeContainerFinishData(containerId1);
- writeApplicationFinishData(appId);
- GetContainersRequest request =
- GetContainersRequest.newInstance(appAttemptId);
- GetContainersResponse response =
- historyServer.getClientService().getClientHandler()
- .getContainers(request);
- List<ContainerReport> containers = response.getContainerList();
- Assert.assertNotNull(containers);
- Assert.assertEquals(containerId, containers.get(1).getContainerId());
- Assert.assertEquals(containerId1, containers.get(0).getContainerId());
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
deleted file mode 100644
index aad23d9..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-public class TestApplicationHistoryManagerImpl extends
- ApplicationHistoryStoreTestUtils {
- ApplicationHistoryManagerImpl applicationHistoryManagerImpl = null;
-
- @Before
- public void setup() throws Exception {
- Configuration config = new Configuration();
- config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
- MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
- applicationHistoryManagerImpl = new ApplicationHistoryManagerImpl();
- applicationHistoryManagerImpl.init(config);
- applicationHistoryManagerImpl.start();
- store = applicationHistoryManagerImpl.getHistoryStore();
- }
-
- @After
- public void tearDown() throws Exception {
- applicationHistoryManagerImpl.stop();
- }
-
- @Test
- @Ignore
- public void testApplicationReport() throws IOException, YarnException {
- ApplicationId appId = null;
- appId = ApplicationId.newInstance(0, 1);
- writeApplicationStartData(appId);
- writeApplicationFinishData(appId);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- writeApplicationAttemptStartData(appAttemptId);
- writeApplicationAttemptFinishData(appAttemptId);
- ApplicationReport appReport =
- applicationHistoryManagerImpl.getApplication(appId);
- Assert.assertNotNull(appReport);
- Assert.assertEquals(appId, appReport.getApplicationId());
- Assert.assertEquals(appAttemptId,
- appReport.getCurrentApplicationAttemptId());
- Assert.assertEquals(appAttemptId.toString(), appReport.getHost());
- Assert.assertEquals("test type", appReport.getApplicationType().toString());
- Assert.assertEquals("test queue", appReport.getQueue().toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
deleted file mode 100644
index 3720852..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.Service.STATE;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .timeline.DefaultPhoenixDataSource;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .timeline.PhoenixHBaseAccessor;
-import org.apache.zookeeper.ClientCnxn;
-import org.easymock.EasyMock;
-import org.junit.*;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.Statement;
-
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
- .timeline.TimelineMetricConfiguration.METRICS_SITE_CONFIGURATION_FILE;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.*;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.junit.Assert.*;
-import static org.powermock.api.easymock.PowerMock.*;
-import static org.powermock.api.support.membermodification.MemberMatcher.method;
-import static org.powermock.api.support.membermodification.MemberModifier
- .suppress;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({ PhoenixHBaseAccessor.class, UserGroupInformation.class,
- ClientCnxn.class, DefaultPhoenixDataSource.class})
-@PowerMockIgnore( {"javax.management.*"})
-public class TestApplicationHistoryServer {
-
- ApplicationHistoryServer historyServer = null;
- Configuration metricsConf = null;
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Before
- @SuppressWarnings("all")
- public void setup() throws URISyntaxException, IOException {
- folder.create();
- File hbaseSite = folder.newFile("hbase-site.xml");
- File amsSite = folder.newFile("ams-site.xml");
-
- FileUtils.writeStringToFile(hbaseSite, "<configuration>\n" +
- " <property>\n" +
- " <name>hbase.defaults.for.version.skip</name>\n" +
- " <value>true</value>\n" +
- " </property>" +
- " <property> " +
- " <name>hbase.zookeeper.quorum</name>\n" +
- " <value>localhost</value>\n" +
- " </property>" +
- "</configuration>");
-
- FileUtils.writeStringToFile(amsSite, "<configuration>\n" +
- " <property>\n" +
- " <name>test</name>\n" +
- " <value>testReady</value>\n" +
- " </property>\n" +
- " <property>\n" +
- " <name>timeline.metrics.host.aggregator.hourly.disabled</name>\n" +
- " <value>true</value>\n" +
- " <description>\n" +
- " Disable host based hourly aggregations.\n" +
- " </description>\n" +
- " </property>\n" +
- " <property>\n" +
- " <name>timeline.metrics.host.aggregator.minute.disabled</name>\n" +
- " <value>true</value>\n" +
- " <description>\n" +
- " Disable host based minute aggregations.\n" +
- " </description>\n" +
- " </property>\n" +
- " <property>\n" +
- " <name>timeline.metrics.cluster.aggregator.hourly.disabled</name>\n" +
- " <value>true</value>\n" +
- " <description>\n" +
- " Disable cluster based hourly aggregations.\n" +
- " </description>\n" +
- " </property>\n" +
- " <property>\n" +
- " <name>timeline.metrics.cluster.aggregator.minute.disabled</name>\n" +
- " <value>true</value>\n" +
- " <description>\n" +
- " Disable cluster based minute aggregations.\n" +
- " </description>\n" +
- " </property>" +
- "</configuration>");
-
- ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
-
- // Add the conf dir to the classpath
- // Chain the current thread classloader
- URLClassLoader urlClassLoader = null;
- try {
- urlClassLoader = new URLClassLoader(new URL[] {
- folder.getRoot().toURI().toURL() }, currentClassLoader);
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
-
- Thread.currentThread().setContextClassLoader(urlClassLoader);
- metricsConf = new Configuration(false);
- metricsConf.addResource(Thread.currentThread().getContextClassLoader()
- .getResource(METRICS_SITE_CONFIGURATION_FILE).toURI().toURL());
- assertNotNull(metricsConf.get("test"));
- }
-
- // simple test init/start/stop ApplicationHistoryServer. Status should change.
- @Test(timeout = 50000)
- public void testStartStopServer() throws Exception {
- Configuration config = new YarnConfiguration();
- UserGroupInformation ugi =
- UserGroupInformation.createUserForTesting("ambari", new String[] {"ambari"});
-
- mockStatic(UserGroupInformation.class);
- expect(UserGroupInformation.getCurrentUser()).andReturn(ugi).anyTimes();
- expect(UserGroupInformation.isSecurityEnabled()).andReturn(false).anyTimes();
- config.set(YarnConfiguration.APPLICATION_HISTORY_STORE,
- "org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore");
-
- Connection connection = createNiceMock(Connection.class);
- Statement stmt = createNiceMock(Statement.class);
- mockStatic(DriverManager.class);
- expect(DriverManager.getConnection("jdbc:phoenix:localhost:2181:/hbase"))
- .andReturn(connection).anyTimes();
- expect(connection.createStatement()).andReturn(stmt).anyTimes();
- suppress(method(Statement.class, "executeUpdate", String.class));
- connection.close();
- expectLastCall();
-
- EasyMock.replay(connection, stmt);
- replayAll();
-
- historyServer = new ApplicationHistoryServer();
- historyServer.init(config);
-
- verifyAll();
-
- assertEquals(STATE.INITED, historyServer.getServiceState());
- assertEquals(4, historyServer.getServices().size());
- ApplicationHistoryClientService historyService =
- historyServer.getClientService();
- assertNotNull(historyServer.getClientService());
- assertEquals(STATE.INITED, historyService.getServiceState());
-
- historyServer.start();
- assertEquals(STATE.STARTED, historyServer.getServiceState());
- assertEquals(STATE.STARTED, historyService.getServiceState());
- historyServer.stop();
- assertEquals(STATE.STOPPED, historyServer.getServiceState());
- }
-
- // test launch method
- @Ignore
- @Test(timeout = 60000)
- public void testLaunch() throws Exception {
-
- UserGroupInformation ugi =
- UserGroupInformation.createUserForTesting("ambari", new String[]{"ambari"});
- mockStatic(UserGroupInformation.class);
- expect(UserGroupInformation.getCurrentUser()).andReturn(ugi).anyTimes();
- expect(UserGroupInformation.isSecurityEnabled()).andReturn(false).anyTimes();
-
- ExitUtil.disableSystemExit();
- try {
- historyServer = ApplicationHistoryServer.launchAppHistoryServer(new String[0]);
- } catch (ExitUtil.ExitException e) {
- assertEquals(0, e.status);
- ExitUtil.resetFirstExitException();
- fail();
- }
- }
-
- @After
- public void stop() {
- if (historyServer != null) {
- historyServer.stop();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
deleted file mode 100644
index bc16d36..0000000
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.applicationhistoryservice;
-
-import java.io.IOException;
-import java.net.URI;
-
-import junit.framework.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestFileSystemApplicationHistoryStore extends
- ApplicationHistoryStoreTestUtils {
-
- private FileSystem fs;
- private Path fsWorkingPath;
-
- @Before
- public void setup() throws Exception {
- fs = new RawLocalFileSystem();
- Configuration conf = new Configuration();
- fs.initialize(new URI("/"), conf);
- fsWorkingPath = new Path("Test");
- fs.delete(fsWorkingPath, true);
- conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString());
- store = new FileSystemApplicationHistoryStore();
- store.init(conf);
- store.start();
- }
-
- @After
- public void tearDown() throws Exception {
- store.stop();
- fs.delete(fsWorkingPath, true);
- fs.close();
- }
-
- @Test
- public void testReadWriteHistoryData() throws IOException {
- testWriteHistoryData(5);
- testReadHistoryData(5);
- }
-
- private void testWriteHistoryData(int num) throws IOException {
- testWriteHistoryData(num, false, false);
- }
-
- private void testWriteHistoryData(
- int num, boolean missingContainer, boolean missingApplicationAttempt)
- throws IOException {
- // write application history data
- for (int i = 1; i <= num; ++i) {
- ApplicationId appId = ApplicationId.newInstance(0, i);
- writeApplicationStartData(appId);
-
- // write application attempt history data
- for (int j = 1; j <= num; ++j) {
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, j);
- writeApplicationAttemptStartData(appAttemptId);
-
- if (missingApplicationAttempt && j == num) {
- continue;
- }
- // write container history data
- for (int k = 1; k <= num; ++k) {
- ContainerId containerId = ContainerId.newInstance(appAttemptId, k);
- writeContainerStartData(containerId);
- if (missingContainer && k == num) {
- continue;
- }
- writeContainerFinishData(containerId);
- }
- writeApplicationAttemptFinishData(appAttemptId);
- }
- writeApplicationFinishData(appId);
- }
- }
-
- private void testReadHistoryData(int num) throws IOException {
- testReadHistoryData(num, false, false);
- }
-
- private void testReadHistoryData(
- int num, boolean missingContainer, boolean missingApplicationAttempt)
- throws IOException {
- // read application history data
- Assert.assertEquals(num, store.getAllApplications().size());
- for (int i = 1; i <= num; ++i) {
- ApplicationId appId = ApplicationId.newInstance(0, i);
- ApplicationHistoryData appData = store.getApplication(appId);
- Assert.assertNotNull(appData);
- Assert.assertEquals(appId.toString(), appData.getApplicationName());
- Assert.assertEquals(appId.toString(), appData.getDiagnosticsInfo());
-
- // read application attempt history data
- Assert.assertEquals(num, store.getApplicationAttempts(appId).size());
- for (int j = 1; j <= num; ++j) {
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, j);
- ApplicationAttemptHistoryData attemptData =
- store.getApplicationAttempt(appAttemptId);
- Assert.assertNotNull(attemptData);
- Assert.assertEquals(appAttemptId.toString(), attemptData.getHost());
-
- if (missingApplicationAttempt && j == num) {
- Assert.assertNull(attemptData.getDiagnosticsInfo());
- continue;
- } else {
- Assert.assertEquals(appAttemptId.toString(),
- attemptData.getDiagnosticsInfo());
- }
-
- // read container history data
- Assert.assertEquals(num, store.getContainers(appAttemptId).size());
- for (int k = 1; k <= num; ++k) {
- ContainerId containerId = ContainerId.newInstance(appAttemptId, k);
- ContainerHistoryData containerData = store.getContainer(containerId);
- Assert.assertNotNull(containerData);
- Assert.assertEquals(Priority.newInstance(containerId.getId()),
- containerData.getPriority());
- if (missingContainer && k == num) {
- Assert.assertNull(containerData.getDiagnosticsInfo());
- } else {
- Assert.assertEquals(containerId.toString(),
- containerData.getDiagnosticsInfo());
- }
- }
- ContainerHistoryData masterContainer =
- store.getAMContainer(appAttemptId);
- Assert.assertNotNull(masterContainer);
- Assert.assertEquals(ContainerId.newInstance(appAttemptId, 1),
- masterContainer.getContainerId());
- }
- }
- }
-
- @Test
- public void testWriteAfterApplicationFinish() throws IOException {
- ApplicationId appId = ApplicationId.newInstance(0, 1);
- writeApplicationStartData(appId);
- writeApplicationFinishData(appId);
- // write application attempt history data
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- try {
- writeApplicationAttemptStartData(appAttemptId);
- Assert.fail();
- } catch (IOException e) {
- Assert.assertTrue(e.getMessage().contains("is not opened"));
- }
- try {
- writeApplicationAttemptFinishData(appAttemptId);
- Assert.fail();
- } catch (IOException e) {
- Assert.assertTrue(e.getMessage().contains("is not opened"));
- }
- // write container history data
- ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
- try {
- writeContainerStartData(containerId);
- Assert.fail();
- } catch (IOException e) {
- Assert.assertTrue(e.getMessage().contains("is not opened"));
- }
- try {
- writeContainerFinishData(containerId);
- Assert.fail();
- } catch (IOException e) {
- Assert.assertTrue(e.getMessage().contains("is not opened"));
- }
- }
-
- @Test
- public void testMassiveWriteContainerHistoryData() throws IOException {
- long mb = 1024 * 1024;
- long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
- ApplicationId appId = ApplicationId.newInstance(0, 1);
- writeApplicationStartData(appId);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- for (int i = 1; i <= 100000; ++i) {
- ContainerId containerId = ContainerId.newInstance(appAttemptId, i);
- writeContainerStartData(containerId);
- writeContainerFinishData(containerId);
- }
- writeApplicationFinishData(appId);
- long usedDiskAfter = fs.getContentSummary(fsWorkingPath).getLength() / mb;
- Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20);
- }
-
- @Test
- public void testMissingContainerHistoryData() throws IOException {
- testWriteHistoryData(3, true, false);
- testReadHistoryData(3, true, false);
- }
-
- @Test
- public void testMissingApplicationAttemptHistoryData() throws IOException {
- testWriteHistoryData(3, false, true);
- testReadHistoryData(3, false, true);
- }
-}