You are viewing a plain text version of this content. The canonical link for it is here.

Posted to reviews@ambari.apache.org by Dmytro Sen <ds...@hortonworks.com> on 2017/04/13 16:54:48 UTC

Review Request 58424: Aggregate local metrics for minute aggregation time window

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/
-----------------------------------------------------------

Review request for Ambari, Aravindan Vijayan and Sid Wagle.


Bugs: AMBARI-20758
    https://issues.apache.org/jira/browse/AMBARI-20758


Repository: ambari


Description
-------

Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
Handshake / Ping to make sure its the monitor and is accepting writes
Start sending metrics only to the monitor
Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
The monitor interface look the same as that of collector (http server: maybe Flask)
Security concerns need to be addressed as well. (https/spnego?)


Diffs
-----

  ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java 942c0b4 
  ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java bb2f0a9 
  ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java 8ee6d00 
  ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
  ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 46f32f9 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java 8ecca54 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java 383079a 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java 0bed7d0 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java 18d973c 
  ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 3fdf3f4 
  ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java bd4ae6a 
  ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java a112ef2 
  ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java 5777639 
  ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
  ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 9d492cb 
  ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 9a55f10 
  ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 60c1427 
  ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 535fae0 
  ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java f58f549 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java 17c58f0 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java 8b0d84b 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java d049e33 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java aa53430 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java 1446ec2 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java 5310906 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java 7eb2457 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java 634e51d 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 304a8e0 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java 611d82e 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b2e8cac 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java 81da5c8 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b243e0b 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java bd54bad 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b093a2a 
  ambari-metrics/pom.xml 2d88912 
  ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java 76c1167 
  ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java 46e791b 
  ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDataTransferMethodFactory.java 207a79d 
  ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java f517555 
  ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java cceaafb 
  ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java c4c2ddc 
  ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java c1576a3 
  ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java ecb59ac 
  ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java 932345b 
  ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java 19e0e60 
  ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheValue.java ba64234 
  ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java f268149 
  ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java 0ba6fb5 
  ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java 366f5a0 
  ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java fbc0b56 
  ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java e5fc4e1 
  ambari-views/src/test/java/org/apache/ambari/AppTest.java PRE-CREATION 
  contrib/views/pig/src/main/java/org/apache/ambari/Aggregator.java PRE-CREATION 
  contrib/views/pig/src/main/java/org/apache/ambari/AggregatorWebService.java PRE-CREATION 
  contrib/views/pig/src/main/java/org/apache/ambari/App.java PRE-CREATION 
  contrib/views/pig/src/main/java/org/apache/ambari/Sender.java PRE-CREATION 
  contrib/views/pig/src/main/java/org/apache/ambari/TimelineMetricsHolder.java PRE-CREATION 


Diff: https://reviews.apache.org/r/58424/diff/1/


Testing
-------

Under development


Thanks,

Dmytro Sen

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Dmytro Sen <ds...@hortonworks.com>.


> On \u0410\u043f\u0440\u0435\u043b\u044c 13, 2017, 5:11 \u043f.\u043f., Sid Wagle wrote:
> > ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
> > Line 498 (original), 498 (patched)
> > <https://reviews.apache.org/r/58424/diff/1/?file=1691920#file1691920line498>
> >
> >     This should be conditional based on whether monitor is available.

I agree, that was just for testing, just for draft


> On \u0410\u043f\u0440\u0435\u043b\u044c 13, 2017, 5:11 \u043f.\u043f., Sid Wagle wrote:
> > ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py
> > Lines 12 (patched)
> > <https://reviews.apache.org/r/58424/diff/1/?file=1691936#file1691936line12>
> >
> >     java_home should be puicked from env variable.

In current patch almost everything is hardcoded, I will fix it in the final version


> On \u0410\u043f\u0440\u0435\u043b\u044c 13, 2017, 5:11 \u043f.\u043f., Sid Wagle wrote:
> > ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py
> > Lines 18 (patched)
> > <https://reviews.apache.org/r/58424/diff/1/?file=1691936#file1691936line18>
> >
> >     Can the kill have a timeout? So we have kill handler trying to flush for 2 secs before kill -9?

Still working on the stop handler


> On \u0410\u043f\u0440\u0435\u043b\u044c 13, 2017, 5:11 \u043f.\u043f., Sid Wagle wrote:
> > contrib/views/pig/src/main/java/org/apache/ambari/AggregatorWebService.java
> > Lines 1 (patched)
> > <https://reviews.apache.org/r/58424/diff/1/?file=1691993#file1691993line1>
> >
> >     Why is this test code in contrib/views?

It should be in ambari-metrics


- Dmytro


-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review171899
-----------------------------------------------------------


On \u0410\u043f\u0440\u0435\u043b\u044c 13, 2017, 4:54 \u043f.\u043f., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated \u0410\u043f\u0440\u0435\u043b\u044c 13, 2017, 4:54 \u043f.\u043f.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java 942c0b4 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java bb2f0a9 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java 8ee6d00 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 46f32f9 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java 8ecca54 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java 383079a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java 0bed7d0 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java 18d973c 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 3fdf3f4 
>   ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java bd4ae6a 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java a112ef2 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java 5777639 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 9d492cb 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 9a55f10 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 60c1427 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 535fae0 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java f58f549 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java 17c58f0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java 8b0d84b 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java d049e33 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java aa53430 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java 1446ec2 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java 5310906 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java 7eb2457 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java 634e51d 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 304a8e0 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java 611d82e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b2e8cac 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java 81da5c8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b243e0b 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java bd54bad 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b093a2a 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java 76c1167 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java 46e791b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDataTransferMethodFactory.java 207a79d 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java f517555 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java cceaafb 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java c4c2ddc 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java c1576a3 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java ecb59ac 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java 932345b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java 19e0e60 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheValue.java ba64234 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java f268149 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java 0ba6fb5 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java 366f5a0 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java fbc0b56 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java e5fc4e1 
>   ambari-views/src/test/java/org/apache/ambari/AppTest.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Aggregator.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/AggregatorWebService.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/App.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Sender.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/TimelineMetricsHolder.java PRE-CREATION 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/1/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Sid Wagle <sw...@hortonworks.com>.

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review171899
-----------------------------------------------------------




ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
Line 498 (original), 498 (patched)
<https://reviews.apache.org/r/58424/#comment244887>

    This should be conditional based on whether monitor is available.



ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py
Lines 12 (patched)
<https://reviews.apache.org/r/58424/#comment244888>

    java_home should be puicked from env variable.



ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py
Lines 18 (patched)
<https://reviews.apache.org/r/58424/#comment244890>

    Can the kill have a timeout? So we have kill handler trying to flush for 2 secs before kill -9?



contrib/views/pig/src/main/java/org/apache/ambari/AggregatorWebService.java
Lines 1 (patched)
<https://reviews.apache.org/r/58424/#comment244892>

    Why is this test code in contrib/views?


- Sid Wagle


On April 13, 2017, 4:54 p.m., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated April 13, 2017, 4:54 p.m.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java 942c0b4 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java bb2f0a9 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java 8ee6d00 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 46f32f9 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java 8ecca54 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java 383079a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java 0bed7d0 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java 18d973c 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 3fdf3f4 
>   ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java bd4ae6a 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java a112ef2 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java 5777639 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 9d492cb 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 9a55f10 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 60c1427 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 535fae0 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java f58f549 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java 17c58f0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java 8b0d84b 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java d049e33 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java aa53430 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java 1446ec2 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java 5310906 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java 7eb2457 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java 634e51d 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 304a8e0 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java 611d82e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b2e8cac 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java 81da5c8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b243e0b 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java bd54bad 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b093a2a 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java 76c1167 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java 46e791b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDataTransferMethodFactory.java 207a79d 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java f517555 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java cceaafb 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java c4c2ddc 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java c1576a3 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java ecb59ac 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java 932345b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java 19e0e60 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheValue.java ba64234 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java f268149 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java 0ba6fb5 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java 366f5a0 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java fbc0b56 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java e5fc4e1 
>   ambari-views/src/test/java/org/apache/ambari/AppTest.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Aggregator.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/AggregatorWebService.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/App.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Sender.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/TimelineMetricsHolder.java PRE-CREATION 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/1/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Dmytro Sen <ds...@hortonworks.com>.


> On Май 15, 2017, 8:56 п.п., Aravindan Vijayan wrote:
> > ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java
> > Lines 61 (patched)
> > <https://reviews.apache.org/r/58424/diff/4/?file=1703338#file1703338line61>
> >
> >     Why do we need a new config for this? Isn't this the same as ams-site:timeline.metrics.host.aggregator.minute.interval?

Fixed in latest patch


> On Май 15, 2017, 8:56 п.п., Aravindan Vijayan wrote:
> > ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java
> > Lines 29 (patched)
> > <https://reviews.apache.org/r/58424/diff/4/?file=1703341#file1703341line29>
> >
> >     This could lead to a point being counted in 2 successive 5min aggregations. Why not have a model where the aggregator thread empties the cache everytime it runs? The Cache expiry is still ok, since without that, it can lead to metric bloat.

Added lock and cache clearing.


- Dmytro


-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review175020
-----------------------------------------------------------


On Апрель 28, 2017, 4:41 п.п., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated Апрель 28, 2017, 4:41 п.п.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java 2d1bf40 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml ab309a1 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 2c6fae2 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java 9b0cdbe 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java a393a96 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java 32fe32e 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 904c916 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java 11e16c2 
>   ambari-metrics/ambari-metrics-host-aggregator/conf/unix/log4j.properties PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/conf/windows/log4j.properties PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AbstractMetricPublisherThread.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatedMetricsPublisher.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorWebService.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/RawMetricsPublisher.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor 967e133 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py 2670e76 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py e2a7f0d 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py bfb6957 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py d218015 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 211e9cd 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 08f0598 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 20f60e1 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 14f160b 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 425201c 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java c242a2f 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java fb369e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java 0d5042f 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java bde09cb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java a5a3499 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 9da921a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java f00906e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java 8d1f63f 
>   ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py 150b0a8 
>   ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 6873c85 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml cb66537 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml 8e1671e 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml 740a91a 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py a929847 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py 50dde1c 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 9729bbe 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 769ad67 
>   ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py 86a290f 
>   ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b876a3d 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py efea167 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 24535c5 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 9076269 
>   ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml fae61d3 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py 0f03155 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py 69ef08e 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
>   ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml e01dacd 
>   ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py 5b0be54 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py d9fae76 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 51162e8 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 0501039 
>   ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py 7282bb5 
>   ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
>   ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml fae61d3 
>   ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py 678bbde 
>   ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
>   ambari-server/src/test/java/org/apache/ambari/server/metric/system/impl/TestAmbariMetricsSinkImpl.java 4a60892 
>   contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py 6a70675 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/4/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Aravindan Vijayan <av...@hortonworks.com>.

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review175020
-----------------------------------------------------------




ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java
Lines 61 (patched)
<https://reviews.apache.org/r/58424/#comment248333>

    Why do we need a new config for this? Isn't this the same as ams-site:timeline.metrics.host.aggregator.minute.interval?



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java
Lines 29 (patched)
<https://reviews.apache.org/r/58424/#comment248337>

    This could lead to a point being counted in 2 successive 5min aggregations. Why not have a model where the aggregator thread empties the cache everytime it runs? The Cache expiry is still ok, since without that, it can lead to metric bloat.


- Aravindan Vijayan


On April 28, 2017, 4:41 p.m., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated April 28, 2017, 4:41 p.m.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java 2d1bf40 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml ab309a1 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 2c6fae2 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java 9b0cdbe 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java a393a96 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java 32fe32e 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 904c916 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java 11e16c2 
>   ambari-metrics/ambari-metrics-host-aggregator/conf/unix/log4j.properties PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/conf/windows/log4j.properties PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AbstractMetricPublisherThread.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatedMetricsPublisher.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorWebService.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/RawMetricsPublisher.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor 967e133 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py 2670e76 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py e2a7f0d 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py bfb6957 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py d218015 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 211e9cd 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 08f0598 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 20f60e1 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 14f160b 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 425201c 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java c242a2f 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java fb369e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java 0d5042f 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java bde09cb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java a5a3499 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 9da921a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java f00906e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java 8d1f63f 
>   ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py 150b0a8 
>   ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 6873c85 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml cb66537 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml 8e1671e 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml 740a91a 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py a929847 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py 50dde1c 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 9729bbe 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 769ad67 
>   ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py 86a290f 
>   ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b876a3d 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py efea167 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 24535c5 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 9076269 
>   ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml fae61d3 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py 0f03155 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py 69ef08e 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
>   ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml e01dacd 
>   ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py 5b0be54 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py d9fae76 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 51162e8 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 0501039 
>   ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py 7282bb5 
>   ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
>   ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml fae61d3 
>   ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py 678bbde 
>   ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
>   ambari-server/src/test/java/org/apache/ambari/server/metric/system/impl/TestAmbariMetricsSinkImpl.java 4a60892 
>   contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py 6a70675 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/4/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Sid Wagle <sw...@hortonworks.com>.

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review175038
-----------------------------------------------------------



Without goind throught full patch it is always a good idea to have a writeLock for emptying the cache, +1 Aravindan's idea of simply cachce and empty and drop on the floor if cannot submit.

- Sid Wagle


On April 28, 2017, 4:41 p.m., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated April 28, 2017, 4:41 p.m.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java 2d1bf40 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml ab309a1 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 2c6fae2 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java 9b0cdbe 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java a393a96 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java 32fe32e 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 904c916 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java 11e16c2 
>   ambari-metrics/ambari-metrics-host-aggregator/conf/unix/log4j.properties PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/conf/windows/log4j.properties PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AbstractMetricPublisherThread.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatedMetricsPublisher.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorWebService.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/RawMetricsPublisher.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor 967e133 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py 2670e76 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py e2a7f0d 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py bfb6957 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py d218015 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 211e9cd 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 08f0598 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 20f60e1 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 14f160b 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 425201c 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java c242a2f 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java fb369e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java 0d5042f 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java bde09cb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java a5a3499 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 9da921a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java f00906e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java 8d1f63f 
>   ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py 150b0a8 
>   ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 6873c85 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml cb66537 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml 8e1671e 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml 740a91a 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py a929847 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py 50dde1c 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 9729bbe 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 769ad67 
>   ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py 86a290f 
>   ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b876a3d 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py efea167 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 24535c5 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 9076269 
>   ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml fae61d3 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py 0f03155 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py 69ef08e 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
>   ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml e01dacd 
>   ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py 5b0be54 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py d9fae76 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 51162e8 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 0501039 
>   ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py 7282bb5 
>   ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
>   ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml fae61d3 
>   ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py 678bbde 
>   ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
>   ambari-server/src/test/java/org/apache/ambari/server/metric/system/impl/TestAmbariMetricsSinkImpl.java 4a60892 
>   contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py 6a70675 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/4/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Dmytro Sen <ds...@hortonworks.com>.

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/
-----------------------------------------------------------

(Updated Май 16, 2017, 4:32 п.п.)


Review request for Ambari, Aravindan Vijayan and Sid Wagle.


Bugs: AMBARI-20758
    https://issues.apache.org/jira/browse/AMBARI-20758


Repository: ambari


Description
-------

Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
Handshake / Ping to make sure its the monitor and is accepting writes
Start sending metrics only to the monitor
Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
The monitor interface look the same as that of collector (http server: maybe Flask)
Security concerns need to be addressed as well. (https/spnego?)


Diffs (updated)
-----

  ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java 2d1bf40 
  ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
  ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml ab309a1 
  ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 2c6fae2 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java 9b0cdbe 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java a393a96 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java 32fe32e 
  ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 904c916 
  ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java 11e16c2 
  ambari-metrics/ambari-metrics-host-aggregator/conf/unix/log4j.properties PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/conf/windows/log4j.properties PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AbstractMetricPublisherThread.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatedMetricsPublisher.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorWebService.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/RawMetricsPublisher.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor 967e133 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py 2670e76 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py e2a7f0d 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py bfb6957 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py d218015 
  ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 211e9cd 
  ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 08f0598 
  ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 20f60e1 
  ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 14f160b 
  ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 425201c 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java c242a2f 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java fb369e8 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java 0d5042f 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java bde09cb 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 0934356 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java a5a3499 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 9da921a 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java f00906e 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java f083731 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
  ambari-metrics/pom.xml 2d88912 
  ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java 8d1f63f 
  ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py 150b0a8 
  ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 6873c85 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml cb66537 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml 8e1671e 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml 740a91a 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py a929847 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py 50dde1c 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 9729bbe 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 769ad67 
  ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py 86a290f 
  ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b876a3d 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py efea167 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 24535c5 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 9076269 
  ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml fae61d3 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py d854451 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py c0ac535 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
  ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml e01dacd 
  ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py 5b0be54 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py d9fae76 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 51162e8 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 0501039 
  ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py 7282bb5 
  ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
  ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml fae61d3 
  ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py 678bbde 
  ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
  ambari-server/src/test/java/org/apache/ambari/server/metric/system/impl/TestAmbariMetricsSinkImpl.java 4a60892 
  contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py 6a70675 


Diff: https://reviews.apache.org/r/58424/diff/5/

Changes: https://reviews.apache.org/r/58424/diff/4-5/


Testing
-------

Under development


Thanks,

Dmytro Sen

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Dmytro Sen <ds...@hortonworks.com>.

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/
-----------------------------------------------------------

(Updated \u0410\u043f\u0440\u0435\u043b\u044c 28, 2017, 4:41 \u043f.\u043f.)


Review request for Ambari, Aravindan Vijayan and Sid Wagle.


Bugs: AMBARI-20758
    https://issues.apache.org/jira/browse/AMBARI-20758


Repository: ambari


Description
-------

Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
Handshake / Ping to make sure its the monitor and is accepting writes
Start sending metrics only to the monitor
Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
The monitor interface look the same as that of collector (http server: maybe Flask)
Security concerns need to be addressed as well. (https/spnego?)


Diffs (updated)
-----

  ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java 2d1bf40 
  ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
  ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml ab309a1 
  ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 2c6fae2 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java 9b0cdbe 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java a393a96 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java 32fe32e 
  ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 904c916 
  ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java 11e16c2 
  ambari-metrics/ambari-metrics-host-aggregator/conf/unix/log4j.properties PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/conf/windows/log4j.properties PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AbstractMetricPublisherThread.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatedMetricsPublisher.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorWebService.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/RawMetricsPublisher.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor 967e133 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py 2670e76 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py e2a7f0d 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py bfb6957 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py d218015 
  ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 211e9cd 
  ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 08f0598 
  ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 20f60e1 
  ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 14f160b 
  ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 425201c 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java c242a2f 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java fb369e8 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java 0d5042f 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java bde09cb 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java a5a3499 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 9da921a 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java f00906e 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
  ambari-metrics/pom.xml 2d88912 
  ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java 8d1f63f 
  ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py 150b0a8 
  ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 6873c85 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml cb66537 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml 8e1671e 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml 740a91a 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py a929847 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py 50dde1c 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 9729bbe 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 769ad67 
  ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py 86a290f 
  ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b876a3d 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py efea167 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 24535c5 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 9076269 
  ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml fae61d3 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py 0f03155 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py 69ef08e 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
  ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml e01dacd 
  ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py 5b0be54 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py d9fae76 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 51162e8 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 0501039 
  ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py 7282bb5 
  ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
  ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml fae61d3 
  ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py 678bbde 
  ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
  ambari-server/src/test/java/org/apache/ambari/server/metric/system/impl/TestAmbariMetricsSinkImpl.java 4a60892 
  contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py 6a70675 


Diff: https://reviews.apache.org/r/58424/diff/4/

Changes: https://reviews.apache.org/r/58424/diff/3-4/


Testing
-------

Under development


Thanks,

Dmytro Sen

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Dmytro Sen <ds...@hortonworks.com>.

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/
-----------------------------------------------------------

(Updated \u0410\u043f\u0440\u0435\u043b\u044c 28, 2017, 1:26 \u043f.\u043f.)


Review request for Ambari, Aravindan Vijayan and Sid Wagle.


Bugs: AMBARI-20758
    https://issues.apache.org/jira/browse/AMBARI-20758


Repository: ambari


Description
-------

Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
Handshake / Ping to make sure its the monitor and is accepting writes
Start sending metrics only to the monitor
Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
The monitor interface look the same as that of collector (http server: maybe Flask)
Security concerns need to be addressed as well. (https/spnego?)


Diffs (updated)
-----

  ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java 2d1bf40 
  ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
  ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml ab309a1 
  ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 2c6fae2 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java 9b0cdbe 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java a393a96 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java 32fe32e 
  ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 904c916 
  ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java 11e16c2 
  ambari-metrics/ambari-metrics-host-aggregator/conf/unix/log4j.properties PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/conf/windows/log4j.properties PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AbstractMetricPublisherThread.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatedMetricsPublisher.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorWebService.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/RawMetricsPublisher.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/host/aggregator/AppTest.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py 2670e76 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py e2a7f0d 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py bfb6957 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py d218015 
  ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 211e9cd 
  ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 08f0598 
  ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 20f60e1 
  ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 14f160b 
  ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 425201c 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java c242a2f 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java fb369e8 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java 0d5042f 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java bde09cb 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java a5a3499 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 9da921a 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java f00906e 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
  ambari-metrics/pom.xml 2d88912 
  ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java 8d1f63f 
  ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py 150b0a8 
  ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 6873c85 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml cb66537 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml 8e1671e 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml 740a91a 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py a929847 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py 50dde1c 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 9729bbe 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 769ad67 
  ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py 86a290f 
  ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b876a3d 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py efea167 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 24535c5 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 9076269 
  ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml fae61d3 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py 0f03155 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py 69ef08e 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
  ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml e01dacd 
  ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py 5b0be54 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py d9fae76 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 51162e8 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 0501039 
  ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py 7282bb5 
  ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
  ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml fae61d3 
  ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py 678bbde 
  ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
  ambari-server/src/test/java/org/apache/ambari/server/metric/system/impl/TestAmbariMetricsSinkImpl.java 4a60892 
  contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py 6a70675 


Diff: https://reviews.apache.org/r/58424/diff/3/

Changes: https://reviews.apache.org/r/58424/diff/2-3/


Testing
-------

Under development


Thanks,

Dmytro Sen

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Dmytro Sen <ds...@hortonworks.com>.


> On \u0410\u043f\u0440\u0435\u043b\u044c 26, 2017, 8:48 \u043f.\u043f., Sid Wagle wrote:
> > ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py
> > Lines 44 (patched)
> > <https://reviews.apache.org/r/58424/diff/1-2/?file=1691936#file1691936line49>
> >
> >     Protocol support for https? Maybe not important for local write but maybe it should be parameterized.

I think, we can implement https support in a separate jira


> On \u0410\u043f\u0440\u0435\u043b\u044c 26, 2017, 8:48 \u043f.\u043f., Sid Wagle wrote:
> > ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java
> > Lines 32 (patched)
> > <https://reviews.apache.org/r/58424/diff/2/?file=1700473#file1700473line32>
> >
> >     Case on arguments should be camel vs pythonic

Fixed in latest patch. Will upload it with test fixed.


> On \u0410\u043f\u0440\u0435\u043b\u044c 26, 2017, 8:48 \u043f.\u043f., Sid Wagle wrote:
> > ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorThread.java
> > Lines 81 (patched)
> > <https://reviews.apache.org/r/58424/diff/2/?file=1700474#file1700474line81>
> >
> >     Can we move initiazliation outside?

Fixed (Created abstract publisher class)


- Dmytro


-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review173087
-----------------------------------------------------------


On \u0410\u043f\u0440\u0435\u043b\u044c 28, 2017, 1:26 \u043f.\u043f., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated \u0410\u043f\u0440\u0435\u043b\u044c 28, 2017, 1:26 \u043f.\u043f.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java 2d1bf40 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml ab309a1 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 2c6fae2 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java 9b0cdbe 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java a393a96 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java 32fe32e 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 904c916 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java 11e16c2 
>   ambari-metrics/ambari-metrics-host-aggregator/conf/unix/log4j.properties PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/conf/windows/log4j.properties PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AbstractMetricPublisherThread.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatedMetricsPublisher.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorWebService.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/RawMetricsPublisher.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/host/aggregator/AppTest.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py 2670e76 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py e2a7f0d 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py bfb6957 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py d218015 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 211e9cd 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 08f0598 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 20f60e1 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 14f160b 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 425201c 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java c242a2f 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java fb369e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java 0d5042f 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java bde09cb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java a5a3499 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 9da921a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java f00906e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java 8d1f63f 
>   ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py 150b0a8 
>   ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 6873c85 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml cb66537 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml 8e1671e 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml 740a91a 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py a929847 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py 50dde1c 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 9729bbe 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 769ad67 
>   ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py 86a290f 
>   ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b876a3d 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py efea167 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 24535c5 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 9076269 
>   ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml fae61d3 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py 0f03155 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py 69ef08e 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 9328f9f 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 9a7f9dc 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2 e9fe024 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 bd7eb0c 
>   ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml e01dacd 
>   ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py 5b0be54 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py d9fae76 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 51162e8 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 0501039 
>   ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py 7282bb5 
>   ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
>   ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml fae61d3 
>   ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py 678bbde 
>   ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 1b02a97 
>   ambari-server/src/test/java/org/apache/ambari/server/metric/system/impl/TestAmbariMetricsSinkImpl.java 4a60892 
>   contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py 6a70675 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/3/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Sid Wagle <sw...@hortonworks.com>.

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review173087
-----------------------------------------------------------




ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py
Lines 21 (patched)
<https://reviews.apache.org/r/58424/#comment246166>

    Can we replace version with expression or make it parameterized for future proofing? Easier to change config vs code-edit.



ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py
Lines 44 (patched)
<https://reviews.apache.org/r/58424/#comment246167>

    Protocol support for https? Maybe not important for local write but maybe it should be parameterized.



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java
Lines 32 (patched)
<https://reviews.apache.org/r/58424/#comment246153>

    Case on arguments should be camel vs pythonic



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java
Lines 126 (patched)
<https://reviews.apache.org/r/58424/#comment246155>

    Potential NPE without default.



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorThread.java
Lines 47 (patched)
<https://reviews.apache.org/r/58424/#comment246156>

    This looks funky, instead of static method we should be calling a member on the cache reference.



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorThread.java
Lines 81 (patched)
<https://reviews.apache.org/r/58424/#comment246158>

    Can we move initiazliation outside?



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorThread.java
Lines 110 (patched)
<https://reviews.apache.org/r/58424/#comment246160>

    Since AggregatorThread also send data we should rename to something like AggregatorPublisher and RawMetricPublisher.



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorThread.java
Lines 114 (patched)
<https://reviews.apache.org/r/58424/#comment246159>

    formatting.



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/SenderThread.java
Lines 48 (patched)
<https://reviews.apache.org/r/58424/#comment246161>

    Again intialize once.



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/SenderThread.java
Lines 60 (patched)
<https://reviews.apache.org/r/58424/#comment246162>

    Can we move this to a helper class to avoid dup?



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java
Lines 12 (patched)
<https://reviews.apache.org/r/58424/#comment246164>

    Why static, not a best prcatice just make it a singleton.



ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java
Lines 23 (patched)
<https://reviews.apache.org/r/58424/#comment246165>

    Again adding Raw vs aggreate will imprive readbility.


- Sid Wagle


On April 26, 2017, 4:44 p.m., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated April 26, 2017, 4:44 p.m.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java 2d1bf40 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 46f32f9 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java 9b0cdbe 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java a393a96 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java 32fe32e 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 3fdf3f4 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java 8e0de03 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorThread.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorWebService.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/SenderThread.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/main/resources/log4j.properties PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/host/aggregator/AppTest.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py 5686c50 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py ba3f18e 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py bfb6957 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py d218015 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 9d492cb 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 9a55f10 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 60c1427 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 535fae0 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java f58f549 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java fa095a0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java 65bbc4c 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java 0d5042f 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java 121a8ae 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java a5a3499 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 6278c59 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b40481d 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java f268149 
>   ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py 53cf002 
>   ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 f7926d0 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml 8e1671e 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml 740a91a 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py 919f26d 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 8c6f86f 
>   ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 1505f9b 
>   ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py b143941 
>   ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 cf2bd6c 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py 3177643 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 e808f5d 
>   ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 7e1abbc 
>   ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml 4aadb83 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py 7615ee7 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 82f71c5 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 f1bfa03 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 d63dfb7 
>   ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 4c9c981 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py 7615ee7 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 82f71c5 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 f1bfa03 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2 d63dfb7 
>   ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 4c9c981 
>   ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml f2b4a76 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py 44b256e 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 f2a75cf 
>   ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 001ff73 
>   ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py 70ebfeb 
>   ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 2f3aab6 
>   ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml 4aadb83 
>   ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py 5a5361c 
>   ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 2f3aab6 
>   ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/configuration/hadoop-metrics2.properties.xml 4aadb83 
>   ambari-server/src/test/java/org/apache/ambari/server/metric/system/impl/TestAmbariMetricsSinkImpl.java 4a60892 
>   contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py 6a70675 
>   contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 fcd9b23 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/2/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Dmytro Sen <ds...@hortonworks.com>.

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/
-----------------------------------------------------------

(Updated \u0410\u043f\u0440\u0435\u043b\u044c 26, 2017, 4:44 \u043f.\u043f.)


Review request for Ambari, Aravindan Vijayan and Sid Wagle.


Bugs: AMBARI-20758
    https://issues.apache.org/jira/browse/AMBARI-20758


Repository: ambari


Description
-------

Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
Handshake / Ping to make sure its the monitor and is accepting writes
Start sending metrics only to the monitor
Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
The monitor interface look the same as that of collector (http server: maybe Flask)
Security concerns need to be addressed as well. (https/spnego?)


Diffs (updated)
-----

  ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java 2d1bf40 
  ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 46f32f9 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
  ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java 9b0cdbe 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java a393a96 
  ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java 32fe32e 
  ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 3fdf3f4 
  ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java 8e0de03 
  ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorThread.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorWebService.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/SenderThread.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/TimelineMetricsHolder.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/main/resources/log4j.properties PRE-CREATION 
  ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/host/aggregator/AppTest.java PRE-CREATION 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py 5686c50 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py ba3f18e 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py bfb6957 
  ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py d218015 
  ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 9d492cb 
  ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 9a55f10 
  ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 60c1427 
  ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 535fae0 
  ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java f58f549 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java fa095a0 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java 65bbc4c 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java 0d5042f 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java 121a8ae 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java a5a3499 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
  ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 6278c59 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b40481d 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
  ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
  ambari-metrics/pom.xml 2d88912 
  ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java f268149 
  ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py 53cf002 
  ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 f7926d0 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml 8e1671e 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml 740a91a 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py 919f26d 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 8c6f86f 
  ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 1505f9b 
  ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py b143941 
  ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 cf2bd6c 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py 3177643 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 e808f5d 
  ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 7e1abbc 
  ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml 4aadb83 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py 7615ee7 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 82f71c5 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 f1bfa03 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 d63dfb7 
  ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 4c9c981 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py 7615ee7 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 82f71c5 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 f1bfa03 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2 d63dfb7 
  ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 4c9c981 
  ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml f2b4a76 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py 44b256e 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 f2a75cf 
  ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 001ff73 
  ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py 70ebfeb 
  ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 2f3aab6 
  ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml 4aadb83 
  ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py 5a5361c 
  ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 2f3aab6 
  ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHDFS/configuration/hadoop-metrics2.properties.xml 4aadb83 
  ambari-server/src/test/java/org/apache/ambari/server/metric/system/impl/TestAmbariMetricsSinkImpl.java 4a60892 
  contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py 6a70675 
  contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 fcd9b23 


Diff: https://reviews.apache.org/r/58424/diff/2/

Changes: https://reviews.apache.org/r/58424/diff/1-2/


Testing
-------

Under development


Thanks,

Dmytro Sen

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Dmytro Sen <ds...@hortonworks.com>.


> On \u0410\u043f\u0440\u0435\u043b\u044c 13, 2017, 5:31 \u043f.\u043f., Aravindan Vijayan wrote:
> > ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
> > Line 153 (original), 157 (patched)
> > <https://reviews.apache.org/r/58424/diff/1/?file=1691943#file1691943line157>
> >
> >     Is this needed after this change? 
> >     
> >     How do we handle the case where the local monitor is down on some host, and part of the metrics will need to be aggregated at the collecor side and part of them have already been aggregated and written to METRIC_RECORD_MINUTE?
> 
> Sid Wagle wrote:
>     Lets handle this as a mode of operation. If monitor aggregation is enabled we do not do collector side 5 minute aggregation. If monitor is down we do not get metrics from that host. Lets add an enhancement Jira to handle situation where daemon metrics from a host are available per minute but aggregates are not, fo this scenario run a the aggregator with subset of metrics. This could be based on liveliness check of a monitor. More like a heartbeat ping. When we loose ping for > 1 min < 5 min we can aggregate metrics for that time period and hostId.

agree, but maybe it's better to implement it in separate task, and for now just switch off a minute aggregation if in-memory one is configured?


- Dmytro


-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review171905
-----------------------------------------------------------


On \u0410\u043f\u0440\u0435\u043b\u044c 13, 2017, 4:54 \u043f.\u043f., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated \u0410\u043f\u0440\u0435\u043b\u044c 13, 2017, 4:54 \u043f.\u043f.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java 942c0b4 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java bb2f0a9 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java 8ee6d00 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 46f32f9 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java 8ecca54 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java 383079a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java 0bed7d0 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java 18d973c 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 3fdf3f4 
>   ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java bd4ae6a 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java a112ef2 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java 5777639 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 9d492cb 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 9a55f10 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 60c1427 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 535fae0 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java f58f549 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java 17c58f0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java 8b0d84b 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java d049e33 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java aa53430 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java 1446ec2 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java 5310906 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java 7eb2457 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java 634e51d 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 304a8e0 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java 611d82e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b2e8cac 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java 81da5c8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b243e0b 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java bd54bad 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b093a2a 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java 76c1167 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java 46e791b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDataTransferMethodFactory.java 207a79d 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java f517555 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java cceaafb 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java c4c2ddc 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java c1576a3 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java ecb59ac 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java 932345b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java 19e0e60 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheValue.java ba64234 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java f268149 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java 0ba6fb5 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java 366f5a0 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java fbc0b56 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java e5fc4e1 
>   ambari-views/src/test/java/org/apache/ambari/AppTest.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Aggregator.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/AggregatorWebService.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/App.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Sender.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/TimelineMetricsHolder.java PRE-CREATION 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/1/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Sid Wagle <sw...@hortonworks.com>.


> On April 13, 2017, 5:31 p.m., Aravindan Vijayan wrote:
> > ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
> > Line 153 (original), 157 (patched)
> > <https://reviews.apache.org/r/58424/diff/1/?file=1691943#file1691943line157>
> >
> >     Is this needed after this change? 
> >     
> >     How do we handle the case where the local monitor is down on some host, and part of the metrics will need to be aggregated at the collecor side and part of them have already been aggregated and written to METRIC_RECORD_MINUTE?

Lets handle this as a mode of operation. If monitor aggregation is enabled we do not do collector side 5 minute aggregation. If monitor is down we do not get metrics from that host. Lets add an enhancement Jira to handle situation where daemon metrics from a host are available per minute but aggregates are not, fo this scenario run a the aggregator with subset of metrics. This could be based on liveliness check of a monitor. More like a heartbeat ping. When we loose ping for > 1 min < 5 min we can aggregate metrics for that time period and hostId.


- Sid


-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review171905
-----------------------------------------------------------


On April 13, 2017, 4:54 p.m., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated April 13, 2017, 4:54 p.m.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java 942c0b4 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java bb2f0a9 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java 8ee6d00 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 46f32f9 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java 8ecca54 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java 383079a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java 0bed7d0 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java 18d973c 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 3fdf3f4 
>   ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java bd4ae6a 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java a112ef2 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java 5777639 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 9d492cb 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 9a55f10 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 60c1427 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 535fae0 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java f58f549 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java 17c58f0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java 8b0d84b 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java d049e33 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java aa53430 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java 1446ec2 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java 5310906 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java 7eb2457 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java 634e51d 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 304a8e0 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java 611d82e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b2e8cac 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java 81da5c8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b243e0b 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java bd54bad 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b093a2a 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java 76c1167 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java 46e791b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDataTransferMethodFactory.java 207a79d 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java f517555 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java cceaafb 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java c4c2ddc 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java c1576a3 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java ecb59ac 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java 932345b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java 19e0e60 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheValue.java ba64234 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java f268149 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java 0ba6fb5 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java 366f5a0 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java fbc0b56 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java e5fc4e1 
>   ambari-views/src/test/java/org/apache/ambari/AppTest.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Aggregator.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/AggregatorWebService.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/App.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Sender.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/TimelineMetricsHolder.java PRE-CREATION 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/1/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Sid Wagle <sw...@hortonworks.com>.


> On April 13, 2017, 5:31 p.m., Aravindan Vijayan wrote:
> > ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
> > Line 153 (original), 157 (patched)
> > <https://reviews.apache.org/r/58424/diff/1/?file=1691943#file1691943line157>
> >
> >     Is this needed after this change? 
> >     
> >     How do we handle the case where the local monitor is down on some host, and part of the metrics will need to be aggregated at the collecor side and part of them have already been aggregated and written to METRIC_RECORD_MINUTE?
> 
> Sid Wagle wrote:
>     Lets handle this as a mode of operation. If monitor aggregation is enabled we do not do collector side 5 minute aggregation. If monitor is down we do not get metrics from that host. Lets add an enhancement Jira to handle situation where daemon metrics from a host are available per minute but aggregates are not, fo this scenario run a the aggregator with subset of metrics. This could be based on liveliness check of a monitor. More like a heartbeat ping. When we loose ping for > 1 min < 5 min we can aggregate metrics for that time period and hostId.
> 
> Dmytro Sen wrote:
>     agree, but maybe it's better to implement it in separate task, and for now just switch off a minute aggregation if in-memory one is configured?

+1


- Sid


-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review171905
-----------------------------------------------------------


On April 13, 2017, 4:54 p.m., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated April 13, 2017, 4:54 p.m.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java 942c0b4 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java bb2f0a9 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java 8ee6d00 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 46f32f9 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java 8ecca54 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java 383079a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java 0bed7d0 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java 18d973c 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 3fdf3f4 
>   ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java bd4ae6a 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java a112ef2 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java 5777639 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 9d492cb 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 9a55f10 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 60c1427 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 535fae0 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java f58f549 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java 17c58f0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java 8b0d84b 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java d049e33 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java aa53430 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java 1446ec2 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java 5310906 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java 7eb2457 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java 634e51d 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 304a8e0 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java 611d82e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b2e8cac 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java 81da5c8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b243e0b 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java bd54bad 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b093a2a 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java 76c1167 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java 46e791b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDataTransferMethodFactory.java 207a79d 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java f517555 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java cceaafb 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java c4c2ddc 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java c1576a3 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java ecb59ac 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java 932345b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java 19e0e60 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheValue.java ba64234 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java f268149 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java 0ba6fb5 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java 366f5a0 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java fbc0b56 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java e5fc4e1 
>   ambari-views/src/test/java/org/apache/ambari/AppTest.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Aggregator.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/AggregatorWebService.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/App.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Sender.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/TimelineMetricsHolder.java PRE-CREATION 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/1/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Aravindan Vijayan <av...@hortonworks.com>.

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review171905
-----------------------------------------------------------




ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
Line 153 (original), 157 (patched)
<https://reviews.apache.org/r/58424/#comment244895>

    Is this needed after this change? 
    
    How do we handle the case where the local monitor is down on some host, and part of the metrics will need to be aggregated at the collecor side and part of them have already been aggregated and written to METRIC_RECORD_MINUTE?


- Aravindan Vijayan


On April 13, 2017, 4:54 p.m., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated April 13, 2017, 4:54 p.m.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java 942c0b4 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java bb2f0a9 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java 8ee6d00 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 46f32f9 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java 8ecca54 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java 383079a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java 0bed7d0 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java 18d973c 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 3fdf3f4 
>   ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java bd4ae6a 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java a112ef2 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java 5777639 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 9d492cb 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 9a55f10 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 60c1427 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 535fae0 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java f58f549 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java 17c58f0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java 8b0d84b 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java d049e33 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java aa53430 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java 1446ec2 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java 5310906 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java 7eb2457 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java 634e51d 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 304a8e0 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java 611d82e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b2e8cac 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java 81da5c8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b243e0b 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java bd54bad 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b093a2a 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java 76c1167 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java 46e791b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDataTransferMethodFactory.java 207a79d 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java f517555 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java cceaafb 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java c4c2ddc 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java c1576a3 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java ecb59ac 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java 932345b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java 19e0e60 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheValue.java ba64234 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java f268149 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java 0ba6fb5 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java 366f5a0 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java fbc0b56 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java e5fc4e1 
>   ambari-views/src/test/java/org/apache/ambari/AppTest.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Aggregator.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/AggregatorWebService.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/App.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Sender.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/TimelineMetricsHolder.java PRE-CREATION 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/1/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>

Re: Review Request 58424: Aggregate local metrics for minute aggregation time window

Posted by Aravindan Vijayan <av...@hortonworks.com>.

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58424/#review171904
-----------------------------------------------------------




ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
Line 498 (original), 498 (patched)
<https://reviews.apache.org/r/58424/#comment244894>

    Yes I agree. 'localhost' can be the default first option added to the set of configured hosts.


- Aravindan Vijayan


On April 13, 2017, 4:54 p.m., Dmytro Sen wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58424/
> -----------------------------------------------------------
> 
> (Updated April 13, 2017, 4:54 p.m.)
> 
> 
> Review request for Ambari, Aravindan Vijayan and Sid Wagle.
> 
> 
> Bugs: AMBARI-20758
>     https://issues.apache.org/jira/browse/AMBARI-20758
> 
> 
> Repository: ambari
> 
> 
> Description
> -------
> 
> Abstract Sink checks if there is local known monitor port available to write (e.g.: 61888)
> Handshake / Ping to make sure its the monitor and is accepting writes
> Start sending metrics only to the monitor
> Monitor sends metrics at the global send interval (1 minute) with all the granularity of the sink
> Monitor aggregates first interval (5 min) data and sends it every interval seconds. (5 min)
> The monitor interface look the same as that of collector (http server: maybe Flask)
> Security concerns need to be addressed as well. (https/spnego?)
> 
> 
> Diffs
> -----
> 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java 942c0b4 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java bb2f0a9 
>   ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java 8ee6d00 
>   ambari-metrics/ambari-metrics-assembly/pom.xml a4b87de 
>   ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml 99a41c3 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java 46f32f9 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregationResult.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricClusterAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/MetricHostAggregate.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java 8ecca54 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java 44c9d4a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricWithAggregatedValues.java PRE-CREATION 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java 383079a 
>   ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java 0bed7d0 
>   ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java 18d973c 
>   ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java 3fdf3f4 
>   ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java bd4ae6a 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java a112ef2 
>   ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java 5777639 
>   ambari-metrics/ambari-metrics-host-aggregator/pom.xml PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py PRE-CREATION 
>   ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py c0feed5 
>   ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java 9d492cb 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 9a55f10 
>   ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java 60c1427 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java 535fae0 
>   ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java f58f549 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java 17c58f0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java 8b0d84b 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java d049e33 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java aa53430 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorSink.java 65d54c0 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java 1446ec2 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricAggregate.java 825ac25 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricClusterAggregate.java 9c837b6 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/MetricHostAggregate.java 340ec75 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java 44aca03 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java 02677b9 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java 5310906 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java 0ea9c08 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b5f49fb 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java 7eb2457 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java 634e51d 
>   ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java 304a8e0 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java 611d82e 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java 0087fd9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java 37ec134 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessorTest.java a910cc2 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestMetricHostAggregate.java 44f48e8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b2e8cac 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsAggregatorMemorySink.java fa0cfe9 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java 81da5c8 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java 590f82a 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java 9873643 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java 78db11d 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b243e0b 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java bd54bad 
>   ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b093a2a 
>   ambari-metrics/pom.xml 2d88912 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java 76c1167 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java 46e791b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDataTransferMethodFactory.java 207a79d 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java f517555 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java cceaafb 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java c4c2ddc 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java c1576a3 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java ecb59ac 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java 932345b 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java 19e0e60 
>   ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheValue.java ba64234 
>   ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java f268149 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java 0ba6fb5 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java 366f5a0 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java fbc0b56 
>   ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java e5fc4e1 
>   ambari-views/src/test/java/org/apache/ambari/AppTest.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Aggregator.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/AggregatorWebService.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/App.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/Sender.java PRE-CREATION 
>   contrib/views/pig/src/main/java/org/apache/ambari/TimelineMetricsHolder.java PRE-CREATION 
> 
> 
> Diff: https://reviews.apache.org/r/58424/diff/1/
> 
> 
> Testing
> -------
> 
> Under development
> 
> 
> Thanks,
> 
> Dmytro Sen
> 
>