You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2014/11/21 19:16:12 UTC
ambari git commit: AMBARI-5707. Aggregator exceptions due to
misconfiguration.
Repository: ambari
Updated Branches:
refs/heads/branch-metrics-dev dd066b945 -> 063335855
AMBARI-5707. Aggregator exceptions due to misconfiguration.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/06333585
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/06333585
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/06333585
Branch: refs/heads/branch-metrics-dev
Commit: 0633358555468d2c8c5d6d8d30902a82a5036247
Parents: dd066b9
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Fri Nov 21 10:15:58 2014 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Fri Nov 21 10:15:58 2014 -0800
----------------------------------------------------------------------
.../timeline/AbstractTimelineAggregator.java | 19 ++++++++++---------
.../timeline/TimelineMetricAggregatorHourly.java | 11 ++++++-----
.../timeline/TimelineMetricAggregatorMinute.java | 10 ++++++----
.../TimelineMetricClusterAggregator.java | 19 +++++++++++--------
.../TimelineMetricClusterAggregatorHourly.java | 19 ++++++++++---------
.../timeline/TimelineMetricConfiguration.java | 3 ++-
6 files changed, 45 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java
index 6bd6507..43ec648 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java
@@ -33,6 +33,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Date;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
.timeline.TimelineMetricConfiguration.AGGREGATOR_CHECKPOINT_DELAY;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
@@ -42,7 +43,7 @@ public abstract class AbstractTimelineAggregator implements Runnable {
protected final PhoenixHBaseAccessor hBaseAccessor;
private final Log LOG;
private static final ObjectMapper mapper;
- protected final long checkpointDelay;
+ protected final long checkpointDelayMillis;
protected final Integer resultsetFetchSize;
protected Configuration metricsConf;
@@ -54,8 +55,8 @@ public abstract class AbstractTimelineAggregator implements Runnable {
Configuration metricsConf) {
this.hBaseAccessor = hBaseAccessor;
this.metricsConf = metricsConf;
- this.checkpointDelay = metricsConf.getInt(AGGREGATOR_CHECKPOINT_DELAY,
- 120000);
+ this.checkpointDelayMillis = SECONDS.toMillis(
+ metricsConf.getInt(AGGREGATOR_CHECKPOINT_DELAY, 120));
this.resultsetFetchSize = metricsConf.getInt(RESULTSET_FETCH_SIZE, 2000);
this.LOG = LogFactory.getLog(this.getClass());
}
@@ -63,7 +64,7 @@ public abstract class AbstractTimelineAggregator implements Runnable {
@Override
public void run() {
LOG.info("Started Timeline aggregator thread @ " + new Date());
- Long SLEEP_INTERVAL = getSleepInterval();
+ Long SLEEP_INTERVAL = getSleepIntervalMillis();
while (true) {
long currentTime = System.currentTimeMillis();
@@ -80,7 +81,7 @@ public abstract class AbstractTimelineAggregator implements Runnable {
// Assuming first run, save checkpoint and sleep.
// Set checkpoint to 2 minutes in the past to allow the
// agents/collectors to catch up
- saveCheckPoint(currentTime - checkpointDelay);
+ saveCheckPoint(currentTime - checkpointDelayMillis);
}
} catch (IOException io) {
LOG.warn("Unable to write last checkpoint time. Resuming sleep.", io);
@@ -131,7 +132,7 @@ public abstract class AbstractTimelineAggregator implements Runnable {
private boolean isLastCheckPointTooOld(long checkpoint) {
return checkpoint != -1 &&
((System.currentTimeMillis() - checkpoint) >
- getCheckpointCutOffInterval());
+ getCheckpointCutOffIntervalMillis());
}
private long readCheckPoint() {
@@ -164,12 +165,12 @@ public abstract class AbstractTimelineAggregator implements Runnable {
// TODO: Abstract out doWork implementation for cluster and host levels
protected abstract boolean doWork(long startTime, long endTime);
- protected abstract Long getSleepInterval();
+ protected abstract Long getSleepIntervalMillis();
protected abstract Integer getCheckpointCutOffMultiplier();
- protected Long getCheckpointCutOffInterval() {
- return getCheckpointCutOffMultiplier() * getSleepInterval();
+ protected Long getCheckpointCutOffIntervalMillis() {
+ return getCheckpointCutOffMultiplier() * getSleepIntervalMillis();
}
protected abstract boolean isDisabled();
http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java
index 6a84024..16f5ab9 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java
@@ -34,6 +34,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Map;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
.timeline.PhoenixTransactSQL.*;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
@@ -55,7 +56,7 @@ public class TimelineMetricAggregatorHourly extends AbstractTimelineAggregator {
private static final String MINUTE_AGGREGATE_HOURLY_CHECKPOINT_FILE =
"timeline-metrics-host-aggregator-hourly-checkpoint";
private final String checkpointLocation;
- private final Long sleepInterval;
+ private final Long sleepIntervalMillis;
private final Integer checkpointCutOffMultiplier;
public TimelineMetricAggregatorHourly(PhoenixHBaseAccessor hBaseAccessor,
@@ -69,8 +70,8 @@ public class TimelineMetricAggregatorHourly extends AbstractTimelineAggregator {
checkpointLocation = FilenameUtils.concat(checkpointDir,
MINUTE_AGGREGATE_HOURLY_CHECKPOINT_FILE);
- sleepInterval = metricsConf.getLong(HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL,
- 3600000l);
+ sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l));
checkpointCutOffMultiplier =
metricsConf.getInt(HOST_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
}
@@ -181,8 +182,8 @@ public class TimelineMetricAggregatorHourly extends AbstractTimelineAggregator {
}
@Override
- protected Long getSleepInterval() {
- return sleepInterval;
+ protected Long getSleepIntervalMillis() {
+ return sleepIntervalMillis;
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java
index 274d0b5..ac9d12e 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java
@@ -32,6 +32,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Map;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.Condition;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
@@ -48,7 +49,7 @@ public class TimelineMetricAggregatorMinute extends AbstractTimelineAggregator {
private static final String MINUTE_AGGREGATE_CHECKPOINT_FILE =
"timeline-metrics-host-aggregator-checkpoint";
private final String checkpointLocation;
- private final Long sleepInterval;
+ private final Long sleepIntervalMillis;
private final Integer checkpointCutOffMultiplier;
public TimelineMetricAggregatorMinute(PhoenixHBaseAccessor hBaseAccessor,
@@ -61,7 +62,8 @@ public class TimelineMetricAggregatorMinute extends AbstractTimelineAggregator {
checkpointLocation = FilenameUtils.concat(checkpointDir,
MINUTE_AGGREGATE_CHECKPOINT_FILE);
- sleepInterval = metricsConf.getLong(HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300000l); // 5 mins
+ sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300l)); // 5 mins
checkpointCutOffMultiplier =
metricsConf.getInt(HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 3);
}
@@ -163,8 +165,8 @@ public class TimelineMetricAggregatorMinute extends AbstractTimelineAggregator {
}
@Override
- protected Long getSleepInterval() {
- return sleepInterval;
+ protected Long getSleepIntervalMillis() {
+ return sleepIntervalMillis;
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java
index 080703e..c52451e 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java
@@ -34,6 +34,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.Condition;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.GET_METRIC_SQL;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
@@ -54,8 +55,8 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator
private static final String CLUSTER_AGGREGATOR_CHECKPOINT_FILE =
"timeline-metrics-cluster-aggregator-checkpoint";
private final String checkpointLocation;
- private final Long sleepInterval;
- public final int timeSliceInterval;
+ private final Long sleepIntervalMillis;
+ public final int timeSliceIntervalMillis;
private final Integer checkpointCutOffMultiplier;
public TimelineMetricClusterAggregator(PhoenixHBaseAccessor hBaseAccessor,
@@ -68,8 +69,10 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator
checkpointLocation = FilenameUtils.concat(checkpointDir,
CLUSTER_AGGREGATOR_CHECKPOINT_FILE);
- sleepInterval = metricsConf.getLong(CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 120000l);
- timeSliceInterval = metricsConf.getInt(CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL, 15000);
+ sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 120l));
+ timeSliceIntervalMillis = (int)SECONDS.toMillis(metricsConf.getInt
+ (CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL, 15));
checkpointCutOffMultiplier =
metricsConf.getInt(CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
}
@@ -108,8 +111,8 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator
// Create time slices
long sliceStartTime = startTime;
while (sliceStartTime < endTime) {
- timeSlices.add(new Long[] { sliceStartTime, sliceStartTime + timeSliceInterval});
- sliceStartTime += timeSliceInterval;
+ timeSlices.add(new Long[] { sliceStartTime, sliceStartTime + timeSliceIntervalMillis});
+ sliceStartTime += timeSliceIntervalMillis;
}
while (rs.next()) {
@@ -172,8 +175,8 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator
}
@Override
- protected Long getSleepInterval() {
- return sleepInterval;
+ protected Long getSleepIntervalMillis() {
+ return sleepIntervalMillis;
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java
index c380adb..e886b71 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java
@@ -32,6 +32,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Map;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
.timeline.PhoenixHBaseAccessor.getMetricClusterAggregateFromResultSet;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
@@ -60,9 +61,9 @@ public class TimelineMetricClusterAggregatorHourly extends
private static final String CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE =
"timeline-metrics-cluster-aggregator-hourly-checkpoint";
private final String checkpointLocation;
- private final long sleepInterval;
+ private final long sleepIntervalMillis;
private final Integer checkpointCutOffMultiplier;
- private long checkpointCutOffInterval;
+ private long checkpointCutOffIntervalMillis;
public TimelineMetricClusterAggregatorHourly(
PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf) {
@@ -74,9 +75,9 @@ public class TimelineMetricClusterAggregatorHourly extends
checkpointLocation = FilenameUtils.concat(checkpointDir,
CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE);
- sleepInterval = metricsConf.getLong
- (CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600000l);
- checkpointCutOffInterval = 7200000l;
+ sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+ (CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l));
+ checkpointCutOffIntervalMillis = 7200000l;
checkpointCutOffMultiplier = metricsConf.getInt
(CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
}
@@ -202,8 +203,8 @@ public class TimelineMetricClusterAggregatorHourly extends
}
@Override
- protected Long getSleepInterval() {
- return sleepInterval;
+ protected Long getSleepIntervalMillis() {
+ return sleepIntervalMillis;
}
@Override
@@ -212,8 +213,8 @@ public class TimelineMetricClusterAggregatorHourly extends
}
@Override
- protected Long getCheckpointCutOffInterval() {
- return checkpointCutOffInterval;
+ protected Long getCheckpointCutOffIntervalMillis() {
+ return checkpointCutOffIntervalMillis;
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index dd952b4..6b19847 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -21,7 +21,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
- * Configuration class that reads properties from ams-site.xml
+ * Configuration class that reads properties from ams-site.xml. All values
+ * for time or intervals are given in seconds.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving