You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/11/26 16:36:27 UTC
[3/9] incubator-slider git commit: SLIDER-319 add slf4j reporting and
try to pass this down to the test run. This is not appearing in the
logs...the settings are not getting passed in
SLIDER-319 add slf4j reporting and try to pass this down to the test run. This is not appearing in the logs...the settings are not getting passed in
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/35935db1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/35935db1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/35935db1
Branch: refs/heads/feature/SLIDER-671_publish_to_maven_central
Commit: 35935db198f6fa21d2bb2d188745bc01b3d0d9ce
Parents: 49db62c
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 26 14:10:31 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 26 14:10:31 2014 +0000
----------------------------------------------------------------------
.../management/MetricsBindingService.java | 110 ++++++++++---------
.../appmaster/management/MetricsKeys.java | 93 ++++++++++++++++
.../standalone/TestStandaloneAgentWeb.groovy | 25 ++---
3 files changed, 160 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/35935db1/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
index 7fa102f..e591716 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
@@ -21,57 +21,35 @@ package org.apache.slider.server.appmaster.management;
import com.codahale.metrics.JmxReporter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Slf4jReporter;
import com.codahale.metrics.ganglia.GangliaReporter;
+import com.google.common.base.Preconditions;
import info.ganglia.gmetric4j.gmetric.GMetric;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.slider.server.services.workflow.ClosingService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.concurrent.TimeUnit;
/**
* YARN service which hooks up Codahale metrics to
- * Ganglia (if enabled)
+ * JMX, and, if enabled Ganglia and/or an SLF4J log.
*/
-public class MetricsBindingService extends AbstractService {
-
- /**
- * {@value}
- */
- public static final String METRICS_GANGLIA_ENABLED =
- "metrics.ganglia.enabled";
- /**
- * {@value}
- */
- public static final String METRICS_GANGLIA_HOST = "metrics.ganglia.host";
-
- /**
- * {@value}
- */
- public static final String METRICS_GANGLIA_PORT = "metrics.ganglia.port";
-
- /**
- * {@value}
- */
- public static final String METRICS_GANGLIA_VERSION_31 = "metrics.ganglia.version.31";
-
- /**
- * {@value}
- */
- public static final String METRICS_GANGLIA_REPORT_INTERVAL = "metrics.ganglia.report.interval";
+public class MetricsBindingService extends CompositeService
+ implements MetricsKeys {
+ protected static final Logger log =
+ LoggerFactory.getLogger(MetricsBindingService.class);
+ private final MetricRegistry metrics;
- /**
- * {@value}
- */
- public static final int DEFAULT_GANGLIA_PORT = 8649;
+ private String reportingDetails = "not started";
- private final MetricRegistry metrics;
- private ScheduledReporter gangliaReporter;
- private JmxReporter jmxReporter;
public MetricsBindingService(String name,
MetricRegistry metrics) {
super(name);
+ Preconditions.checkArgument(metrics != null, "Null metrics");
this.metrics = metrics;
}
@@ -83,6 +61,10 @@ public class MetricsBindingService extends AbstractService {
this(name, new MetricRegistry());
}
+ /**
+ * Accessor for the metrics instance
+ * @return the metrics
+ */
public MetricRegistry getMetrics() {
return metrics;
}
@@ -90,16 +72,21 @@ public class MetricsBindingService extends AbstractService {
@Override
protected void serviceStart() throws Exception {
super.serviceStart();
+
+ StringBuilder summary = new StringBuilder();
Configuration conf = getConfig();
-
+
+ summary.append("Reporting to JMX");
// always start the JMX binding
- jmxReporter = JmxReporter.forRegistry(metrics).build();
+ JmxReporter jmxReporter;
+ jmxReporter = JmxReporter.forRegistry(metrics).build();
jmxReporter.start();
-
-
- boolean gangliaEnabled = conf.getBoolean(METRICS_GANGLIA_ENABLED, false);
+ addService(new ClosingService<JmxReporter>(jmxReporter));
+
- if (gangliaEnabled) {
+ // Ganglia
+ if (conf.getBoolean(METRICS_GANGLIA_ENABLED, false)) {
+ GangliaReporter gangliaReporter;
String host = conf.getTrimmed(METRICS_GANGLIA_HOST, "");
int port = conf.getInt(METRICS_GANGLIA_PORT, DEFAULT_GANGLIA_PORT);
int interval = conf.getInt(METRICS_GANGLIA_REPORT_INTERVAL, 60);
@@ -110,24 +97,45 @@ public class MetricsBindingService extends AbstractService {
final GMetric ganglia =
new GMetric(
- host,
+ host,
port,
mcast,
ttl,
ganglia31);
gangliaReporter = GangliaReporter.forRegistry(metrics)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build(ganglia);
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(
+ TimeUnit.MILLISECONDS)
+ .build(ganglia);
gangliaReporter.start(interval, TimeUnit.SECONDS);
+ addService(new ClosingService<ScheduledReporter>(gangliaReporter));
+ summary.append(String.format(", Ganglia at %s:%d interval=%d",
+ host, port, interval));
}
+
+ // Logging
+ if (conf.getBoolean(METRICS_LOGGING_ENABLED, false)) {
+ ScheduledReporter reporter;
+ String logName =
+ conf.getTrimmed(METRICS_LOGGING_LOG, METRICS_DEFAULT_LOG);
+ int interval = conf.getInt(METRICS_LOGGING_LOG_INTERVAL,
+ METRICS_DEFAULT_LOG_INTERVAL);
+ reporter = Slf4jReporter.forRegistry(metrics)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .outputTo(LoggerFactory.getLogger(logName))
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ reporter.start(interval, TimeUnit.SECONDS);
+ addService(new ClosingService<ScheduledReporter>(reporter));
+ summary.append(String.format(", SLF4J to log %s interval=%d",
+ logName, interval));
+ }
+ reportingDetails = summary.toString();
+ log.info(reportingDetails);
}
@Override
- protected void serviceStop() throws Exception {
- IOUtils.closeStream(gangliaReporter);
- IOUtils.closeStream(jmxReporter);
- super.serviceStop();
+ public String toString() {
+ return super.toString() + " " + reportingDetails;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/35935db1/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java
new file mode 100644
index 0000000..8a4b075
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+public interface MetricsKeys {
+
+ /**
+ * Prefix for metrics configuration options: {@value}
+ */
+ String METRICS_PREFIX = "metrics.codahale.";
+
+ /**
+ * Boolean to enable Ganglia metrics reporting
+ * {@value}
+ */
+ String METRICS_GANGLIA_ENABLED =
+ METRICS_PREFIX + "ganglia.enabled";
+ /**
+ * {@value}
+ */
+ String METRICS_GANGLIA_HOST = METRICS_PREFIX + "ganglia.host";
+ /**
+ * {@value}
+ */
+ String METRICS_GANGLIA_PORT = METRICS_PREFIX + "ganglia.port";
+ /**
+ * {@value}
+ */
+ String METRICS_GANGLIA_VERSION_31 = METRICS_PREFIX + "ganglia.version.31";
+ /**
+ * {@value}
+ */
+ String METRICS_GANGLIA_REPORT_INTERVAL = METRICS_PREFIX + "ganglia.report.interval";
+ /**
+ * {@value}
+ */
+ int DEFAULT_GANGLIA_PORT = 8649;
+
+
+ /**
+ * Boolean to enable Logging metrics reporting
+ * {@value}
+ */
+ String METRICS_LOGGING_ENABLED =
+ METRICS_PREFIX + "logging.enabled";
+
+ /**
+ * String name of log to log to
+ * {@value}
+ */
+ String METRICS_LOGGING_LOG =
+ METRICS_PREFIX + "logging.log";
+
+ /**
+ * Default log name: {@value}
+ */
+ String METRICS_DEFAULT_LOG = "org.apache.hadoop.metrics.log";
+
+
+ /**
+ * Int log interval in seconds
+ * {@value}
+ */
+ String METRICS_LOGGING_LOG_INTERVAL =
+ METRICS_PREFIX + "logging.interval";
+
+
+ /**
+ * Default log interval: {@value}.
+ * This is a big interval as in a long lived service, log overflows are easy
+ * to create.
+ */
+ int METRICS_DEFAULT_LOG_INTERVAL = 60 * 10;
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/35935db1/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index e868b8f..1553b2f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -20,29 +20,15 @@ package org.apache.slider.agent.standalone
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.hadoop.yarn.exceptions.YarnException
import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.api.ClusterNode
import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.params.ActionRegistryArgs
-import org.apache.slider.common.tools.Duration
-import org.apache.slider.core.build.InstanceBuilder
-import org.apache.slider.core.conf.AggregateConf
-import org.apache.slider.core.exceptions.SliderException
-import org.apache.slider.core.launch.LaunchedApplication
-import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.persist.LockAcquireFailedException
-import org.apache.slider.core.registry.retrieve.AMWebClient
import org.apache.slider.server.appmaster.web.rest.RestPaths
-import org.junit.After
import org.junit.Test
+import static org.apache.slider.server.appmaster.management.MetricsKeys.*
+
@CompileStatic
@Slf4j
class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
@@ -53,6 +39,9 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
describe "create a standalone AM then perform actions on it"
//launch fake master
+ def configuration = configuration
+ configuration.setBoolean(METRICS_LOGGING_ENABLED, true)
+ configuration.setInt(METRICS_LOGGING_LOG_INTERVAL, 1)
String clustername = createMiniCluster("", configuration, 1, true)
@@ -66,7 +55,8 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
GET(realappmaster)
def metrics = GET(realappmaster, RestPaths.SYSTEM_METRICS)
log.info metrics
-
+
+ sleep(5000)
def appmaster = report.trackingUrl
GET(appmaster)
@@ -74,6 +64,7 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
log.info GET(appmaster, RestPaths.SYSTEM_PING)
log.info GET(appmaster, RestPaths.SYSTEM_THREADS)
log.info GET(appmaster, RestPaths.SYSTEM_HEALTHCHECK)
+ log.info GET(appmaster, RestPaths.SYSTEM_METRICS_JSON)
}