You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2008/12/16 02:02:52 UTC
svn commit: r726900 [1/2] - in /hadoop/core/trunk: ./
src/core/org/apache/hadoop/ipc/ src/core/org/apache/hadoop/ipc/metrics/
src/core/org/apache/hadoop/metrics/ src/core/org/apache/hadoop/metrics/util/
src/hdfs/org/apache/hadoop/hdfs/server/datanode/ ...
Author: acmurthy
Date: Mon Dec 15 17:02:51 2008
New Revision: 726900
URL: http://svn.apache.org/viewvc?rev=726900&view=rev
Log:
HADOOP-4838. Added a registry to automate metrics and mbeans management. Contributed by Sanjay Radia.
Added:
hadoop/core/trunk/src/core/org/apache/hadoop/ipc/metrics/RpcActivityMBean.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsBase.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsRegistry.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingLong.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeActivityMBean.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeActivtyMBean.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/core/org/apache/hadoop/ipc/RPC.java
hadoop/core/trunk/src/core/org/apache/hadoop/ipc/metrics/RpcMetrics.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsUtil.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MBeanUtil.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsIntValue.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsLongValue.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingInt.java
hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingRate.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeStatistics.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeStatisticsMBean.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMetrics.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatistics.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatisticsMBean.java
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Dec 15 17:02:51 2008
@@ -258,6 +258,9 @@
HADOOP-4699. Remove checksum validation from map output servlet. (cdouglas)
+ HADOOP-4838. Added a registry to automate metrics and mbeans management.
+ (Sanjay Radia via acmurthy)
+
OPTIMIZATIONS
HADOOP-3293. Fixes FileInputFormat to do provide locations for splits
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/ipc/RPC.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/ipc/RPC.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/ipc/RPC.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/ipc/RPC.java Mon Dec 15 17:02:51 2008
@@ -516,16 +516,13 @@
rpcMetrics.rpcQueueTime.inc(qTime);
rpcMetrics.rpcProcessingTime.inc(processingTime);
- MetricsTimeVaryingRate m = rpcMetrics.metricsList.get(call.getMethodName());
-
- if (m != null) {
- m.inc(processingTime);
- }
- else {
- rpcMetrics.metricsList.put(call.getMethodName(), new MetricsTimeVaryingRate(call.getMethodName()));
- m = rpcMetrics.metricsList.get(call.getMethodName());
- m.inc(processingTime);
- }
+ MetricsTimeVaryingRate m =
+ (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
+ if (m == null) {
+ m = new MetricsTimeVaryingRate(call.getMethodName(),
+ rpcMetrics.registry);
+ }
+ m.inc(processingTime);
if (verbose) log("Return: "+value);
Added: hadoop/core/trunk/src/core/org/apache/hadoop/ipc/metrics/RpcActivityMBean.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/ipc/metrics/RpcActivityMBean.java?rev=726900&view=auto
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/ipc/metrics/RpcActivityMBean.java (added)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/ipc/metrics/RpcActivityMBean.java Mon Dec 15 17:02:51 2008
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc.metrics;
+
+import javax.management.ObjectName;
+
+import org.apache.hadoop.metrics.util.MBeanUtil;
+import org.apache.hadoop.metrics.util.MetricsDynamicMBeanBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+
+
+/**
+ *
+ * This is the JMX MBean for reporting the RPC layer Activity.
+ * The MBean is register using the name
+ * "hadoop:service=<RpcServiceName>,name=RpcActivityForPort<port>"
+ *
+ * Many of the activity metrics are sampled and averaged on an interval
+ * which can be specified in the metrics config file.
+ * <p>
+ * For the metrics that are sampled and averaged, one must specify
+ * a metrics context that does periodic update calls. Most metrics contexts do.
+ * The default Null metrics context however does NOT. So if you aren't
+ * using any other metrics context then you can turn on the viewing and averaging
+ * of sampled metrics by specifying the following two lines
+ * in the hadoop-meterics.properties file:
+ * <pre>
+ * rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+ * rpc.period=10
+ * </pre>
+ *<p>
+ * Note that the metrics are collected regardless of the context used.
+ * The context with the update thread is used to average the data periodically
+ *
+ *
+ *
+ * Impl details: We use a dynamic mbean that gets the list of the metrics
+ * from the metrics registry passed as an argument to the constructor
+ */
+
+public class RpcActivityMBean extends MetricsDynamicMBeanBase {
+ final private ObjectName mbeanName;
+
+ /**
+ *
+ * @param mr - the metrics registry that has all the metrics
+ * @param serviceName - the service name for the rpc service
+ * @param port - the rpc port.
+ */
+ public RpcActivityMBean(final MetricsRegistry mr, final String serviceName, final String port) {
+
+
+ super(mr, "Rpc layer statistics");
+ mbeanName = MBeanUtil.registerMBean(serviceName,
+ "RpcActivityForPort" + port, this);
+ }
+
+
+ public void shutdown() {
+ if (mbeanName != null)
+ MBeanUtil.unregisterMBean(mbeanName);
+ }
+
+}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/ipc/metrics/RpcMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/ipc/metrics/RpcMetrics.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/ipc/metrics/RpcMetrics.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/ipc/metrics/RpcMetrics.java Mon Dec 15 17:02:51 2008
@@ -24,12 +24,11 @@
import org.apache.hadoop.metrics.MetricsRecord;
import org.apache.hadoop.metrics.MetricsUtil;
import org.apache.hadoop.metrics.Updater;
-import org.apache.hadoop.metrics.jvm.JvmMetrics;
+import org.apache.hadoop.metrics.util.MetricsBase;
+import org.apache.hadoop.metrics.util.MetricsIntValue;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
-import java.util.HashMap;
-import java.util.*;
-
/**
*
* This class is for maintaining the various RPC statistics
@@ -43,11 +42,14 @@
*
*/
public class RpcMetrics implements Updater {
+ public MetricsRegistry registry = new MetricsRegistry();
private MetricsRecord metricsRecord;
+ private Server myServer;
private static Log LOG = LogFactory.getLog(RpcMetrics.class);
- RpcMgt rpcMgt;
+ RpcActivityMBean rpcMBean;
public RpcMetrics(String hostName, String port, Server server) {
+ myServer = server;
MetricsContext context = MetricsUtil.getContext("rpc");
metricsRecord = MetricsUtil.createRecord(context, "metrics");
@@ -59,7 +61,7 @@
context.registerUpdater(this);
// Need to clean up the interface to RpcMgt - don't need both metrics and server params
- rpcMgt = new RpcMgt(hostName, port, this, server);
+ rpcMBean = new RpcActivityMBean(registry, hostName, port);
}
@@ -68,39 +70,35 @@
* - they can be set directly by calling their set/inc methods
* -they can also be read directly - e.g. JMX does this.
*/
-
- public MetricsTimeVaryingRate rpcQueueTime = new MetricsTimeVaryingRate("RpcQueueTime");
- public MetricsTimeVaryingRate rpcProcessingTime = new MetricsTimeVaryingRate("RpcProcessingTime");
-
- public Map <String, MetricsTimeVaryingRate> metricsList = Collections.synchronizedMap(new HashMap<String, MetricsTimeVaryingRate>());
-
+ public MetricsTimeVaryingRate rpcQueueTime =
+ new MetricsTimeVaryingRate("RpcQueueTime", registry);
+ public MetricsTimeVaryingRate rpcProcessingTime =
+ new MetricsTimeVaryingRate("RpcProcessingTime", registry);
+ public MetricsIntValue numOpenConnections =
+ new MetricsIntValue("NumOpenConnections", registry);
+ public MetricsIntValue callQueueLen =
+ new MetricsIntValue("callQueueLen", registry);
/**
* Push the metrics to the monitoring subsystem on doUpdate() call.
*/
public void doUpdates(MetricsContext context) {
- rpcQueueTime.pushMetric(metricsRecord);
- rpcProcessingTime.pushMetric(metricsRecord);
-
- synchronized (metricsList) {
- // Iterate through the rpcMetrics hashmap to propogate the different rpc metrics.
- Set keys = metricsList.keySet();
-
- Iterator keyIter = keys.iterator();
-
- while (keyIter.hasNext()) {
- Object key = keyIter.next();
- MetricsTimeVaryingRate value = metricsList.get(key);
-
- value.pushMetric(metricsRecord);
- }
+
+ synchronized (this) {
+ // ToFix - fix server to use the following two metrics directly so
+ // the metrics do not have be copied here.
+ numOpenConnections.set(myServer.getNumOpenConnections());
+ callQueueLen.set(myServer.getCallQueueLen());
+ for (MetricsBase m : registry.getMetricsList()) {
+ m.pushMetric(metricsRecord);
+ }
}
metricsRecord.update();
}
public void shutdown() {
- if (rpcMgt != null)
- rpcMgt.shutdown();
+ if (rpcMBean != null)
+ rpcMBean.shutdown();
}
}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java Mon Dec 15 17:02:51 2008
@@ -98,5 +98,10 @@
* @param updater object to be removed from the callback list
*/
public abstract void unregisterUpdater(Updater updater);
+
+ /**
+ * Returns the timer period.
+ */
+ public abstract int getPeriod();
}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsUtil.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsUtil.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsUtil.java Mon Dec 15 17:02:51 2008
@@ -33,7 +33,7 @@
*/
public class MetricsUtil {
- private static final Log LOG =
+ public static final Log LOG =
LogFactory.getLog(MetricsUtil.class);
/**
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MBeanUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MBeanUtil.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MBeanUtil.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MBeanUtil.java Mon Dec 15 17:02:51 2008
@@ -35,8 +35,8 @@
public class MBeanUtil {
/**
- * Register the mbean using out standard MBeanName format
- * "hadoop.dfs:service=<serviceName>,name=<nameName>"
+ * Register the MBean using our standard MBeanName format
+ * "hadoop:service=<serviceName>,name=<nameName>"
* Where the <serviceName> and <nameName> are the supplied parameters
*
* @param serviceName
@@ -77,7 +77,7 @@
final String nameName) {
ObjectName name = null;
try {
- name = new ObjectName("hadoop.dfs:" +
+ name = new ObjectName("hadoop:" +
"service=" + serviceName + ",name=" + nameName);
} catch (MalformedObjectNameException e) {
e.printStackTrace();
Added: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsBase.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsBase.java?rev=726900&view=auto
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsBase.java (added)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsBase.java Mon Dec 15 17:02:51 2008
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics.util;
+
+import org.apache.hadoop.metrics.MetricsRecord;
+
+/**
+ *
+ * This is base class for all metrics
+ *
+ */
+public abstract class MetricsBase {
+ public static final String NO_DESCRIPTION = "NoDescription";
+ final private String name;
+ final private String description;
+
+ protected MetricsBase(final String nam) {
+ name = nam;
+ description = NO_DESCRIPTION;
+ }
+
+ protected MetricsBase(final String nam, final String desc) {
+ name = nam;
+ description = desc;
+ }
+
+ public abstract void pushMetric(final MetricsRecord mr);
+
+ public String getName() { return name; }
+ public String getDescription() { return description; };
+
+}
Added: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java?rev=726900&view=auto
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java (added)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java Mon Dec 15 17:02:51 2008
@@ -0,0 +1,226 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics.util;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.management.Attribute;
+import javax.management.AttributeList;
+import javax.management.AttributeNotFoundException;
+import javax.management.DynamicMBean;
+import javax.management.InvalidAttributeValueException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanException;
+import javax.management.MBeanInfo;
+import javax.management.MBeanOperationInfo;
+import javax.management.ReflectionException;
+
+import org.apache.hadoop.metrics.MetricsUtil;
+
+
+
+/**
+ * This abstract base class facilitates creating dynamic mbeans automatically from
+ * metrics.
+ * The metrics constructors registers metrics in a registry.
+ * Different categories of metrics should be in differnt classes with their own
+ * registry (as in NameNodeMetrics and DataNodeMetrics).
+ * Then the MBean can be created passing the registry to the constructor.
+ * The MBean should be then registered using a mbean name (example):
+ * MetricsHolder myMetrics = new MetricsHolder(); // has metrics and registry
+ * MetricsTestMBean theMBean = new MetricsTestMBean(myMetrics.mregistry);
+ * ObjectName mbeanName = MBeanUtil.registerMBean("ServiceFoo",
+ * "TestStatistics", theMBean);
+ *
+ *
+ */
+public abstract class MetricsDynamicMBeanBase implements DynamicMBean {
+ private final static String AVG_TIME = "AvgTime";
+ private final static String MIN_TIME = "MinTime";
+ private final static String MAX_TIME = "MaxTime";
+ private final static String NUM_OPS = "NumOps";
+ private final static String RESET_ALL_MIN_MAX_OP = "resetAllMinMax";
+ private MetricsRegistry metricsRegistry;
+ private MBeanInfo mbeanInfo;
+ private Map<String, MetricsBase> metricsRateAttributeMod;
+ private int numEntriesInRegistry = 0;
+ private String mbeanDescription;
+
+ protected MetricsDynamicMBeanBase(final MetricsRegistry mr, final String aMBeanDescription) {
+ metricsRegistry = mr;
+ mbeanDescription = aMBeanDescription;
+ createMBeanInfo();
+ }
+
+ private void updateMbeanInfoIfMetricsListChanged() {
+ if (numEntriesInRegistry != metricsRegistry.size())
+ createMBeanInfo();
+ }
+
+ private void createMBeanInfo() {
+ metricsRateAttributeMod = new HashMap<String, MetricsBase>();
+ boolean needsMinMaxResetOperation = false;
+ List<MBeanAttributeInfo> attributesInfo = new ArrayList<MBeanAttributeInfo>();
+ MBeanOperationInfo[] operationsInfo = null;
+ numEntriesInRegistry = metricsRegistry.size();
+
+ for (MetricsBase o : metricsRegistry.getMetricsList()) {
+
+ if (MetricsTimeVaryingRate.class.isInstance(o)) {
+ // For each of the metrics there are 3 different attributes
+ attributesInfo.add(new MBeanAttributeInfo(o.getName() + NUM_OPS, "java.lang.Integer",
+ o.getDescription(), true, false, false));
+ attributesInfo.add(new MBeanAttributeInfo(o.getName() + AVG_TIME, "java.lang.Long",
+ o.getDescription(), true, false, false));
+ attributesInfo.add(new MBeanAttributeInfo(o.getName() + MIN_TIME, "java.lang.Long",
+ o.getDescription(), true, false, false));
+ attributesInfo.add(new MBeanAttributeInfo(o.getName() + MAX_TIME, "java.lang.Long",
+ o.getDescription(), true, false, false));
+ needsMinMaxResetOperation = true; // the min and max can be reset.
+
+ // Note the special attributes (AVG_TIME, MIN_TIME, ..) are derived from metrics
+ // Rather than check for the suffix we store them in a map.
+ metricsRateAttributeMod.put(o.getName() + NUM_OPS, o);
+ metricsRateAttributeMod.put(o.getName() + AVG_TIME, o);
+ metricsRateAttributeMod.put(o.getName() + MIN_TIME, o);
+ metricsRateAttributeMod.put(o.getName() + MAX_TIME, o);
+
+ } else if ( MetricsIntValue.class.isInstance(o) || MetricsTimeVaryingInt.class.isInstance(o) ) {
+ attributesInfo.add(new MBeanAttributeInfo(o.getName(), "java.lang.Integer",
+ o.getDescription(), true, false, false));
+ } else if ( MetricsLongValue.class.isInstance(o) || MetricsTimeVaryingLong.class.isInstance(o) ) {
+ attributesInfo.add(new MBeanAttributeInfo(o.getName(), "java.lang.Long",
+ o.getDescription(), true, false, false));
+ } else {
+ MetricsUtil.LOG.error("unknown metrics type: " + o.getClass().getName());
+ }
+
+ if (needsMinMaxResetOperation) {
+ operationsInfo = new MBeanOperationInfo[] {
+ new MBeanOperationInfo(RESET_ALL_MIN_MAX_OP, "Reset (zero) All Min Max",
+ null, "void", MBeanOperationInfo.ACTION) };
+ }
+ }
+ MBeanAttributeInfo[] attrArray = new MBeanAttributeInfo[attributesInfo.size()];
+ mbeanInfo = new MBeanInfo(this.getClass().getName(), mbeanDescription,
+ attributesInfo.toArray(attrArray), null, operationsInfo, null);
+ }
+
+ @Override
+ public Object getAttribute(String attributeName) throws AttributeNotFoundException,
+ MBeanException, ReflectionException {
+ if (attributeName == null || attributeName.equals(""))
+ throw new IllegalArgumentException();
+
+ updateMbeanInfoIfMetricsListChanged();
+
+ Object o = metricsRateAttributeMod.get(attributeName);
+ if (o == null) {
+ o = metricsRegistry.get(attributeName);
+ }
+ if (o == null)
+ throw new AttributeNotFoundException();
+
+ if (o instanceof MetricsIntValue)
+ return ((MetricsIntValue) o).get();
+ else if (o instanceof MetricsLongValue)
+ return ((MetricsLongValue) o).get();
+ else if (o instanceof MetricsTimeVaryingInt)
+ return ((MetricsTimeVaryingInt) o).getPreviousIntervalValue();
+ else if (o instanceof MetricsTimeVaryingLong)
+ return ((MetricsTimeVaryingLong) o).getPreviousIntervalValue();
+ else if (o instanceof MetricsTimeVaryingRate) {
+ MetricsTimeVaryingRate or = (MetricsTimeVaryingRate) o;
+ if (attributeName.endsWith(NUM_OPS))
+ return or.getPreviousIntervalNumOps();
+ else if (attributeName.endsWith(AVG_TIME))
+ return or.getPreviousIntervalAverageTime();
+ else if (attributeName.endsWith(MIN_TIME))
+ return or.getMinTime();
+ else if (attributeName.endsWith(MAX_TIME))
+ return or.getMaxTime();
+ else {
+ MetricsUtil.LOG.error("Unexpected attrubute suffix");
+ throw new AttributeNotFoundException();
+ }
+ } else {
+ MetricsUtil.LOG.error("unknown metrics type: " + o.getClass().getName());
+ throw new AttributeNotFoundException();
+ }
+ }
+
+ @Override
+ public AttributeList getAttributes(String[] attributeNames) {
+ if (attributeNames == null || attributeNames.length == 0)
+ throw new IllegalArgumentException();
+
+ updateMbeanInfoIfMetricsListChanged();
+
+ AttributeList result = new AttributeList(attributeNames.length);
+ for (String iAttributeName : attributeNames) {
+ try {
+ Object value = getAttribute(iAttributeName);
+ result.add(new Attribute(iAttributeName, value));
+ } catch (Exception e) {
+ continue;
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public MBeanInfo getMBeanInfo() {
+ return mbeanInfo;
+ }
+
+ @Override
+ public Object invoke(String actionName, Object[] parms, String[] signature)
+ throws MBeanException, ReflectionException {
+
+ if (actionName == null || actionName.equals(""))
+ throw new IllegalArgumentException();
+
+
+ // Right now we support only one fixed operation (if it applies)
+ if (!(actionName.equals(RESET_ALL_MIN_MAX_OP)) ||
+ mbeanInfo.getOperations().length != 1) {
+ throw new ReflectionException(new NoSuchMethodException(actionName));
+ }
+ for (MetricsBase m : metricsRegistry.getMetricsList()) {
+ if ( MetricsTimeVaryingRate.class.isInstance(m) ) {
+ MetricsTimeVaryingRate.class.cast(m).resetMinMax();
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void setAttribute(Attribute attribute)
+ throws AttributeNotFoundException, InvalidAttributeValueException,
+ MBeanException, ReflectionException {
+ throw new ReflectionException(new NoSuchMethodException("set" + attribute));
+ }
+
+ @Override
+ public AttributeList setAttributes(AttributeList attributes) {
+ return null;
+ }
+}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsIntValue.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsIntValue.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsIntValue.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsIntValue.java Mon Dec 15 17:02:51 2008
@@ -30,25 +30,39 @@
* call.
*
*/
-public class MetricsIntValue {
+public class MetricsIntValue extends MetricsBase {
private static final Log LOG =
LogFactory.getLog("org.apache.hadoop.metrics.util");
- private String name;
private int value;
private boolean changed;
+
/**
* Constructor - create a new metric
* @param nam the name of the metrics to be used to publish the metric
+ * @param registry - where the metrics object will be registered
*/
- public MetricsIntValue(final String nam) {
- name = nam;
+ public MetricsIntValue(final String nam, final MetricsRegistry registry, final String description) {
+ super(nam, description);
value = 0;
changed = false;
+ registry.add(nam, this);
+ }
+
+ /**
+ * Constructor - create a new metric
+ * @param nam the name of the metrics to be used to publish the metric
+ * @param registry - where the metrics object will be registered
+ * A description of {@link #NO_DESCRIPTION} is used
+ */
+ public MetricsIntValue(final String nam, MetricsRegistry registry) {
+ this(nam, registry, NO_DESCRIPTION);
}
+
+
/**
* Set the value
* @param newValue
@@ -65,44 +79,7 @@
public synchronized int get() {
return value;
}
-
- /**
- * Inc metrics for incr vlaue
- * @param incr - value to be added
- */
- public synchronized void inc(final int incr) {
- value += incr;
- changed = true;
- }
- /**
- * Inc metrics by one
- */
- public synchronized void inc() {
- value++;
- changed = true;
- }
-
- /**
- * Inc metrics for incr vlaue
- * @param decr - value to subtract
- */
- public synchronized void dec(final int decr) {
- value -= decr;
- if (value < 0)
- value = 0;
- changed = true;
- }
-
- /**
- * Dec metrics by one
- */
- public synchronized void dec() {
- value--;
- if (value < 0)
- value = 0;
- changed = true;
- }
/**
* Push the metric to the mr.
@@ -116,9 +93,9 @@
public synchronized void pushMetric(final MetricsRecord mr) {
if (changed) {
try {
- mr.setMetric(name, value);
+ mr.setMetric(getName(), value);
} catch (Exception e) {
- LOG.info("pushMetric failed for " + name + "\n" +
+ LOG.info("pushMetric failed for " + getName() + "\n" +
StringUtils.stringifyException(e));
}
}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsLongValue.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsLongValue.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsLongValue.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsLongValue.java Mon Dec 15 17:02:51 2008
@@ -27,19 +27,30 @@
* call.
*
*/
-public class MetricsLongValue {
- private String name;
+public class MetricsLongValue extends MetricsBase{
private long value;
private boolean changed;
/**
* Constructor - create a new metric
* @param nam the name of the metrics to be used to publish the metric
+ * @param registry - where the metrics object will be registered
*/
- public MetricsLongValue(final String nam) {
- name = nam;
+ public MetricsLongValue(final String nam, final MetricsRegistry registry, final String description) {
+ super(nam, description);
value = 0;
changed = false;
+ registry.add(nam, this);
+ }
+
+ /**
+ * Constructor - create a new metric
+ * @param nam the name of the metrics to be used to publish the metric
+ * @param registry - where the metrics object will be registered
+ * A description of {@link #NO_DESCRIPTION} is used
+ */
+ public MetricsLongValue(final String nam, MetricsRegistry registry) {
+ this(nam, registry, NO_DESCRIPTION);
}
/**
@@ -58,44 +69,7 @@
public synchronized long get() {
return value;
}
-
- /**
- * Inc metrics for incr vlaue
- * @param incr - value to be added
- */
- public synchronized void inc(final long incr) {
- value += incr;
- changed = true;
- }
-
- /**
- * Inc metrics by one
- */
- public synchronized void inc() {
- value++;
- changed = true;
- }
-
- /**
- * Inc metrics for incr vlaue
- * @param decr - value to subtract
- */
- public synchronized void dec(final long decr) {
- value -= decr;
- if (value < 0)
- value = 0;
- changed = true;
- }
-
- /**
- * Dec metrics by one
- */
- public synchronized void dec() {
- value--;
- if (value < 0)
- value = 0;
- changed = true;
- }
+
/**
* Push the metric to the mr.
@@ -108,7 +82,7 @@
*/
public synchronized void pushMetric(final MetricsRecord mr) {
if (changed)
- mr.setMetric(name, value);
+ mr.setMetric(getName(), value);
changed = false;
}
}
Added: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsRegistry.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsRegistry.java?rev=726900&view=auto
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsRegistry.java (added)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsRegistry.java Mon Dec 15 17:02:51 2008
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics.util;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+
+/**
+ *
+ * This is the registry for metrics.
+ * Related set of metrics should be declared in a holding class and registered
+ * in a registry for those metrics which is also stored in the the holding class.
+ *
+ */
+public class MetricsRegistry {
+ private Map<String, MetricsBase> metricsList = new HashMap<String, MetricsBase>();
+
+ public MetricsRegistry() {
+ }
+
+ /**
+ *
+ * @return number of metrics in the registry
+ */
+ public int size() {
+ return metricsList.size();
+ }
+
+ /**
+ * Add a new metrics to the registry
+ * @param metricsName - the name
+ * @param theMetricsObj - the metrics
+ * @throws IllegalArgumentException if a name is already registered
+ */
+ public synchronized void add(final String metricsName, final MetricsBase theMetricsObj) {
+ if (metricsList.containsKey(metricsName)) {
+ throw new IllegalArgumentException("Duplicate metricsName:" + metricsName);
+ }
+ metricsList.put(metricsName, theMetricsObj);
+ }
+
+
+ /**
+ *
+ * @param metricsName
+ * @return the metrics if there is one registered by the supplied name.
+ * Returns null if none is registered
+ */
+ public synchronized MetricsBase get(final String metricsName) {
+ return metricsList.get(metricsName);
+ }
+
+
+ /**
+ *
+ * @return the list of metrics names
+ */
+ public synchronized Collection<String> getKeyList() {
+ return metricsList.keySet();
+ }
+
+ /**
+ *
+ * @return the list of metrics
+ */
+ public synchronized Collection<MetricsBase> getMetricsList() {
+ return metricsList.values();
+ }
+}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingInt.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingInt.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingInt.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingInt.java Mon Dec 15 17:02:51 2008
@@ -25,32 +25,51 @@
/**
* The MetricsTimeVaryingInt class is for a metric that naturally
- * varies over time (e.g. number of files created).
- * The metric is is published at interval heart beat (the interval
- * is set in the metrics config file).
+ * varies over time (e.g. number of files created). The metrics is accumulated
+ * over an interval (set in the metrics config file); the metrics is
+ * published at the end of each interval and then
+ * reset to zero. Hence the counter has the value in the current interval.
+ *
* Note if one wants a time associated with the metric then use
* @see org.apache.hadoop.metrics.util.MetricsTimeVaryingRate
*
*/
-public class MetricsTimeVaryingInt {
+public class MetricsTimeVaryingInt extends MetricsBase {
private static final Log LOG =
LogFactory.getLog("org.apache.hadoop.metrics.util");
- private String name;
private int currentValue;
private int previousIntervalValue;
+
/**
* Constructor - create a new metric
* @param nam the name of the metrics to be used to publish the metric
+ * @param registry - where the metrics object will be registered
+ * @param description - the description
*/
- public MetricsTimeVaryingInt(final String nam) {
- name = nam;
+ public MetricsTimeVaryingInt(final String nam,
+ final MetricsRegistry registry,
+ final String description) {
+ super(nam, description);
currentValue = 0;
previousIntervalValue = 0;
+ registry.add(nam, this);
+ }
+
+ /**
+ * Constructor - create a new metric
+ * @param nam the name of the metrics to be used to publish the metric
+ * @param registry - where the metrics object will be registered
+ * A description of {@link #NO_DESCRIPTION} is used
+ */
+ public MetricsTimeVaryingInt(final String nam, final MetricsRegistry registry) {
+ this(nam, registry, NO_DESCRIPTION);
}
+
+
/**
* Inc metrics for incr vlaue
* @param incr - number of operations
@@ -83,9 +102,9 @@
public synchronized void pushMetric(final MetricsRecord mr) {
intervalHeartBeat();
try {
- mr.incrMetric(name, getPreviousIntervalValue());
+ mr.incrMetric(getName(), getPreviousIntervalValue());
} catch (Exception e) {
- LOG.info("pushMetric failed for " + name + "\n" +
+ LOG.info("pushMetric failed for " + getName() + "\n" +
StringUtils.stringifyException(e));
}
}
@@ -97,5 +116,13 @@
*/
public synchronized int getPreviousIntervalValue() {
return previousIntervalValue;
+ }
+
+ /**
+ * The Value at the current interval
+ * @return prev interval value
+ */
+ public synchronized int getCurrentIntervalValue() {
+ return currentValue;
}
}
Added: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingLong.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingLong.java?rev=726900&view=auto
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingLong.java (added)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingLong.java Mon Dec 15 17:02:51 2008
@@ -0,0 +1,124 @@
+package org.apache.hadoop.metrics.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.util.StringUtils;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The MetricsTimeVaryingLong class is for a metric that naturally
+ * varies over time (e.g. number of files created). The metrics is accumulated
+ * over an interval (set in the metrics config file); the metrics is
+ * published at the end of each interval and then
+ * reset to zero. Hence the counter has the value in the current interval.
+ *
+ * Note if one wants a time associated with the metric then use
+ * @see org.apache.hadoop.metrics.util.MetricsTimeVaryingRate
+ *
+ */
+public class MetricsTimeVaryingLong extends MetricsBase{
+
+ private static final Log LOG =
+ LogFactory.getLog("org.apache.hadoop.metrics.util");
+
+ private long currentValue;
+ private long previousIntervalValue;
+
+ /**
+ * Constructor - create a new metric
+ * @param nam the name of the metrics to be used to publish the metric
+ * @param registry - where the metrics object will be registered
+ */
+ public MetricsTimeVaryingLong(final String nam, MetricsRegistry registry, final String description) {
+ super(nam, description);
+ currentValue = 0;
+ previousIntervalValue = 0;
+ registry.add(nam, this);
+ }
+
+
+ /**
+ * Constructor - create a new metric
+ * @param nam the name of the metrics to be used to publish the metric
+ * @param registry - where the metrics object will be registered
+ * A description of {@link #NO_DESCRIPTION} is used
+ */
+ public MetricsTimeVaryingLong(final String nam, MetricsRegistry registry) {
+ this(nam, registry, NO_DESCRIPTION);
+ }
+
+ /**
+ * Inc metrics for incr vlaue
+ * @param incr - number of operations
+ */
+ public synchronized void inc(final long incr) {
+ currentValue += incr;
+ }
+
+ /**
+ * Inc metrics by one
+ */
+ public synchronized void inc() {
+ currentValue++;
+ }
+
+ private synchronized void intervalHeartBeat() {
+ previousIntervalValue = currentValue;
+ currentValue = 0;
+ }
+
+ /**
+ * Push the delta metrics to the mr.
+ * The delta is since the last push/interval.
+ *
+ * Note this does NOT push to JMX
+ * (JMX gets the info via {@link #previousIntervalValue}
+ *
+ * @param mr
+ */
+ public synchronized void pushMetric(final MetricsRecord mr) {
+ intervalHeartBeat();
+ try {
+ mr.incrMetric(getName(), getPreviousIntervalValue());
+ } catch (Exception e) {
+ LOG.info("pushMetric failed for " + getName() + "\n" +
+ StringUtils.stringifyException(e));
+ }
+ }
+
+
+ /**
+ * The Value at the Previous interval
+ * @return prev interval value
+ */
+ public synchronized long getPreviousIntervalValue() {
+ return previousIntervalValue;
+ }
+
+ /**
+ * The Value at the current interval
+ * @return prev interval value
+ */
+ public synchronized long getCurrentIntervalValue() {
+ return currentValue;
+ }
+}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingRate.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingRate.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingRate.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/util/MetricsTimeVaryingRate.java Mon Dec 15 17:02:51 2008
@@ -32,7 +32,7 @@
* a method to reset the min-max.
*
*/
-public class MetricsTimeVaryingRate {
+public class MetricsTimeVaryingRate extends MetricsBase {
private static final Log LOG =
LogFactory.getLog("org.apache.hadoop.metrics.util");
@@ -71,7 +71,6 @@
maxTime = Math.max(maxTime, time);
}
}
- private String name;
private Metrics currentData;
private Metrics previousIntervalData;
private MinMax minMax;
@@ -79,13 +78,26 @@
/**
* Constructor - create a new metric
- * @param n the name of the metrics to be used to publish the metric
+ * @param nam the name of the metrics to be used to publish the metric
+ * @param registry - where the metrics object will be registered
*/
- public MetricsTimeVaryingRate(final String n) {
- name = n;
+ public MetricsTimeVaryingRate(final String nam, final MetricsRegistry registry, final String description) {
+ super(nam, description);
currentData = new Metrics();
previousIntervalData = new Metrics();
minMax = new MinMax();
+ registry.add(nam, this);
+ }
+
+ /**
+ * Constructor - create a new metric
+ * @param nam the name of the metrics to be used to publish the metric
+ * @param registry - where the metrics object will be registered
+ * A description of {@link #NO_DESCRIPTION} is used
+ */
+ public MetricsTimeVaryingRate(final String nam, MetricsRegistry registry) {
+ this(nam, registry, NO_DESCRIPTION);
+
}
@@ -133,10 +145,10 @@
public synchronized void pushMetric(final MetricsRecord mr) {
intervalHeartBeat();
try {
- mr.incrMetric(name + "_num_ops", getPreviousIntervalNumOps());
- mr.setMetric(name + "_avg_time", getPreviousIntervalAverageTime());
+ mr.incrMetric(getName() + "_num_ops", getPreviousIntervalNumOps());
+ mr.setMetric(getName() + "_avg_time", getPreviousIntervalAverageTime());
} catch (Exception e) {
- LOG.info("pushMetric failed for " + name + "\n" +
+ LOG.info("pushMetric failed for " + getName() + "\n" +
StringUtils.stringifyException(e));
}
}
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java Mon Dec 15 17:02:51 2008
@@ -1383,23 +1383,25 @@
private ObjectName mbeanName;
private Random rand = new Random();
+
/**
- * Register the FSDataset MBean
+ * Register the FSDataset MBean using the name
+ * "hadoop:service=DataNode,name=FSDatasetState-<storageid>"
*/
void registerMBean(final String storageId) {
// We wrap to bypass standard mbean naming convetion.
// This wraping can be removed in java 6 as it is more flexible in
// package naming for mbeans and their impl.
StandardMBean bean;
- String serverName;
- if (storageId.equals("")) {// Temp fix for the uninitialized storage
- serverName = "DataNode-UndefinedStorageId" + rand.nextInt();
+ String storageName;
+ if (storageId == null || storageId.equals("")) {// Temp fix for the uninitialized storage
+ storageName = "UndefinedStorageId" + rand.nextInt();
} else {
- serverName = "DataNode-" + storageId;
+ storageName = storageId;
}
try {
bean = new StandardMBean(this,FSDatasetMBean.class);
- mbeanName = MBeanUtil.registerMBean(serverName, "FSDatasetStatus", bean);
+ mbeanName = MBeanUtil.registerMBean("DataNode", "FSDatasetState-" + storageName, bean);
} catch (NotCompliantMBeanException e) {
e.printStackTrace();
}
Added: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeActivityMBean.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeActivityMBean.java?rev=726900&view=auto
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeActivityMBean.java (added)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeActivityMBean.java Mon Dec 15 17:02:51 2008
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.metrics;
+import java.util.Random;
+
+import javax.management.ObjectName;
+import org.apache.hadoop.metrics.util.MBeanUtil;
+import org.apache.hadoop.metrics.util.MetricsDynamicMBeanBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+/**
+ *
+ * This is the JMX MBean for reporting the DataNode Activity.
+ * The MBean is register using the name
+ * "hadoop:service=DataNode,name=DataNodeActivity-<storageid>"
+ *
+ * Many of the activity metrics are sampled and averaged on an interval
+ * which can be specified in the metrics config file.
+ * <p>
+ * For the metrics that are sampled and averaged, one must specify
+ * a metrics context that does periodic update calls. Most metrics contexts do.
+ * The default Null metrics context however does NOT. So if you aren't
+ * using any other metrics context then you can turn on the viewing and averaging
+ * of sampled metrics by specifying the following two lines
+ * in the hadoop-meterics.properties file:
+* <pre>
+ * dfs.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+ * dfs.period=10
+ * </pre>
+ *<p>
+ * Note that the metrics are collected regardless of the context used.
+ * The context with the update thread is used to average the data periodically
+ *
+ *
+ *
+ * Impl details: We use a dynamic mbean that gets the list of the metrics
+ * from the metrics registry passed as an argument to the constructor
+ */
+
+public class DataNodeActivityMBean extends MetricsDynamicMBeanBase {
+ final private ObjectName mbeanName;
+ private Random rand = new Random();
+
+ public DataNodeActivityMBean(final MetricsRegistry mr, final String storageId) {
+ super(mr, "Activity statistics at the DataNode");
+ String storageName;
+ if (storageId.equals("")) {// Temp fix for the uninitialized storage
+ storageName = "UndefinedStorageId" + rand.nextInt();
+ } else {
+ storageName = storageId;
+ }
+ mbeanName = MBeanUtil.registerMBean("DataNode", "DataNodeActivity-" + storageName, this);
+ }
+
+
+ public void shutdown() {
+ if (mbeanName != null)
+ MBeanUtil.unregisterMBean(mbeanName);
+ }
+}
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java Mon Dec 15 17:02:51 2008
@@ -23,8 +23,10 @@
import org.apache.hadoop.metrics.MetricsUtil;
import org.apache.hadoop.metrics.Updater;
import org.apache.hadoop.metrics.jvm.JvmMetrics;
+import org.apache.hadoop.metrics.util.MetricsBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
import org.apache.hadoop.metrics.util.MetricsTimeVaryingInt;
-import org.apache.hadoop.metrics.util.MetricsLongValue;
+import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
@@ -42,51 +44,52 @@
*/
public class DataNodeMetrics implements Updater {
private final MetricsRecord metricsRecord;
- private DataNodeStatistics datanodeStats;
+ private DataNodeActivityMBean datanodeActivityMBean;
+ public MetricsRegistry registry = new MetricsRegistry();
- public MetricsLongValue bytesWritten =
- new MetricsLongValue("bytes_written");
- public MetricsLongValue bytesRead =
- new MetricsLongValue("bytes_read");
+ public MetricsTimeVaryingLong bytesWritten =
+ new MetricsTimeVaryingLong("bytes_written", registry);
+ public MetricsTimeVaryingLong bytesRead =
+ new MetricsTimeVaryingLong("bytes_read", registry);
public MetricsTimeVaryingInt blocksWritten =
- new MetricsTimeVaryingInt("blocks_written");
+ new MetricsTimeVaryingInt("blocks_written", registry);
public MetricsTimeVaryingInt blocksRead =
- new MetricsTimeVaryingInt("blocks_read");
+ new MetricsTimeVaryingInt("blocks_read", registry);
public MetricsTimeVaryingInt blocksReplicated =
- new MetricsTimeVaryingInt("blocks_replicated");
+ new MetricsTimeVaryingInt("blocks_replicated", registry);
public MetricsTimeVaryingInt blocksRemoved =
- new MetricsTimeVaryingInt("blocks_removed");
+ new MetricsTimeVaryingInt("blocks_removed", registry);
public MetricsTimeVaryingInt blocksVerified =
- new MetricsTimeVaryingInt("blocks_verified");
+ new MetricsTimeVaryingInt("blocks_verified", registry);
public MetricsTimeVaryingInt blockVerificationFailures =
- new MetricsTimeVaryingInt("block_verification_failures");
+ new MetricsTimeVaryingInt("block_verification_failures", registry);
public MetricsTimeVaryingInt readsFromLocalClient =
- new MetricsTimeVaryingInt("reads_from_local_client");
+ new MetricsTimeVaryingInt("reads_from_local_client", registry);
public MetricsTimeVaryingInt readsFromRemoteClient =
- new MetricsTimeVaryingInt("reads_from_remote_client");
+ new MetricsTimeVaryingInt("reads_from_remote_client", registry);
public MetricsTimeVaryingInt writesFromLocalClient =
- new MetricsTimeVaryingInt("writes_from_local_client");
+ new MetricsTimeVaryingInt("writes_from_local_client", registry);
public MetricsTimeVaryingInt writesFromRemoteClient =
- new MetricsTimeVaryingInt("writes_from_remote_client");
+ new MetricsTimeVaryingInt("writes_from_remote_client", registry);
public MetricsTimeVaryingRate readBlockOp =
- new MetricsTimeVaryingRate("readBlockOp");
+ new MetricsTimeVaryingRate("readBlockOp", registry);
public MetricsTimeVaryingRate writeBlockOp =
- new MetricsTimeVaryingRate("writeBlockOp");
+ new MetricsTimeVaryingRate("writeBlockOp", registry);
public MetricsTimeVaryingRate readMetadataOp =
- new MetricsTimeVaryingRate("readMetadataOp");
+ new MetricsTimeVaryingRate("readMetadataOp", registry);
public MetricsTimeVaryingRate blockChecksumOp =
- new MetricsTimeVaryingRate("blockChecksumOp");
+ new MetricsTimeVaryingRate("blockChecksumOp", registry);
public MetricsTimeVaryingRate copyBlockOp =
- new MetricsTimeVaryingRate("copyBlockOp");
+ new MetricsTimeVaryingRate("copyBlockOp", registry);
public MetricsTimeVaryingRate replaceBlockOp =
- new MetricsTimeVaryingRate("replaceBlockOp");
+ new MetricsTimeVaryingRate("replaceBlockOp", registry);
public MetricsTimeVaryingRate heartbeats =
- new MetricsTimeVaryingRate("heartBeats");
+ new MetricsTimeVaryingRate("heartBeats", registry);
public MetricsTimeVaryingRate blockReports =
- new MetricsTimeVaryingRate("blockReports");
+ new MetricsTimeVaryingRate("blockReports", registry);
public DataNodeMetrics(Configuration conf, String storageId) {
@@ -96,7 +99,7 @@
// Now the MBean for the data node
- datanodeStats = new DataNodeStatistics(this, storageId);
+ datanodeActivityMBean = new DataNodeActivityMBean(registry, storageId);
// Create record for DataNode metrics
MetricsContext context = MetricsUtil.getContext("dfs");
@@ -106,8 +109,8 @@
}
public void shutdown() {
- if (datanodeStats != null)
- datanodeStats.shutdown();
+ if (datanodeActivityMBean != null)
+ datanodeActivityMBean.shutdown();
}
/**
@@ -116,28 +119,9 @@
*/
public void doUpdates(MetricsContext unused) {
synchronized (this) {
-
- bytesWritten.pushMetric(metricsRecord);
- bytesRead.pushMetric(metricsRecord);
- blocksWritten.pushMetric(metricsRecord);
- blocksRead.pushMetric(metricsRecord);
- blocksReplicated.pushMetric(metricsRecord);
- blocksRemoved.pushMetric(metricsRecord);
- blocksVerified.pushMetric(metricsRecord);
- blockVerificationFailures.pushMetric(metricsRecord);
- readsFromLocalClient.pushMetric(metricsRecord);
- writesFromLocalClient.pushMetric(metricsRecord);
- readsFromRemoteClient.pushMetric(metricsRecord);
- writesFromRemoteClient.pushMetric(metricsRecord);
-
- readBlockOp.pushMetric(metricsRecord);
- writeBlockOp.pushMetric(metricsRecord);
- readMetadataOp.pushMetric(metricsRecord);
- blockChecksumOp.pushMetric(metricsRecord);
- copyBlockOp.pushMetric(metricsRecord);
- replaceBlockOp.pushMetric(metricsRecord);
- heartbeats.pushMetric(metricsRecord);
- blockReports.pushMetric(metricsRecord);
+ for (MetricsBase m : registry.getMetricsList()) {
+ m.pushMetric(metricsRecord);
+ }
}
metricsRecord.update();
}
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeStatistics.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeStatistics.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeStatistics.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeStatistics.java Mon Dec 15 17:02:51 2008
@@ -1,371 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.metrics;
-
-import java.util.Random;
-
-import javax.management.ObjectName;
-
-import org.apache.hadoop.metrics.util.MBeanUtil;
-
-public class DataNodeStatistics implements DataNodeStatisticsMBean {
- private DataNodeMetrics myMetrics;
- private ObjectName mbeanName;
- private Random rand = new Random();
-
-
-
- /**
- * This constructs and registers the DataNodeStatisticsMBean
- * @param dataNodeMetrics - the metrics from which the mbean gets its info
- */
- public DataNodeStatistics(DataNodeMetrics dataNodeMetrics, String storageId) {
- myMetrics = dataNodeMetrics;
- String serverName;
- if (storageId.equals("")) {// Temp fix for the uninitialized storage
- serverName = "DataNode-UndefinedStorageId" + rand.nextInt();
- } else {
- serverName = "DataNode-" + storageId;
- }
- mbeanName = MBeanUtil.registerMBean(serverName, "DataNodeStatistics", this);
- }
-
- /**
- * Shuts down the statistics
- * - unregisters the mbean
- */
- public void shutdown() {
- if (mbeanName != null)
- MBeanUtil.unregisterMBean(mbeanName);
- }
-
- /**
- * @inheritDoc
- */
- public void resetAllMinMax() {
- myMetrics.resetAllMinMax();
- }
-
- /**
- * @inheritDoc
- */
- public int getBlocksRead() {
- return myMetrics.blocksRead.getPreviousIntervalValue();
- }
-
- /**
- * @inheritDoc
- */
- public int getBlocksRemoved() {
- return myMetrics.blocksRemoved.getPreviousIntervalValue();
- }
-
- /**
- * @inheritDoc
- */
- public int getBlocksReplicated() {
- return myMetrics.blocksReplicated.getPreviousIntervalValue();
- }
-
- /**
- * @inheritDoc
- */
- public int getBlocksWritten() {
- return myMetrics.blocksWritten.getPreviousIntervalValue();
- }
-
- /**
- * @inheritDoc
- */
- public long getBytesRead() {
- return myMetrics.bytesRead.get();
- }
-
- /**
- * {@inheritDoc}
- */
- public long getBytesWritten() {
- return myMetrics.bytesWritten.get();
- }
-
- /**
- * @inheritDoc
- */
- public int getBlockVerificationFailures() {
- return myMetrics.blockVerificationFailures.getPreviousIntervalValue();
- }
-
- /**
- * @inheritDoc
- */
- public int getBlocksVerified() {
- return myMetrics.blocksVerified.getPreviousIntervalValue();
- }
-
- /**
- * @inheritDoc
- */
- public int getReadsFromLocalClient() {
- return myMetrics.readsFromLocalClient.getPreviousIntervalValue();
- }
-
- /**
- * @inheritDoc
- */
- public int getReadsFromRemoteClient() {
- return myMetrics.readsFromRemoteClient.getPreviousIntervalValue();
- }
-
- /**
- * @inheritDoc
- */
- public int getWritesFromLocalClient() {
- return myMetrics.writesFromLocalClient.getPreviousIntervalValue();
- }
-
- /**
- * @inheritDoc
- */
- public int getWritesFromRemoteClient() {
- return myMetrics.writesFromRemoteClient.getPreviousIntervalValue();
- }
-
- /**
- * @inheritDoc
- */
- public long getReadBlockOpAverageTime() {
- return myMetrics.readBlockOp.getPreviousIntervalAverageTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getReadBlockOpMaxTime() {
- return myMetrics.readBlockOp.getMaxTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getReadBlockOpMinTime() {
- return myMetrics.readBlockOp.getMinTime();
- }
-
- /**
- * @inheritDoc
- */
- public int getReadBlockOpNum() {
- return myMetrics.readBlockOp.getPreviousIntervalNumOps();
- }
-
- /**
- * @inheritDoc
- */
- public long getReadMetadataOpAverageTime() {
- return myMetrics.readMetadataOp.getPreviousIntervalAverageTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getReadMetadataOpMaxTime() {
- return myMetrics.readMetadataOp.getMaxTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getReadMetadataOpMinTime() {
- return myMetrics.readMetadataOp.getMinTime();
- }
-
- /**
- * @inheritDoc
- */
- public int getReadMetadataOpNum() {
- return myMetrics.readMetadataOp.getPreviousIntervalNumOps();
- }
-
- /**
- * @inheritDoc
- */
- public long getBlockChecksumOpAverageTime() {
- return myMetrics.blockChecksumOp.getPreviousIntervalAverageTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getBlockChecksumOpMaxTime() {
- return myMetrics.blockChecksumOp.getMaxTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getBlockChecksumOpMinTime() {
- return myMetrics.blockChecksumOp.getMinTime();
- }
-
- /**
- * @inheritDoc
- */
- public int getBlockChecksumOpNum() {
- return myMetrics.blockChecksumOp.getPreviousIntervalNumOps();
- }
-
- /**
- * @inheritDoc
- */
- public long getReplaceBlockOpAverageTime() {
- return myMetrics.replaceBlockOp.getPreviousIntervalAverageTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getReplaceBlockOpMaxTime() {
- return myMetrics.replaceBlockOp.getMaxTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getReplaceBlockOpMinTime() {
- return myMetrics.replaceBlockOp.getMinTime();
- }
-
- /**
- * @inheritDoc
- */
- public int getReplaceBlockOpNum() {
- return myMetrics.replaceBlockOp.getPreviousIntervalNumOps();
- }
-
- /**
- * @inheritDoc
- */
- public long getWriteBlockOpAverageTime() {
- return myMetrics.writeBlockOp.getPreviousIntervalAverageTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getWriteBlockOpMaxTime() {
- return myMetrics.writeBlockOp.getMaxTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getWriteBlockOpMinTime() {
- return myMetrics.writeBlockOp.getMinTime();
- }
-
- /**
- * @inheritDoc
- */
- public int getWriteBlockOpNum() {
- return myMetrics.writeBlockOp.getPreviousIntervalNumOps();
- }
-
- /**
- * @inheritDoc
- */
- public long getCopyBlockOpAverageTime() {
- return myMetrics.copyBlockOp.getPreviousIntervalAverageTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getCopyBlockOpMaxTime() {
- return myMetrics.copyBlockOp.getMaxTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getCopyBlockOpMinTime() {
- return myMetrics.copyBlockOp.getMinTime();
- }
-
- /**
- * @inheritDoc
- */
- public int getCopyBlockOpNum() {
- return myMetrics.copyBlockOp.getPreviousIntervalNumOps();
- }
-
- /**
- * @inheritDoc
- */
- public long getBlockReportsAverageTime() {
- return myMetrics.blockReports.getPreviousIntervalAverageTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getBlockReportsMaxTime() {
- return myMetrics.blockReports.getMaxTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getBlockReportsMinTime() {
- return myMetrics.blockReports.getMinTime();
- }
-
- /**
- * @inheritDoc
- */
- public int getBlockReportsNum() {
- return myMetrics.blockReports.getPreviousIntervalNumOps();
- }
-
- /**
- * @inheritDoc
- */
- public long getHeartbeatsAverageTime() {
- return myMetrics.heartbeats.getPreviousIntervalAverageTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getHeartbeatsMaxTime() {
- return myMetrics.heartbeats.getMaxTime();
- }
-
- /**
- * @inheritDoc
- */
- public long getHeartbeatsMinTime() {
- return myMetrics.heartbeats.getMinTime();
- }
-
- /**
- * @inheritDoc
- */
- public int getHeartbeatsNum() {
- return myMetrics.heartbeats.getPreviousIntervalNumOps();
- }
-}
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeStatisticsMBean.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeStatisticsMBean.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeStatisticsMBean.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeStatisticsMBean.java Mon Dec 15 17:02:51 2008
@@ -1,320 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.datanode.metrics;
-
-/**
- *
- * This is the JMX interface for the runtime statistics for the data node.
- * Many of the statistics are sampled and averaged on an interval
- * which can be specified in the config file.
- * <p>
- * For the statistics that are sampled and averaged, one must specify
- * a metrics context that does periodic update calls. Most do.
- * The default Null metrics context however does NOT. So if you aren't
- * using any other metrics context then you can turn on the viewing and averaging
- * of sampled metrics by specifying the following two lines
- * in the hadoop-meterics.properties file:
- * <pre>
- * dfs.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
- * dfs.period=10
- * </pre>
- *<p>
- * Note that the metrics are collected regardless of the context used.
- * The context with the update thread is used to average the data periodically.
- * <p>
- * Name Node Status info is reported in another MBean
- * @see org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean
- *
- */
-public interface DataNodeStatisticsMBean {
-
- /**
- * Number of bytes written in the last interval
- * @return number of bytes written
- */
- long getBytesWritten();
-
- /**
- * Number of bytes read in the last interval
- * @return number of bytes read
- */
- long getBytesRead();
-
- /**
- * Number of blocks written in the last interval
- * @return number of blocks written
- */
- int getBlocksWritten();
-
- /**
- * Number of blocks read in the last interval
- * @return number of blocks read
- */
- int getBlocksRead();
-
- /**
- * Number of blocks replicated in the last interval
- * @return number of blocks replicated
- */
- int getBlocksReplicated();
-
- /**
- * Number of blocks removed in the last interval
- * @return number of blocks removed
- */
- int getBlocksRemoved();
-
- /**
- * Number of blocks verified in the last interval
- * @return number of blocks verified
- */
- int getBlocksVerified();
-
- /**
- * Number of block verification failures in the last interval
- * @return number of block verification failures
- */
- int getBlockVerificationFailures();
-
- /**
- * Number of reads from local clients in the last interval
- * @return number of reads from local clients
- */
- int getReadsFromLocalClient();
-
-
- /**
- * Number of reads from remote clients in the last interval
- * @return number of reads from remote clients
- */
- int getReadsFromRemoteClient();
-
-
- /**
- * Number of writes from local clients in the last interval
- * @return number of writes from local clients
- */
- int getWritesFromLocalClient();
-
-
- /**
- * Number of writes from remote clients in the last interval
- * @return number of writes from remote clients
- */
- int getWritesFromRemoteClient();
-
- /**
- * Number of ReadBlock Operation in last interval
- * @return number of operations
- */
- int getReadBlockOpNum();
-
- /**
- * Average time for ReadBlock Operation in last interval
- * @return time in msec
- */
- long getReadBlockOpAverageTime();
-
- /**
- * The Minimum ReadBlock Operation Time since reset was called
- * @return time in msec
- */
- long getReadBlockOpMinTime();
-
- /**
- * The Maximum ReadBlock Operation Time since reset was called
- * @return time in msec
- */
- long getReadBlockOpMaxTime();
-
- /**
- * Number of WriteBlock Operation in last interval
- * @return number of operations
- */
- int getWriteBlockOpNum();
-
- /**
- * Average time for WriteBlock Operation in last interval
- * @return time in msec
- */
- long getWriteBlockOpAverageTime();
-
- /**
- * The Minimum WriteBlock Operation Time since reset was called
- * @return time in msec
- */
- long getWriteBlockOpMinTime();
-
- /**
- * The Maximum WriteBlock Operation Time since reset was called
- * @return time in msec
- */
- long getWriteBlockOpMaxTime();
-
- /**
- * Number of ReadMetadata Operation in last interval
- * @return number of operations
- */
- int getReadMetadataOpNum();
-
- /**
- * Average time for ReadMetadata Operation in last interval
- * @return time in msec
- */
- long getReadMetadataOpAverageTime();
-
- /**
- * The Minimum ReadMetadata Operation Time since reset was called
- * @return time in msec
- */
- long getReadMetadataOpMinTime();
-
- /**
- * The Maximum ReadMetadata Operation Time since reset was called
- * @return time in msec
- */
- long getReadMetadataOpMaxTime();
-
- /**
- * Number of block BlockChecksum in last interval
- * @return number of operations
- */
- int getBlockChecksumOpNum();
-
- /**
- * Average time for BlockChecksum Operation in last interval
- * @return time in msec
- */
- long getBlockChecksumOpAverageTime();
-
- /**
- * The Minimum BlockChecksum Operation Time since reset was called
- * @return time in msec
- */
- long getBlockChecksumOpMinTime();
-
- /**
- * The Maximum BlockChecksum Operation Time since reset was called
- * @return time in msec
- */
- long getBlockChecksumOpMaxTime();
-
- /**
- * Number of CopyBlock Operation in last interval
- * @return number of operations
- */
- int getCopyBlockOpNum();
-
- /**
- * Average time for CopyBlock Operation in last interval
- * @return time in msec
- */
- long getCopyBlockOpAverageTime();
-
- /**
- * The Minimum CopyBlock Operation Time since reset was called
- * @return time in msec
- */
- long getCopyBlockOpMinTime();
-
- /**
- * The Maximum CopyBlock Operation Time since reset was called
- * @return time in msec
- */
- long getCopyBlockOpMaxTime();
-
- /**
- * Number of ReplaceBlock Operation in last interval
- * @return number of operations
- */
- int getReplaceBlockOpNum();
-
-
- /**
- * Average time for ReplaceBlock Operation in last interval
- * @return time in msec
- */
- long getReplaceBlockOpAverageTime();
-
- /**
- * The Minimum ReplaceBlock Operation Time since reset was called
- * @return time in msec
- */
- long getReplaceBlockOpMinTime();
-
- /**
- * The Maximum ReplaceBlock Operation Time since reset was called
- * @return time in msec
- */
- long getReplaceBlockOpMaxTime();
-
- /**
- * Number of Block Reports sent in last interval
- * @return number of operations
- */
- int getBlockReportsNum();
-
- /**
- * Average time for Block Reports Operation in last interval
- * @return time in msec
- */
- long getBlockReportsAverageTime();
-
- /**
- * The Minimum Block Reports Operation Time since reset was called
- * @return time in msec
- */
- long getBlockReportsMinTime();
-
- /**
- * The Maximum Block Reports Operation Time since reset was called
- * @return time in msec
- */
- long getBlockReportsMaxTime();
-
- /**
- * Number of Heartbeat Operation in last interval
- * @return number of operations
- */
- int getHeartbeatsNum();
-
- /**
- * Average time for Heartbeat Operation in last interval
- * @return time in msec
- */
- long getHeartbeatsAverageTime();
-
- /**
- * The Minimum Heartbeat Operation Time since reset was called
- * @return time in msec
- */
- long getHeartbeatsMinTime();
-
- /**
- * The Maximum Heartbeat Operation Time since reset was called
- * @return time in msec
- */
- long getHeartbeatsMaxTime();
-
-
- /**
- * Reset all min max times
- */
- public void resetAllMinMax();
-}
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java Mon Dec 15 17:02:51 2008
@@ -25,6 +25,10 @@
* a data node.
* It is also used for publishing via JMX (hence we follow the JMX naming
* convention.)
+ * * Note we have not used the MetricsDynamicMBeanBase to implement this
+ * because the interface for the FSDatasetMBean is stable and should
+ * be published as an interface.
+ *
* <p>
* Data Node runtime statistic info is report in another MBean
* @see org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeStatisticsMBean
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Mon Dec 15 17:02:51 2008
@@ -87,6 +87,7 @@
// these are statistics counters.
private long numTransactions; // number of transactions
+ private long numTransactionsBatchedInSync;
private long totalTimeTransactions; // total time for all transactions
private NameNodeMetrics metrics;
@@ -326,7 +327,7 @@
* @throws IOException
*/
public synchronized void open() throws IOException {
- numTransactions = totalTimeTransactions = 0;
+ numTransactions = totalTimeTransactions = numTransactionsBatchedInSync = 0;
if (editStreams == null)
editStreams = new ArrayList<EditLogOutputStream>();
for (Iterator<StorageDirectory> it =
@@ -376,7 +377,7 @@
return;
}
printStatistics(true);
- numTransactions = totalTimeTransactions = 0;
+ numTransactions = totalTimeTransactions = numTransactionsBatchedInSync = 0;
for (int idx = 0; idx < editStreams.size(); idx++) {
EditLogOutputStream eStream = editStreams.get(idx);
@@ -896,6 +897,9 @@
// If this transaction was already flushed, then nothing to do
//
if (mytxid <= synctxid) {
+ numTransactionsBatchedInSync++;
+ if (metrics != null) // Metrics is non-null only when used inside name node
+ metrics.transactionsBatchedInSync.inc();
return;
}
@@ -957,6 +961,8 @@
buf.append(numTransactions);
buf.append(" Total time for transactions(ms): ");
buf.append(totalTimeTransactions);
+ buf.append("Number of transactions batched in Syncs: ");
+ buf.append(numTransactionsBatchedInSync);
buf.append(" Number of syncs: ");
buf.append(editStreams.get(0).getNumSync());
buf.append(" SyncTimes(ms): ");
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Dec 15 17:02:51 2008
@@ -4354,17 +4354,18 @@
private ObjectName mbeanName;
/**
- * Register the FSNamesystem MBean
+ * Register the FSNamesystem MBean using the name
+ * "hadoop:service=NameNode,name=FSNamesystemState"
*/
void registerMBean(Configuration conf) {
- // We wrap to bypass standard mbean naming convetion.
+ // We wrap to bypass standard mbean naming convention.
// This wraping can be removed in java 6 as it is more flexible in
// package naming for mbeans and their impl.
StandardMBean bean;
try {
myFSMetrics = new FSNamesystemMetrics(conf);
bean = new StandardMBean(this,FSNamesystemMBean.class);
- mbeanName = MBeanUtil.registerMBean("NameNode", "FSNamesystemStatus", bean);
+ mbeanName = MBeanUtil.registerMBean("NameNode", "FSNamesystemState", bean);
} catch (NotCompliantMBeanException e) {
e.printStackTrace();
}
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java Mon Dec 15 17:02:51 2008
@@ -24,9 +24,13 @@
* It is also used for publishing via JMX (hence we follow the JMX naming
* convention.)
*
+ * Note we have not used the MetricsDynamicMBeanBase to implement this
+ * because the interface for the NameNodeStateMBean is stable and should
+ * be published as an interface.
+ *
* <p>
- * Name Node runtime statistic info is report in another MBean
- * @see org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeStatisticsMBean
+ * Name Node runtime activity statistic info is report in another MBean
+ * @see org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeActivityMBean
*
*/
public interface FSNamesystemMBean {
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMetrics.java?rev=726900&r1=726899&r2=726900&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMetrics.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMetrics.java Mon Dec 15 17:02:51 2008
@@ -17,18 +17,22 @@
*/
package org.apache.hadoop.hdfs.server.namenode.metrics;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.metrics.*;
+import org.apache.hadoop.metrics.util.MetricsBase;
import org.apache.hadoop.metrics.util.MetricsIntValue;
+import org.apache.hadoop.metrics.util.MetricsLongValue;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
/**
*
* This class is for maintaining the various FSNamesystem status metrics
* and publishing them through the metrics interfaces.
- * This also registers the JMX MBean for RPC.
+ * The SNamesystem creates and registers the JMX MBean.
* <p>
* This class has a number of metrics variables that are publicly accessible;
* these variables (objects) have methods to update their values;
@@ -39,16 +43,18 @@
public class FSNamesystemMetrics implements Updater {
private static Log log = LogFactory.getLog(FSNamesystemMetrics.class);
private final MetricsRecord metricsRecord;
+ public MetricsRegistry registry = new MetricsRegistry();
+
- public MetricsIntValue filesTotal = new MetricsIntValue("FilesTotal");
- public MetricsIntValue blocksTotal = new MetricsIntValue("BlocksTotal");
- public MetricsIntValue capacityTotalGB = new MetricsIntValue("CapacityTotalGB");
- public MetricsIntValue capacityUsedGB = new MetricsIntValue("CapacityUsedGB");
- public MetricsIntValue capacityRemainingGB = new MetricsIntValue("CapacityRemainingGB");
- public MetricsIntValue totalLoad = new MetricsIntValue("TotalLoad");
- public MetricsIntValue pendingReplicationBlocks = new MetricsIntValue("PendingReplicationBlocks");
- public MetricsIntValue underReplicatedBlocks = new MetricsIntValue("UnderReplicatedBlocks");
- public MetricsIntValue scheduledReplicationBlocks = new MetricsIntValue("ScheduledReplicationBlocks");
+ public MetricsIntValue filesTotal = new MetricsIntValue("FilesTotal", registry);
+ public MetricsLongValue blocksTotal = new MetricsLongValue("BlocksTotal", registry);
+ public MetricsIntValue capacityTotalGB = new MetricsIntValue("CapacityTotalGB", registry);
+ public MetricsIntValue capacityUsedGB = new MetricsIntValue("CapacityUsedGB", registry);
+ public MetricsIntValue capacityRemainingGB = new MetricsIntValue("CapacityRemainingGB", registry);
+ public MetricsIntValue totalLoad = new MetricsIntValue("TotalLoad", registry);
+ public MetricsIntValue pendingReplicationBlocks = new MetricsIntValue("PendingReplicationBlocks", registry);
+ public MetricsIntValue underReplicatedBlocks = new MetricsIntValue("UnderReplicatedBlocks", registry);
+ public MetricsIntValue scheduledReplicationBlocks = new MetricsIntValue("ScheduledReplicationBlocks", registry);
public FSNamesystemMetrics(Configuration conf) {
String sessionId = conf.get("session.id");
@@ -79,37 +85,29 @@
* int, so they are rounded to GB
*/
public void doUpdates(MetricsContext unused) {
+ /**
+ * ToFix
+ * If the metrics counter were instead stored in the metrics objects themselves
+ * we could avoid copying the values on each update.
+ */
synchronized (this) {
FSNamesystem fsNameSystem = FSNamesystem.getFSNamesystem();
filesTotal.set((int)fsNameSystem.getFilesTotal());
- filesTotal.pushMetric(metricsRecord);
-
blocksTotal.set((int)fsNameSystem.getBlocksTotal());
- blocksTotal.pushMetric(metricsRecord);
-
capacityTotalGB.set(roundBytesToGBytes(fsNameSystem.getCapacityTotal()));
- capacityTotalGB.pushMetric(metricsRecord);
-
capacityUsedGB.set(roundBytesToGBytes(fsNameSystem.getCapacityUsed()));
- capacityUsedGB.pushMetric(metricsRecord);
-
capacityRemainingGB.set(roundBytesToGBytes(fsNameSystem.
getCapacityRemaining()));
- capacityRemainingGB.pushMetric(metricsRecord);
-
totalLoad.set(fsNameSystem.getTotalLoad());
- totalLoad.pushMetric(metricsRecord);
-
pendingReplicationBlocks.set((int)fsNameSystem.
getPendingReplicationBlocks());
- pendingReplicationBlocks.pushMetric(metricsRecord);
-
underReplicatedBlocks.set((int)fsNameSystem.getUnderReplicatedBlocks());
- underReplicatedBlocks.pushMetric(metricsRecord);
-
scheduledReplicationBlocks.set((int)fsNameSystem.
getScheduledReplicationBlocks());
- scheduledReplicationBlocks.pushMetric(metricsRecord);
+
+ for (MetricsBase m : registry.getMetricsList()) {
+ m.pushMetric(metricsRecord);
+ }
}
metricsRecord.update();
}