You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ch...@apache.org on 2011/11/28 12:21:29 UTC
svn commit: r1207083 [1/2] - in /incubator/hama/trunk/core/src:
main/java/org/apache/hama/metrics/ main/java/org/apache/hama/util/
test/java/org/apache/hama/metrics/ test/org/ test/org/apache/ test/resources/
Author: chl501
Date: Mon Nov 28 11:21:27 2011
New Revision: 1207083
URL: http://svn.apache.org/viewvc?rev=1207083&view=rev
Log:
[HAMA-363] Add network condition monitoring function to BSPMaster
Added:
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/AbstractPatternFilter.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/DefaultMetricsSystem.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/GlobalFilter.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/JvmMetrics.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Metric.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsConfig.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFactory.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFilter.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsInfo.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsRecord.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSink.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSinkAdaptor.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSource.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSourceAdaptor.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSystem.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsTag.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Pair.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/RegexFilter.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMetrics.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMonitorSink.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/util/ReflectionUtils.java
incubator/hama/trunk/core/src/test/java/org/apache/hama/metrics/
incubator/hama/trunk/core/src/test/java/org/apache/hama/metrics/TestMetricsConfig.java
incubator/hama/trunk/core/src/test/java/org/apache/hama/metrics/TestMetricsSystem.java
incubator/hama/trunk/core/src/test/java/org/apache/hama/metrics/TestPatternFilter.java
incubator/hama/trunk/core/src/test/org/
incubator/hama/trunk/core/src/test/org/apache/
incubator/hama/trunk/core/src/test/resources/hama-metrics-msys.properties
incubator/hama/trunk/core/src/test/resources/hama-metrics-test-config.properties
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/AbstractPatternFilter.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/AbstractPatternFilter.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/AbstractPatternFilter.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/AbstractPatternFilter.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public abstract class AbstractPatternFilter implements MetricsFilter {
+
+ public static final Log LOG = LogFactory.getLog(AbstractPatternFilter.class);
+
+ private final Pattern includePattern;
+ private final Pattern excludePattern;
+ private final Map<String, Pattern> includeTagPatterns;
+ private final Map<String, Pattern> excludeTagPatterns;
+ private final Pattern tagPattern = Pattern.compile("^(\\w+):(.*)");
+ private static final char BACKSLASH = '\\';
+
+ public AbstractPatternFilter(Set<MetricsConfig.Entry<String, String>> set) {
+ Pattern in = null, ex = null;
+ Map<String, Pattern> inTags = new ConcurrentHashMap<String, Pattern>();
+ Map<String, Pattern> exTags = new ConcurrentHashMap<String, Pattern>();
+ for(MetricsConfig.Entry<String, String> e: set){
+ String key = e.key();
+ String value = e.value();
+ if(LOG.isDebugEnabled()){
+ LOG.info("Subset content of MetricsConfig's key "+key+" value "+value);
+ }
+ // data read from properties files should pay attention to excape sequence
+ if(key.matches("(.*)(\\.include)")){
+ in = compile(value);
+ }else if(key.matches("(.*)(\\.exclude)")){
+ ex = compile(value);
+ }
+
+ if(key.matches("(.*)(\\.include.tags)")){
+ if(null != value && !value.isEmpty()) {
+ Matcher m = tagPattern.matcher(value);
+ if(m.matches()){
+ inTags.put(m.group(1), compile(m.group(2)));
+ }
+ }
+ }else if(key.matches("(.*)(\\.exclude.tags)")){
+ if(null != value && !value.isEmpty()) {
+ Matcher m = tagPattern.matcher(value);
+ if(m.matches()){
+ exTags.put(m.group(1), compile(m.group(2)));
+ }
+ }
+ }
+ }
+ this.includePattern = in;
+ this.excludePattern = ex;
+ this.includeTagPatterns = inTags;
+ this.excludeTagPatterns = exTags;
+ }
+
+ /**
+ * Return true if include pattern matches the name; false if exclude pattern
+ * matches the name. Also, if include pattern is provided but no exclude
+ * pattern, name won't be accepted as it is white list only mode.
+ * @return ture if accepted; false otherwise.
+ */
+ @Override
+ public boolean accepts(String name) {
+ if (includePattern != null && includePattern.matcher(name).matches()) {
+ return true;
+ }
+ if ((excludePattern != null && excludePattern.matcher(name).matches())) {
+ return false;
+ }
+ // Reject if no matches in white list only mode.
+ if (includePattern != null && excludePattern == null) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * True if tags are listed as include tag pattern; false if tags
+ * are listed in exclude tag pattern. If include and exclude tags pattern
+ * are empty, accepts returns false.
+ * @param tags to be tested if accepted in filter.
+ * @return boolean value tells if the filter accepts the tag.
+ */
+ @Override
+ public boolean accepts(Iterable<MetricsTag> tags) {
+ // Accept if any include tag pattern matches
+ for (MetricsTag t : tags) {
+ Pattern pat = includeTagPatterns.get(t.name());
+ if (pat != null && pat.matcher(t.value()).matches()) {
+ return true;
+ }
+ }
+ // Reject if any exclude tag pattern matches
+ for (MetricsTag t : tags) {
+ Pattern pat = excludeTagPatterns.get(t.name());
+ if (pat != null && pat.matcher(t.value()).matches()) {
+ return false;
+ }
+ }
+ // Reject if no match in whitelist only mode
+ if (!includeTagPatterns.isEmpty() && excludeTagPatterns.isEmpty()) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * True if the record is accepted; false otherwise.
+ * @param record to filter on
+ * @return true to accept; false otherwise.
+ */
+ public boolean accepts(MetricsRecord record) {
+ return accepts(record.tags());
+ }
+
+ public abstract Pattern compile(String glob);
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/DefaultMetricsSystem.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/DefaultMetricsSystem.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/DefaultMetricsSystem.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/DefaultMetricsSystem.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,281 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import static java.util.concurrent.TimeUnit.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.util.ReflectionUtils;
+
+/**
+ * A basic implementation which moves metrics data from sources to sinks.
+ */
+final class DefaultMetricsSystem implements MetricsSystem, Runnable{
+
+ public static final Log LOG = LogFactory.getLog(DefaultMetricsSystem.class);
+
+ private final String prefix;
+ private final ScheduledExecutorService sched;
+ private final AtomicBoolean state = new AtomicBoolean(false);
+ private final HamaConfiguration conf;
+ private final MetricsConfig config;
+ private final MetricsFilter sourceFilter;
+ private final long period;
+ private final AtomicReference<MetricsSourceAdaptor> sysSource =
+ new AtomicReference<MetricsSourceAdaptor>();
+ private final ConcurrentMap<String, MetricsSourceAdaptor> sources =
+ new ConcurrentHashMap<String, MetricsSourceAdaptor>();
+ private final ConcurrentMap<String, MetricsSinkAdaptor> sinks =
+ new ConcurrentHashMap<String, MetricsSinkAdaptor>();
+ private final BlockingQueue<List<Pair>> queue;
+ public static final String MS_NAME = "MetricsSystem";
+ public static final String MS_STATS_NAME = MS_NAME +",sub=Stats";
+ public static final String MS_STATS_DESC = "Metrics system metrics";
+
+
+ DefaultMetricsSystem(String prefix, HamaConfiguration conf){
+ this.prefix = prefix;
+ this.conf = conf;
+ this.config = MetricsConfig.create(this.conf);
+ this.sourceFilter = this.config.getFilter("default.source.filter");
+ this.period = conf.getInt("bsp.metrics.period", 3);
+ this.queue = new LinkedBlockingQueue<List<Pair>>();
+ int workers = conf.getInt("bsp.metrics.threads_pool", 1);
+ this.sched = Executors.newScheduledThreadPool(workers);
+ }
+
+ public final String prefix(){
+ return this.prefix;
+ }
+
+ public final long period(){
+ return this.period;
+ }
+
+ /**
+ * Start MetricsSystem, scheduling with fix rate. The MetricsSystem
+ * will periodically execute fetching metrics from source to sink.
+ */
+ public void start(){
+ if(true == state.get()){
+ LOG.warn("MetricsSystem has already been started.");
+ return;
+ }
+ configure();
+ state.set(true);
+ this.sched.scheduleAtFixedRate((Runnable)this, 3, period(), SECONDS);
+ }
+
+ private void configure(){
+ configureSinks();
+ configureSources();
+ }
+
+ private void configureSinks(){
+ Set<MetricsConfig.Entry<String, String>> sinkSubset =
+ this.config.subset(prefix(), "sink");
+ for(MetricsConfig.Entry<String, String> entry: sinkSubset){
+ String key = entry.key();
+ String value = entry.value();
+ if(LOG.isDebugEnabled()){
+ LOG.debug("Configure sinks' key "+key+" value "+value);
+ }
+ if(key.endsWith("class")) {
+ try{
+ LOG.info("Sink name to be registered:"+value);
+ MetricsSink sink =
+ ReflectionUtils.newInstance(value);
+ register(value, "sink registered from properties file.", (MetricsSink)sink);
+ }catch(ClassNotFoundException cnfe) {
+ LOG.warn("Class "+value+" not found.", cnfe);
+ }
+ }
+ }
+ }
+
+ private void configureSources(){
+ registerSystemSource();
+ }
+
+ private void registerSystemSource(){
+ // sysSource not in sources list
+ MetricsSource src = MetricsFactory.createSource(SystemMetrics.class,
+ new Class[]{ Map.class, Map.class, HamaConfiguration.class },
+ new Object[]{ sources, sinks, conf });
+ MetricsSourceAdaptor srcAdp =
+ new MetricsSourceAdaptor(prefix(), MS_STATS_NAME, MS_STATS_DESC, src,
+ (period()+1), conf);
+ sysSource.set(srcAdp);
+ sysSource.get().start();
+ }
+
+ /**
+ * Stop MetricsSystem. First stop source, then sinks; evetually shutting down
+ * the thread scheduler.
+ */
+ public void stop(){
+ state.set(false);
+ stopSources();
+ stopSinks();
+ sched.shutdown();
+ }
+
+ void stopSources(){
+ for(MetricsSourceAdaptor adpt: sources.values()){
+ adpt.stop();
+ }
+ sysSource.get().stop();
+ sources.clear();
+ }
+
+ void stopSinks(){
+ for(MetricsSinkAdaptor adpt: sinks.values()){
+ adpt.stop();
+ }
+ sinks.clear();
+ }
+
+ /**
+ * Register source and starts as MBean service.
+ * @param name to be referenced to the source.
+ * @param description describes the purpose.
+ * @param source of metrics.
+ */
+ public void register(String name, String description, MetricsSource source){
+ if(sources.containsKey(name)){
+ LOG.warn("Source with name `"+name+"' is already registered.");
+ return;
+ }
+ MetricsSourceAdaptor adpt =
+ new MetricsSourceAdaptor(prefix(), name, description, source, period()+1,
+ this.conf);
+ sources.putIfAbsent(name, adpt);
+ adpt.start(); // register as MBean
+ LOG.info("Registered source "+name+".");
+ }
+
+ /**
+ * Register sink with a specified name.
+ * @param name to be associated with sink.
+ * @param description describes the purpose.
+ * @param sink is used to consumes metrics.
+ */
+ public void register(String name, String description, MetricsSink sink){
+ if(sinks.containsKey(name)){
+ LOG.warn("Sink with name `"+name+"' is already registered.");
+ return;
+ }
+ MetricsFilter srcFilter = this.config.getFilter(prefix()+".source.filter");
+ MetricsFilter recordFilter = this.config.getFilter(prefix()+".record.filter");
+ MetricsFilter metricFilter = this.config.getFilter(prefix()+".metric.filter");
+ MetricsSinkAdaptor adpt =
+ new MetricsSinkAdaptor(name, description, sink, this.queue, this.conf,
+ srcFilter, recordFilter, metricFilter);
+ sinks.putIfAbsent(name, adpt);
+ adpt.start();
+ LOG.info("Registered sink "+name+".");
+ }
+
+ /**
+ * Periodically harvest metrics data.
+ */
+ public void run() {
+ if(LOG.isDebugEnabled()){ LOG.debug(" Sinks size "+sinks.size()); }
+ if(0 < sinks.size()) publishMetrics(sample());
+ }
+
+ void publishMetrics(final List<Pair> collections){
+ for(MetricsSinkAdaptor adpt: sinks.values()){
+ if(LOG.isDebugEnabled()){ LOG.debug(" sink adptr name "+adpt.name()); }
+ adpt.putMetrics(collections);
+ }
+ }
+
+ /**
+ * With corresponded source adaptor, a list of MetricsRecord are collected
+ * and stored into a list of pair, which will be consumed later by sinks
+ * that has interest.
+ */
+ private List<Pair> sample(){
+ final List<Pair> collector = new ArrayList<Pair>();
+ for(Map.Entry<String, MetricsSourceAdaptor> entry: sources.entrySet()){
+ if(null == this.sourceFilter || sourceFilter.accepts(entry.getKey())){
+ MetricsSourceAdaptor adpt = entry.getValue();
+ collector.add(new Pair(adpt.name(), snapshot(adpt)));
+ }
+ }
+ if(LOG.isDebugEnabled()){
+ LOG.debug(" Source adaptor size "+collector.size());
+ }
+ return collector;
+ }
+
+ /**
+ * Source adaptor harvests metrics information into a list of MetricsRecord.
+ */
+ private List<MetricsRecord> snapshot(MetricsSourceAdaptor srcAdpt){
+ final List<MetricsRecord> collector = new ArrayList<MetricsRecord>();
+ srcAdpt.getMetrics(collector);
+ return collector;
+ }
+
+ static String hostname() {
+ try {
+ return InetAddress.getLocalHost().getHostName();
+ }
+ catch (Exception e) {
+ LOG.error("Error getting localhost name. "+
+ "hostname() would return 'localhost'", e);
+ }
+ return "localhost";
+ }
+
+ @Override
+ public MetricsSink findSink(String name){
+ MetricsSinkAdaptor adpt = sinks.get(name);
+ if(null != adpt){
+ return adpt.sink();
+ }
+ return null;
+ }
+
+ @Override
+ public MetricsSource findSource(String name){
+ MetricsSourceAdaptor adpt = sources.get(name);
+ if(null != adpt){
+ return adpt.source();
+ }
+ return null;
+ }
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/GlobalFilter.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/GlobalFilter.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/GlobalFilter.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/GlobalFilter.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * This class provides an abstract implementation of the BSP interface.
+ */
+public final class GlobalFilter extends AbstractPatternFilter {
+
+ public static final Log LOG = LogFactory.getLog(GlobalFilter.class);
+
+ private static final char BACKSLASH = '\\';
+
+ public GlobalFilter(Set<MetricsConfig.Entry<String, String>> set) {
+ super(set);
+ }
+
+ /**
+ * Set and compile a glob pattern
+ * @param glob the glob pattern string
+ */
+ @Override
+ public Pattern compile(String glob) {
+ StringBuilder regex = new StringBuilder();
+ int setOpen = 0;
+ int curlyOpen = 0;
+ int len = glob.length();
+ boolean hasWildcard = false;
+
+ for (int i = 0; i < len; i++) {
+ char c = glob.charAt(i);
+
+ switch (c) {
+ case BACKSLASH:
+ if (++i >= len) {
+ error("Missing escaped character", glob, i);
+ }
+ regex.append(c).append(glob.charAt(i));
+ continue;
+ case '.':
+ case '$':
+ case '(':
+ case ')':
+ case '|':
+ case '+':
+ // escape regex special chars that are not glob special chars
+ regex.append(BACKSLASH);
+ break;
+ case '*':
+ regex.append('.');
+ hasWildcard = true;
+ break;
+ case '?':
+ regex.append('.');
+ hasWildcard = true;
+ continue;
+ case '{': // start of a group
+ regex.append("(?:"); // non-capturing
+ curlyOpen++;
+ hasWildcard = true;
+ continue;
+ case ',':
+ regex.append(curlyOpen > 0 ? '|' : c);
+ continue;
+ case '}':
+ if (curlyOpen > 0) {
+ // end of a group
+ curlyOpen--;
+ regex.append(")");
+ continue;
+ }
+ break;
+ case '[':
+ if (setOpen > 0) {
+ error("Unclosed character class", glob, i);
+ }
+ setOpen++;
+ hasWildcard = true;
+ break;
+ case '^': // ^ inside [...] can be unescaped
+ if (setOpen == 0) {
+ regex.append(BACKSLASH);
+ }
+ break;
+ case '!': // [! needs to be translated to [^
+ regex.append(setOpen > 0 && '[' == glob.charAt(i - 1) ? '^' : '!');
+ continue;
+ case ']':
+ // Many set errors like [][] could not be easily detected here,
+ // as []], []-] and [-] are all valid POSIX glob and java regex.
+ // We'll just let the regex compiler do the real work.
+ setOpen = 0;
+ break;
+ default:
+ }
+ regex.append(c);
+ }
+
+ if (setOpen > 0) {
+ error("Unclosed character class", glob, len);
+ }
+ if (curlyOpen > 0) {
+ error("Unclosed group", glob, len);
+ }
+ return Pattern.compile(regex.toString());
+ }
+
+ private static void error(String message, String pattern, int pos) {
+ throw new PatternSyntaxException(message, pattern, pos);
+ }
+
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/JvmMetrics.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/JvmMetrics.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/JvmMetrics.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/JvmMetrics.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.ArrayList;
+import java.util.List;
+import static java.lang.Thread.State.*;
+import static org.apache.hama.metrics.JvmMetrics.Metrics.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+
+public final class JvmMetrics implements MetricsSource {
+
+ public static final Log LOG = LogFactory.getLog(JvmMetrics.class);
+
+ private final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+ private final List<GarbageCollectorMXBean> gcBeans =
+ ManagementFactory.getGarbageCollectorMXBeans();
+ private final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+ private static final long M = 1024*1024;
+ private final String name, description, processName, sessionId;
+
+ public enum Metrics{
+ MemNonHeapUsedM("Non-heap memory used in MB"),
+ MemNonHeapCommittedM("Non-heap memory committed in MB"),
+ MemHeapUsedM("Heap memory used in MB"),
+ MemHeapCommittedM("Heap memory committed in MB"),
+ GcCount("Total GC count"),
+ GcTimeMillis("Total GC time in milliseconds"),
+ ThreadsNew("Number of new threads"),
+ ThreadsRunnable("Number of runnable threads"),
+ ThreadsBlocked("Number of blocked threads"),
+ ThreadsWaiting("Number of waiting threads"),
+ ThreadsTimedWaiting("Number of timed waiting threads"),
+ ThreadsTerminated("Number of terminated threads");
+
+ private final String desc;
+
+ Metrics(String desc) { this.desc = desc; }
+ public String description() { return desc; }
+
+ @Override public String toString(){
+ return "name "+name()+" description "+desc;
+ }
+ }
+
+ public JvmMetrics(String processName, String sessionId, HamaConfiguration conf) {
+ this.name = "JvmMetric";
+ this.description = "JVM metrics information.";
+ this.processName = processName;
+ this.sessionId = sessionId;
+ }
+
+ @Override
+ public final String name(){
+ return this.name;
+ }
+
+ @Override
+ public final String description(){
+ return this.description;
+ }
+
+ /**
+ * Record metrics data, and collect those information as records.
+ * @param collector of records to be collected.
+ */
+ public void snapshot(final List<MetricsRecord> collector){
+ final MetricsRecord record = new MetricsRecord(name, description);
+ record.tag("ProcessName", processName);
+ record.tag("SessionId", sessionId);
+ memory(record);
+ gc(record);
+ threads(record);
+ collector.add(record);
+ }
+
+ private void memory(final MetricsRecord record){
+ MemoryUsage memNonHeap = memoryMXBean.getNonHeapMemoryUsage();
+ MemoryUsage memHeap = memoryMXBean.getHeapMemoryUsage();
+ record.add(new Metric(MemNonHeapUsedM, memNonHeap.getUsed() / M));
+ record.add(new Metric(MemNonHeapCommittedM,
+ memNonHeap.getCommitted() / M));
+ record.add(new Metric(MemHeapUsedM, memHeap.getUsed() / M));
+ record.add(new Metric(MemHeapCommittedM, memHeap.getCommitted() / M));
+ }
+
+ private void gc(final MetricsRecord record){
+ long count = 0;
+ long timeMillis = 0;
+ for (GarbageCollectorMXBean gcBean : gcBeans) {
+ long c = gcBean.getCollectionCount();
+ long t = gcBean.getCollectionTime();
+ String name = gcBean.getName();
+ record.add(new Metric("GcCount"+name, c));
+ record.add(new Metric("GcTimeMillis"+name, t));
+ count += c;
+ timeMillis += t;
+ }
+ record.add(new Metric(GcCount, count));
+ record.add(new Metric(GcTimeMillis, timeMillis));
+ }
+
+ private void threads(final MetricsRecord record){
+ int threadsNew = 0;
+ int threadsRunnable = 0;
+ int threadsBlocked = 0;
+ int threadsWaiting = 0;
+ int threadsTimedWaiting = 0;
+ int threadsTerminated = 0;
+ long threadIds[] = threadMXBean.getAllThreadIds();
+ for (ThreadInfo threadInfo : threadMXBean.getThreadInfo(threadIds, 0)) {
+ if (threadInfo == null) continue; // race protection
+ Thread.State state = threadInfo.getThreadState();
+ if(NEW.equals(state)){
+ threadsNew++;
+ break;
+ }else if(RUNNABLE.equals(state)){
+ threadsRunnable++;
+ break;
+ }else if(BLOCKED.equals(state)){
+ threadsBlocked++;
+ break;
+ }else if(WAITING.equals(state)){
+ threadsWaiting++;
+ break;
+ }else if(TIMED_WAITING.equals(state)){
+ threadsTimedWaiting++;
+ break;
+ }else if(TERMINATED.equals(state)){
+ threadsTerminated++;
+ break;
+ }
+ }
+ record.add(new Metric(ThreadsNew, threadsNew));
+ record.add(new Metric(ThreadsRunnable, threadsRunnable));
+ record.add(new Metric(ThreadsBlocked, threadsBlocked));
+ record.add(new Metric(ThreadsWaiting, threadsWaiting));
+ record.add(new Metric(ThreadsTimedWaiting, threadsTimedWaiting));
+ record.add(new Metric(ThreadsTerminated, threadsTerminated));
+ }
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Metric.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Metric.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Metric.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Metric.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+public final class Metric<T> {
+
+ private final String name;
+ private final String description;
+ private final T value;
+
+ public Metric(Enum name, T value){
+ this(name.toString(), name.toString()+" record", value);
+ }
+
+ public Metric(String name, T value){
+ this(name, name+" metric.", value);
+ }
+
+ public Metric(String name, String description, T value){
+ this.name = name;
+ this.description = description;
+ this.value = value;
+ }
+
+ public final String name(){
+ return this.name;
+ }
+
+ public final String description(){
+ return this.description;
+ }
+
+ public final T value(){
+ return this.value;
+ }
+
+ @Override
+ public boolean equals(Object target){
+ if (target == this)
+ return true;
+ if (null == target)
+ return false;
+ if (getClass() != target.getClass())
+ return false;
+
+ Metric m = (Metric) target;
+ if(!name().equals(m.name))
+ return false;
+ if(!description().equals(m.description))
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode(){
+ int result = 17;
+ result = 37 * result + name.hashCode();
+ result = 37 * result + description.hashCode();
+ return result;
+ }
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsConfig.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsConfig.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsConfig.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsConfig.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,218 @@
+package org.apache.hama.metrics;
+
+import java.io.InputStream;
+import java.io.IOException;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.util.ReflectionUtils;
+
+public final class MetricsConfig {
+
+ public static final Log LOG = LogFactory.getLog(MetricsConfig.class);
+
+ public static final String DEFAULT_METRICS_CONFIG = "hama-metrics.properties";
+ private final Properties properties;
+
+ public static final class Entry<KEY, VALUE>{
+ private final KEY key;
+ private VALUE value;
+ Entry(KEY key, VALUE value){
+ this.key = key;
+ this.value = value;
+ }
+ public KEY key(){
+ return this.key;
+ }
+ public VALUE value(){
+ return this.value;
+ }
+ }
+
+ /**
+ * Internal use only. The purpose is to filter out unnecessary key value pair.
+ */
+ abstract class Filter {
+ public abstract void doFilter(String key, String value);
+ }
+
+ MetricsConfig(Properties properties){
+ this.properties = properties;
+ }
+
+ /**
+ * Create MetricsConfig with HamaConfiguration file specified.
+ * Within HamaConfiguration file, `bsp.metrics.conf_files' can specify
+ * multiple config files with comma as dilemeter. It loads with whichever
+ * succeeded first.
+ * @param conf file contains multiple metrics properties configuration files.
+ * @return MetricsConfig file which contains all properties.
+ */
+ public static MetricsConfig create(HamaConfiguration conf){
+ String configFiles =
+ conf.get("bsp.metrics.conf_files", DEFAULT_METRICS_CONFIG);
+ return loadFirst(conf, configFiles.split(","));
+ }
+
+ static MetricsConfig loadFirst(HamaConfiguration conf, String... fileNames) {
+ StringBuilder names = new StringBuilder();
+ for (String name : fileNames) {
+ InputStream stream = null;
+ ClassLoader loader = Thread.currentThread().getContextClassLoader();
+ if(null != loader) {
+ stream = loader.getSystemResourceAsStream(name.trim());
+ }
+ if(null != stream) {
+ Properties prop = new Properties();
+ try{
+ prop.load(stream);
+ }catch(IOException ioe){
+ names.append(name+" ");
+ LOG.warn("Can not load properties from file "+name, ioe);
+ }
+ return new MetricsConfig(prop);
+ }else{
+ names.append(name+" ");
+ LOG.warn("Can not load properties from file "+name);
+ }
+ }
+ throw new RuntimeException("Can not load configuration files "+
+ names.toString());
+ }
+
+ /**
+ * Return all properties stored as MetricsConfig.Entry. It is accessible
+ * by calling key() and value() for obtaining key, value data respectively.
+ * @return Set that contains all properties.
+ */
+ public Set<Entry<String, String>> entrySet(){
+ final Set<Entry<String, String>> set = new HashSet<Entry<String, String>>();
+ list(new Filter(){
+ public void doFilter(String key, String value){
+ set.add(new Entry(key, value));
+ }
+ });
+ return set;
+ }
+
+ private void list(Filter filter){
+ for(Enumeration e = properties.propertyNames(); e.hasMoreElements(); ){
+ String key = (String)e.nextElement();
+ String value = properties.getProperty(key);
+ filter.doFilter(key, value);
+ }
+ }
+
+ private Set<Entry<String, String>> filter(String pattern){
+ final Pattern p = Pattern.compile(pattern);
+ final Set<Entry<String, String>> set =
+ new HashSet<Entry<String, String>>();
+ list(new Filter(){
+ public void doFilter(String key, String value){
+ Matcher m = p.matcher(key);
+ if(m.matches()){
+ set.add(new Entry(key, value));
+ }
+ }
+ });
+ return set;
+ }
+
+ /**
+ * Default delimeter set to `.'
+ */
+ public Set<Entry<String, String>> subset(String key) {
+ return subset(key, '.');
+ }
+
+ public Set<Entry<String, String>> subset(String key,
+ char delimeter){
+ String[] array = key.split("\\"+delimeter);
+ if(LOG.isDebugEnabled()){
+ LOG.debug("array length:"+array.length);
+ }
+ return subset(array);
+ }
+
+ public Set<Entry<String, String>> subset(String ... keys){
+ int pos = 0;
+ StringBuilder pattern = new StringBuilder();
+ for(String key: keys){
+ if(0==pos){
+ pattern.append("^("+key+")");
+ }else {
+ pattern.append("\\.("+key+")");
+ }
+ pos++;
+ }
+ pattern.append("\\..+");
+ if(LOG.isDebugEnabled()){
+ LOG.debug("Pattern string: "+pattern.toString());
+ }
+ return filter(pattern.toString());
+ }
+
+ /**
+ * List all keys without filter any key in properties file.
+ * @return Set that contains all keys.
+ */
+ public Set<String> keys(){
+ final Set<String> keys = new HashSet<String>();
+ list(new Filter(){
+ public void doFilter(String key, String value){
+ keys.add(key);
+ }
+ });
+ return keys;
+ }
+
+ /**
+ * List all values without filter any value in properties file.
+ * @return Set that contains all values.
+ */
+ public Set<String> values(){
+ final Set<String> values = new HashSet<String>();
+ list(new Filter(){
+ public void doFilter(String key, String value){
+ values.add(value);
+ }
+ });
+ return values;
+ }
+
+ /**
+ * Retrieve metrics filter according to the name provided.
+ * @param name name of the filter.
+ * @return MetricsFilter to be obtained.
+ */
+ public MetricsFilter getFilter(String name){ // default.source.filter
+ if(null == name) return null;
+ if(name.isEmpty()) return null;
+ String[] array = name.split("\\.");
+ if(3 != array.length) {
+ LOG.warn("Name is not in a valid form of [prefix].[source|sink].[filter]"+
+ name);
+ return null;
+ }
+ Set<Entry<String, String>> sub = subset(array);
+ if(null == sub) return null;
+ for(Entry<String, String> e: sub){
+ if((e.key()).equals(name+".class")){
+ try {
+ return (MetricsFilter)ReflectionUtils.newInstance(e.value());
+ }catch(ClassNotFoundException cnfe){
+ LOG.warn("Can not load filter from properties file.", cnfe);
+ }
+ }
+ }
+ return new GlobalFilter(subset(array[0], array[1]));
+ }
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFactory.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFactory.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFactory.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFactory.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.util.ReflectionUtils;
+
+
+/**
+ * Entry point to create MetricsSystem, MetricsSource, and MetricsSink.
+ */
+public final class MetricsFactory {
+
+ public static final Log LOG = LogFactory.getLog(MetricsFactory.class);
+
+ private static final ConcurrentMap<Class<?>, Object> cache =
+ new ConcurrentHashMap<Class<?>, Object>();
+
+ /**
+ * Create MetricsSystem.
+ * @param conf supplies configuration parameters to MetricsSystem.
+ */
+ public static final MetricsSystem createMetricsSystem(String prefix,
+ final HamaConfiguration conf){
+ MetricsSystem mc = new DefaultMetricsSystem(prefix, conf);
+ cache.putIfAbsent(MetricsSystem.class, mc);
+ return mc;
+ }
+
+ /**
+ * Create MetricsSource.
+ * @param clazz is the MetricsSource to be created.
+ * @param values are an array as runtime parameters supplied to clazz.
+ */
+ public static final MetricsSource createSource(Class<?> clazz,
+ Object[] values){
+ return createSource(clazz, values, false);
+ }
+
+ /**
+ * Create MetricsSource.
+ * @param clazz is the MetricsSource to be created.
+ * @param params are class type of object values.
+ * @param values are an array as runtime parameters supplied to clazz.
+ */
+ public static final MetricsSource createSource(Class<?> clazz, Class[] params,
+ Object[] values){
+ return createSource(clazz, params, values, false);
+ }
+
+ /**
+ * Create MetricsSource.
+ * @param clazz is the MetricsSource to be created.
+ * @param values are an array as runtime parameters supplied to clazz.
+ * @param toCache indicates if the instance is to be cached.
+ */
+ public static final MetricsSource createSource(Class<?> clazz,
+ Object[] values, boolean toCache){
+ if(!MetricsSource.class.isAssignableFrom(clazz)){
+ throw new IllegalArgumentException("Unknown source "+clazz.getName());
+ }
+ if(toCache){
+ return (MetricsSource) cache.putIfAbsent(clazz,
+ ReflectionUtils.newInstance(clazz, values));
+ }else{
+ return (MetricsSource)ReflectionUtils.newInstance(clazz, values);
+ }
+ }
+
+ /**
+ * Create MetricsSource.
+ * @param clazz is the MetricsSource to be created.
+ * @param params are class type of object values.
+ * @param values are an array as runtime parameters supplied to clazz.
+ * @param toCache indicates if the instance is to be cached.
+ */
+ public static final MetricsSource createSource(Class<?> clazz, Class[] params,
+ Object[] values, boolean toCache){
+ if(!MetricsSource.class.isAssignableFrom(clazz)){
+ throw new IllegalArgumentException("Unknown source "+clazz.getName());
+ }
+ if(toCache){
+ return (MetricsSource) cache.putIfAbsent(clazz,
+ ReflectionUtils.newInstance(clazz, params, values));
+ }else{
+ return (MetricsSource)ReflectionUtils.newInstance(clazz, params, values);
+ }
+ }
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFilter.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFilter.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFilter.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFilter.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+/**
+ * The metrics filter.
+ */
+public interface MetricsFilter {
+
+ /**
+ * Whether to accept the name
+ * @param name to filter on
+ * @return true to accept; false otherwise.
+ */
+ boolean accepts(String name);
+
+ /**
+ * Whether to accept the tags
+ * @param tags to filter on
+ * @return true to accept; false otherwise
+ */
+ boolean accepts(Iterable<MetricsTag> tags);
+
+ /**
+ * Whether to accept the record
+ * @param record to filter on
+ * @return true to accept; false otherwise.
+ */
+ boolean accepts(MetricsRecord record);
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsInfo.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsInfo.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsInfo.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsInfo.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+/**
+ * A class that holds basic Metrics Information, including name and description.
+ */
+public interface MetricsInfo {
+
+ /**
+ * Name of the MetricsSource.
+ * @return String name of MetricsSource.
+ */
+ String name();
+
+ /**
+ * Description of the MetricsSource.
+ * @return String that describes MetricsSource.
+ */
+ String description();
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsRecord.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsRecord.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsRecord.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsRecord.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.List;
+
+/**
+ * Represents a record containing multiple metrics.
+ */
+public final class MetricsRecord {
+
+ private final String name, description;
+ private final List<Metric<? extends Number>> metrics =
+ new CopyOnWriteArrayList<Metric<? extends Number>>();
+ private final List<MetricsTag> tags = new CopyOnWriteArrayList<MetricsTag>();
+
+ public MetricsRecord(String name, String description){
+ this.name = name;
+ this.description = description;
+ }
+
+ public MetricsRecord(String name){
+ this(name, name+" record.");
+ }
+
+ public final String name(){
+ return name;
+ }
+
+ public final String description(){
+ return description;
+ }
+
+ public final void add(Metric<? extends Number> metric){
+ metrics.add(metric);
+ }
+
+ public final void add(List<Metric<? extends Number>> metrics){
+ metrics.addAll(metrics);
+ }
+
+ public final void tag(Enum key, String value){
+ tags.add(new MetricsTag(key.toString(), value));
+ }
+
+ public final void tag(String key, String value){
+ tags.add(new MetricsTag(key, value));
+ }
+
+ public final Collection<MetricsTag> tags(){
+ return Collections.unmodifiableList(tags);
+ }
+
+ public final List<Metric<? extends Number>> metrics(){
+ return Collections.unmodifiableList(metrics);
+ }
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSink.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSink.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSink.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSink.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+public interface MetricsSink {
+
+ /**
+ * Put a metrics reocrd to sink.
+ * @param record to be stored in the sink.
+ */
+ void putMetrics(final MetricsRecord record);
+
+ /**
+ * Flush buffered metrics.
+ */
+ void flush();
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSinkAdaptor.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSinkAdaptor.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSinkAdaptor.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSinkAdaptor.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.List;
+import java.util.Queue;
+import static java.util.concurrent.TimeUnit.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+
+class MetricsSinkAdaptor {
+
+ public static final Log LOG = LogFactory.getLog(MetricsSinkAdaptor.class);
+
+ private final String name, description;
+ private final BlockingQueue<List<Pair>> queue;
+ private final HamaConfiguration conf;
+ private final int MAX_QUEUE_SIZE;
+ private final Worker worker;
+ private final MetricsFilter sourceFilter, recordFilter, metricFilter;
+
+ /**
+ * Resolve and move pair data to corresponded sink.
+ * Each MetricsSinkAdaptor instance will equipped with one
+ * worker consuming metrics data
+ */
+ private class Worker implements Callable {
+ private final ScheduledExecutorService sched;
+ private final BlockingQueue<List<Pair>> queue;
+ private final MetricsSink sink;
+ private final AtomicBoolean state = new AtomicBoolean(false);
+
+ Worker(BlockingQueue<List<Pair>> queue, MetricsSink sink){
+ this.queue = queue;
+ this.sink = sink;
+ this.sched = Executors.newScheduledThreadPool(1);
+ }
+
+ void start(){
+ state.set(true);
+ sched.schedule(this, 0, SECONDS);
+ }
+
+ MetricsSink sink(){
+ return this.sink;
+ }
+
+ /**
+ * Consume metrics with corresponded sink.
+ */
+ public Object call() throws Exception {
+ while(state.get()){
+ List<Pair> pairs = this.queue.take();
+ if(LOG.isDebugEnabled()){
+ LOG.debug(" Pairs (in queue) size "+((null!=pairs)?pairs.size():"0"));
+ }
+ for(Pair nameAndRecords: pairs){ // adpt.name & metricsRecords
+ if(LOG.isDebugEnabled()){
+ LOG.debug(" Pair's name "+nameAndRecords.name()+" record size "+
+ ((null!=nameAndRecords.records())?nameAndRecords.records().size():"0"));
+ }
+ if (sourceFilter == null || sourceFilter.accepts(nameAndRecords.name())) {
+ for(MetricsRecord record: nameAndRecords.records()){
+ if (recordFilter == null || recordFilter.accepts(record)) { // TODO: context
+ sink.putMetrics(record);
+ }
+ }// for
+ }
+ }
+ }
+ return null;
+ }
+
+ void stop(){
+ state.set(false);
+ sched.shutdown();
+ }
+ }
+
+ MetricsSinkAdaptor(String name, String description, MetricsSink sink,
+ BlockingQueue<List<Pair>> queue, HamaConfiguration conf,
+ MetricsFilter sourceFilter, MetricsFilter recordFilter,
+ MetricsFilter metricFilter){
+ this.name = name;
+ this.description = description;
+ this.queue = queue; // contains a list of name and List<MetricsRecords> pair
+ this.conf = conf;
+ this.MAX_QUEUE_SIZE = conf.getInt("bsp.metrics.max_queue_size", 100);
+ worker = new Worker(queue, sink);
+ this.sourceFilter = sourceFilter;
+ this.recordFilter = recordFilter;
+ this.metricFilter = metricFilter;
+ }
+
+ /**
+ * Store metrics data into queue.
+ */
+ void putMetrics(final List<Pair> pairs){
+ try{
+ if(queue.size() == this.MAX_QUEUE_SIZE){
+ queue.take();
+ }
+ queue.put(pairs);
+ }catch(Exception e){
+ LOG.error("Fail to add data to designated queue.", e);
+ }
+ }
+
+ void start(){
+ worker.start();
+ }
+
+ void stop(){
+ worker.stop();
+ }
+
+ final String name(){
+ return this.name;
+ }
+
+ final String description(){
+ return this.description;
+ }
+
+ public MetricsSink sink(){
+ return worker.sink();
+ }
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSource.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSource.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSource.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSource.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.List;
+
+public interface MetricsSource extends MetricsInfo{
+
+ /**
+ * Collect metrics data.
+ * @param collector is used to collect metrics records.
+ */
+ void snapshot(final List<MetricsRecord> collector);
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSourceAdaptor.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSourceAdaptor.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSourceAdaptor.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSourceAdaptor.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import javax.management.Attribute;
+import javax.management.AttributeList;
+import javax.management.AttributeNotFoundException;
+import javax.management.DynamicMBean;
+import javax.management.InvalidAttributeValueException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanException;
+import javax.management.MBeanInfo;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.ReflectionException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+
+class MetricsSourceAdaptor implements DynamicMBean {
+
+ public static final Log LOG = LogFactory.getLog(MetricsSourceAdaptor.class);
+
+ private final String prefix;
+ private final AtomicReference<List<MetricsRecord>> lastRecs =
+ new AtomicReference<List<MetricsRecord>>();
+ private final AtomicReference<MBeanInfo> infoCache =
+ new AtomicReference<MBeanInfo>();
+ private final ConcurrentMap<String, Attribute> attrCache;
+ private final long jmxCacheTTL;
+ private final AtomicReference<ObjectName> mbeanName =
+ new AtomicReference<ObjectName>();
+ private final AtomicBoolean startMBeans = new AtomicBoolean(false);
+ private final MetricsSource source;
+ private String name, description;
+
+ MetricsSourceAdaptor(final String prefix, final String name,
+ final String description, final MetricsSource source,
+ final long jmxCacheTTL, final HamaConfiguration conf){
+ this.prefix = prefix;
+ this.attrCache = new ConcurrentHashMap<String, Attribute>();
+ this.name = name;
+ this.description = description;
+ this.source = source;
+ this.jmxCacheTTL = jmxCacheTTL;
+ startMBeans.set(conf.getBoolean("bsp.metrics.start_mbeans", true));
+ }
+
+ void start(){
+ if(null != mbeanName.get()) return;
+ if(startMBeans.get()) startMBeans();
+ }
+
+ void stop(){
+ stopMBeans();
+ }
+
+ public void stopMBeans(){
+ final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if(null == mbeanName.get()) {
+ LOG.error("MBean object name not found!");
+ throw new NullPointerException("MBean object name not found!");
+ }
+ try {
+ mbs.unregisterMBean(mbeanName.get());
+ } catch (Exception e) {
+ LOG.warn("Error unregistering "+ mbeanName.get(), e);
+ }
+ mbeanName.set(null);
+ }
+
+ private void startMBeans(){
+ final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ String mbeanStr = "Hama:service="+ prefix +",name="+ name();//+",sub=Stats";
+ //String nameStr = "Hadoop:service="+ serviceName +",name="+ nameName; -> haddop metrics2
+ // Hama:service=MetricsSystem,sub=Stats,name=MetricsSystem,sub=Stats,sub=Stats -> error in test
+ try{
+ this.mbeanName.set(new ObjectName(mbeanStr));
+ }catch(Exception e){
+ LOG.error("Unable to create MBean Object name "+mbeanStr, e);
+ }
+ try{
+ mbs.registerMBean(this, mbeanName.get());
+ LOG.info("Registered MBean "+mbeanName);
+ }catch(Exception e){
+ LOG.error("Unable to register MBean "+mbeanStr, e);
+ }
+ }
+
+ @Override
+ public synchronized Object getAttribute(String attribute)
+ throws AttributeNotFoundException, MBeanException, ReflectionException {
+ updateJmxCache();
+ Attribute a = attrCache.get(attribute);
+ if (a == null) {
+ throw new AttributeNotFoundException(attribute +" not found");
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(attribute +": "+ a);
+ }
+ return a.getValue();
+ }
+
+ @Override
+ public synchronized AttributeList getAttributes(String[] attributes) {
+ updateJmxCache();
+ AttributeList ret = new AttributeList();
+ for (String key : attributes) {
+ Attribute attr = attrCache.get(key);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(key +": "+ attr);
+ }
+ ret.add(attr);
+ }
+ return ret;
+ }
+
+ @Override
+ public MBeanInfo getMBeanInfo() {
+ updateJmxCache();
+ return infoCache.get();
+ }
+
+ private void updateJmxCache(){
+ long jmxCacheTS = 0;
+ if (System.currentTimeMillis() - jmxCacheTS >= jmxCacheTTL) {
+ if(null == lastRecs.get()){
+ final List<MetricsRecord> collector = new ArrayList<MetricsRecord>();
+ getMetrics(collector);
+ }
+ int oldCacheSize = attrCache.size();
+ int newCacheSize = updateAttrCache();
+ if(oldCacheSize < newCacheSize){
+ updateInfoCache();
+ }
+ jmxCacheTS = System.currentTimeMillis();
+ lastRecs.set(null);
+ }
+ }
+
+ private void updateInfoCache(){
+ List<MBeanAttributeInfo> attrs = new ArrayList<MBeanAttributeInfo>();
+ for(MetricsRecord record: lastRecs.get()){ // tag
+ for(MetricsTag t: record.tags()){
+ attrs.add(new MBeanAttributeInfo("tag."+t.name(), "java.lang.String",
+ t.name()+" record", true, false, false)); // read-only
+ }
+ for(Metric m: record.metrics()){ // metrics
+ attrs.add(new MBeanAttributeInfo(m.name(), m.value().getClass().toString(),
+ m.name(), true, false, false)); // read-only
+ }
+ }
+ MBeanAttributeInfo[] attrsArray = new MBeanAttributeInfo[attrs.size()];
+ infoCache.set(new MBeanInfo(name(), description(),
+ attrs.toArray(attrsArray), null, null, null));
+ }
+
+ public void getMetrics(final List<MetricsRecord> collector){
+ this.source.snapshot(collector);
+ lastRecs.set(collector);
+ }
+
+ private int updateAttrCache() {
+ int recNo = 0;
+ int numMetrics = 0;
+ for(MetricsRecord record: lastRecs.get()){
+ for(MetricsTag t: record.tags()){
+ setAttrCacheTag(t, recNo);
+ ++numMetrics;
+ }
+ for(Metric m: record.metrics()){
+ setAttrCacheMetric(m, recNo);
+ ++numMetrics;
+ }
+ ++recNo;
+ }
+ return numMetrics;
+ }
+
+ private void setAttrCacheTag(MetricsTag t, int recNo){
+ String key = "tag."+ t.name() + ((0<recNo)?"."+recNo:"");
+ attrCache.put(key, new Attribute(key, t.value()));
+ }
+
+ private void setAttrCacheMetric(Metric m, int recNo){
+ String key = m.name() + ((0<recNo)?"."+recNo:"");
+ attrCache.put(key, new Attribute(key, m.value()));
+ }
+
+
+ @Override
+ public void setAttribute(Attribute attribute)
+ throws AttributeNotFoundException, InvalidAttributeValueException,
+ MBeanException, ReflectionException {
+ throw new UnsupportedOperationException("Metrics are read-only.");
+ }
+
+ @Override
+ public AttributeList setAttributes(AttributeList attributes) {
+ throw new UnsupportedOperationException("Metrics are read-only.");
+ }
+
+ @Override
+ public Object invoke(String actionName, Object[] params, String[] signature)
+ throws MBeanException, ReflectionException {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ String name(){
+ return this.name;
+ }
+
+ String description(){
+ return this.description;
+ }
+
+ public MetricsSource source(){
+ return this.source;
+ }
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSystem.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSystem.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSystem.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSystem.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+/**
+ * A timer thread periodically collects metric information from MetricsSource,
+ * and then put data to MetricsSink.
+ */
+public interface MetricsSystem {
+
+ /**
+ * Prefix value the current metrics system holds.
+ * @return String of prefix value.
+ */
+ String prefix();
+
+ /**
+ * A value the metrics system periodically harvests metrics information.
+ * @return long value the metrics system is configured.
+ */
+ long period();
+
+ /**
+ * Start the metric system.
+ */
+ void start();
+
+ /**
+ * Stop the metric system.
+ */
+ void stop();
+
+ /**
+ * Register the metric source with a name.
+ * @param name as identification for the source.
+ * @param description describes the purpose.
+ * @param source to be used for collecting metrics.
+ */
+ void register(String name, String description, MetricsSource source);
+
+ /**
+ * Register the metric sink with a name.
+ * @param name as identification for the sink.
+ * @param description describes the purpose.
+ * @param sink to be used for consuming metrics.
+ */
+ void register(String name, String description, MetricsSink sink);
+
+ /**
+ * Find MetricsSink with corresponded name.
+ * @param name of sink to be searched.
+ * @return MetricsSink in cache.
+ */
+ MetricsSink findSink(String name);
+
+ /**
+ * Find MetricsSource with corresponded name.
+ * @param name of source to be searched.
+ * @return MetricsSource in cache.
+ */
+ MetricsSource findSource(String name);
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsTag.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsTag.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsTag.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsTag.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+/**
+ * An information classifies MetricsRecord.
+ */
+public final class MetricsTag {
+ private final String name;
+ private final String value;
+
+ public MetricsTag(String name, String value){
+ this.name = name;
+ this.value = value;
+ }
+
+ public final String name(){
+ return this.name;
+ }
+
+ public final String value(){
+ return this.value;
+ }
+
+ @Override
+ public boolean equals(Object target){
+ if (target == this) return true;
+ if (null == target) return false;
+ if (getClass() != target.getClass()) return false;
+
+ MetricsTag t = (MetricsTag) target;
+ if(!t.name.equals(name)) return false;
+ if(!t.value.equals(value)) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode(){
+ int result = 17;
+ result = 37 * result + name().hashCode();
+ result = 37 * result + value().hashCode();
+ return result;
+ }
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Pair.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Pair.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Pair.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Pair.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.List;
+
+final class Pair {
+ private final String name;
+ private final List<MetricsRecord> records;
+
+ Pair(String name, List<MetricsRecord> records){
+ this.name = name;
+ this.records = records;
+ }
+
+ final String name(){
+ return this.name;
+ }
+
+ final List<MetricsRecord> records(){
+ return this.records;
+ }
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/RegexFilter.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/RegexFilter.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/RegexFilter.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/RegexFilter.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * This class provides an abstract implementation of the BSP interface.
+ */
+public final class RegexFilter extends AbstractPatternFilter {
+
+ public static final Log LOG = LogFactory.getLog(RegexFilter.class);
+
+ public RegexFilter(Set<MetricsConfig.Entry<String, String>> set) {
+ super(set);
+ }
+
+ @Override
+ public Pattern compile(String glob) {
+ return Pattern.compile(glob);
+ }
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMetrics.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMetrics.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMetrics.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMetrics.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import static org.apache.hama.metrics.SystemMetrics.Metrics.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+
+public final class SystemMetrics implements MetricsSource {
+
+ public static final Log LOG = LogFactory.getLog(SystemMetrics.class);
+ private final String name, description;
+ private final Map<String, MetricsSource> sources;
+ private final Map<String, MetricsSink> sinks;
+
+ public enum Metrics{
+ NumActiveSources("Number of active metrics sources"),
+ NumActiveSinks("Number of active metrics sinks"),
+ Hostname("Metrics system hostname");
+
+ private final String desc;
+
+ Metrics(String desc) { this.desc = desc; }
+ public String description() { return desc; }
+
+ @Override public String toString(){
+ return "name "+name()+" description "+desc;
+ }
+ }
+
+ public SystemMetrics(final Map<String, MetricsSource> sources,
+ final Map<String, MetricsSink> sinks, HamaConfiguration conf){
+ this.name = "SystemMetrics";
+ this.description = "System metrics information.";
+ this.sources = sources;
+ this.sinks = sinks;
+ }
+
+ @Override
+ public final String name(){
+ return this.name;
+ }
+
+ @Override
+ public final String description(){
+ return this.description;
+ }
+
+ /**
+ * Record metrics data, and collect those information as records.
+ * @param collector of records to be collected.
+ */
+ public void snapshot(final List<MetricsRecord> collector){
+ final MetricsRecord record = new MetricsRecord(name(), description());
+ record.add(new Metric(NumActiveSources, sources.size()));
+ record.add(new Metric(NumActiveSinks, sinks.size()));
+ record.tag(Hostname, DefaultMetricsSystem.hostname());
+ collector.add(record);
+ }
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMonitorSink.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMonitorSink.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMonitorSink.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMonitorSink.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Provide the latest metrics information to master server.
+ */
+public final class SystemMonitorSink implements MetricsSink {
+
+ public static final Log LOG = LogFactory.getLog(SystemMonitorSink.class);
+
+ private final AtomicReference<MetricsRecord> holder =
+ new AtomicReference<MetricsRecord>();
+
+ @Override
+ public void putMetrics(final MetricsRecord record){
+ if(LOG.isDebugEnabled()){
+ LOG.debug("MetricsRecord size "+
+ (null!=record?record.metrics().size():"0"));
+ }
+ holder.set(record);
+ }
+
+ /**
+ * Obtain the latest metrics record.
+ * @return the latest metrics record.
+ */
+ public MetricsRecord get(){
+ return holder.get();
+ }
+
+ @Override
+ public void flush(){}
+
+
+}
Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/util/ReflectionUtils.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/util/ReflectionUtils.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/util/ReflectionUtils.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/util/ReflectionUtils.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.util;
+
+import java.lang.reflect.Constructor;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Refelction utility for BSP programmes.
+ */
+public class ReflectionUtils {
+
+ public static final Log LOG = LogFactory.getLog(ReflectionUtils.class);
+
+ private static final Map<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE =
+ new ConcurrentHashMap<Class<?>, Constructor<?>>();
+
+ @SuppressWarnings("unchecked")
+ public static <T> T newInstance(String className)
+ throws ClassNotFoundException {
+ T result;
+ try{
+ Class<T> theClass = (Class<T>)Class.forName(className);
+ Constructor<T> meth = (Constructor<T>) CONSTRUCTOR_CACHE.get(theClass);
+ if(null == meth) {
+ meth = theClass.getDeclaredConstructor(new Class[0]);
+ CONSTRUCTOR_CACHE.put(theClass, meth);
+ }
+ result = meth.newInstance();
+ }catch(Exception e){
+ throw new RuntimeException(e);
+ }
+ return result;
+ }
+
+ /**
+ * Create an instance using class literal name and object values supplied.
+ * @param className is the string name of the class to be created.
+ * @param values supplied in object array.
+ * @exception ClassNotFoundException
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> T newInstance(String className, Object[] values)
+ throws ClassNotFoundException{
+ return newInstance((Class<T>)Class.forName(className), values);
+ }
+
+ /**
+ * Create an instance with corresponded class and object values supplied. Constructor
+ * @param theClass supplies instance to be created.
+ * @param values are parameters applied when instance is created.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> T newInstance(Class<T> theClass, Object[] values) {
+ T result;
+ try{
+ Constructor<T> meth = (Constructor<T>) CONSTRUCTOR_CACHE.get(theClass);
+ if (null == meth) {
+ Class[] parameters = new Class[values.length];
+ int idx = 0;
+ for(Object value: values){
+ parameters[idx++] = value.getClass();
+ }
+ meth = theClass.getDeclaredConstructor(parameters);
+ meth.setAccessible(true);
+ CONSTRUCTOR_CACHE.put(theClass, meth);
+ }
+ result = meth.newInstance(values);
+ }catch(Exception e){
+ throw new RuntimeException(e);
+ }
+ return result;
+
+ }
+
+ /**
+ * Create an instance with corresponded class and object values supplied. Constructor
+ * @param theClass supplies instance to be created.
+ * @param parameters are class type of object values supplied.
+ * @param values are parameters applied when instance is created.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> T newInstance(Class<T> theClass, Class[] parameters,
+ Object[] values) {
+ T result;
+ try{
+ Constructor<T> meth = (Constructor<T>) CONSTRUCTOR_CACHE.get(theClass);
+ if (null == meth) {
+ meth = theClass.getDeclaredConstructor(parameters);
+ meth.setAccessible(true);
+ CONSTRUCTOR_CACHE.put(theClass, meth);
+ }
+ result = meth.newInstance(values);
+ }catch(Exception e){
+ throw new RuntimeException(e);
+ }
+ return result;
+ }
+
+}