You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ch...@apache.org on 2011/11/28 12:21:29 UTC

svn commit: r1207083 [1/2] - in /incubator/hama/trunk/core/src: main/java/org/apache/hama/metrics/ main/java/org/apache/hama/util/ test/java/org/apache/hama/metrics/ test/org/ test/org/apache/ test/resources/

Author: chl501
Date: Mon Nov 28 11:21:27 2011
New Revision: 1207083

URL: http://svn.apache.org/viewvc?rev=1207083&view=rev
Log:
[HAMA-363] Add network condition monitoring function to BSPMaster

Added:
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/AbstractPatternFilter.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/DefaultMetricsSystem.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/GlobalFilter.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/JvmMetrics.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Metric.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsConfig.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFactory.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFilter.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsInfo.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsRecord.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSink.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSinkAdaptor.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSource.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSourceAdaptor.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSystem.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsTag.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Pair.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/RegexFilter.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMetrics.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMonitorSink.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/util/ReflectionUtils.java
    incubator/hama/trunk/core/src/test/java/org/apache/hama/metrics/
    incubator/hama/trunk/core/src/test/java/org/apache/hama/metrics/TestMetricsConfig.java
    incubator/hama/trunk/core/src/test/java/org/apache/hama/metrics/TestMetricsSystem.java
    incubator/hama/trunk/core/src/test/java/org/apache/hama/metrics/TestPatternFilter.java
    incubator/hama/trunk/core/src/test/org/
    incubator/hama/trunk/core/src/test/org/apache/
    incubator/hama/trunk/core/src/test/resources/hama-metrics-msys.properties
    incubator/hama/trunk/core/src/test/resources/hama-metrics-test-config.properties

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/AbstractPatternFilter.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/AbstractPatternFilter.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/AbstractPatternFilter.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/AbstractPatternFilter.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public abstract class AbstractPatternFilter implements MetricsFilter {
+
+  public static final Log LOG = LogFactory.getLog(AbstractPatternFilter.class);
+  
+  private final Pattern includePattern;
+  private final Pattern excludePattern;
+  private final Map<String, Pattern> includeTagPatterns;
+  private final Map<String, Pattern> excludeTagPatterns;
+  private final Pattern tagPattern = Pattern.compile("^(\\w+):(.*)");
+  private static final char BACKSLASH = '\\';
+
+  public AbstractPatternFilter(Set<MetricsConfig.Entry<String, String>> set) { 
+    Pattern in = null, ex = null;
+    Map<String, Pattern> inTags = new ConcurrentHashMap<String, Pattern>();
+    Map<String, Pattern> exTags = new ConcurrentHashMap<String, Pattern>();
+    for(MetricsConfig.Entry<String, String> e: set){
+      String key = e.key();
+      String value = e.value();
+      if(LOG.isDebugEnabled()){
+        LOG.info("Subset content of MetricsConfig's key "+key+" value "+value);
+      }
+      // data read from properties files should pay attention to excape sequence
+      if(key.matches("(.*)(\\.include)")){
+        in = compile(value);
+      }else if(key.matches("(.*)(\\.exclude)")){
+        ex = compile(value);
+      }
+
+      if(key.matches("(.*)(\\.include.tags)")){
+        if(null != value && !value.isEmpty()) {
+          Matcher m = tagPattern.matcher(value);
+          if(m.matches()){
+            inTags.put(m.group(1), compile(m.group(2))); 
+          }
+        }
+      }else if(key.matches("(.*)(\\.exclude.tags)")){
+         if(null != value && !value.isEmpty()) {
+          Matcher m = tagPattern.matcher(value);
+          if(m.matches()){
+            exTags.put(m.group(1), compile(m.group(2))); 
+          }
+        }
+      }
+    }
+    this.includePattern = in;
+    this.excludePattern = ex;
+    this.includeTagPatterns = inTags;
+    this.excludeTagPatterns = exTags;
+  }
+
+  /**
+   * Return true if include pattern matches the name; false if exclude pattern 
+   * matches the name. Also, if include pattern is provided but no exclude 
+   * pattern, name won't be accepted as it is white list only mode. 
+   * @return ture if accepted; false otherwise.
+   */
+  @Override
+  public boolean accepts(String name) {
+    if (includePattern != null && includePattern.matcher(name).matches()) {
+      return true;
+    }
+    if ((excludePattern != null && excludePattern.matcher(name).matches())) {
+      return false;
+    }
+    // Reject if no matches in white list only mode.
+    if (includePattern != null && excludePattern == null) {
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * True if tags are listed as include tag pattern; false if tags
+   * are listed in exclude tag pattern. If include and exclude tags pattern
+   * are empty, accepts returns false. 
+   * @param tags to be tested if accepted in filter.
+   * @return boolean value tells if the filter accepts the tag.
+   */
+  @Override
+  public boolean accepts(Iterable<MetricsTag> tags) {
+    // Accept if any include tag pattern matches
+    for (MetricsTag t : tags) {
+      Pattern pat = includeTagPatterns.get(t.name());
+      if (pat != null && pat.matcher(t.value()).matches()) {
+        return true;
+      }
+    }
+    // Reject if any exclude tag pattern matches
+    for (MetricsTag t : tags) {
+      Pattern pat = excludeTagPatterns.get(t.name());
+      if (pat != null && pat.matcher(t.value()).matches()) {
+        return false;
+      }
+    }
+    // Reject if no match in whitelist only mode
+    if (!includeTagPatterns.isEmpty() && excludeTagPatterns.isEmpty()) {
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * True if the record is accepted; false otherwise.
+   * @param record to filter on
+   * @return  true to accept; false otherwise.
+   */
+  public boolean accepts(MetricsRecord record) {
+    return accepts(record.tags());
+  }
+
+  public abstract Pattern compile(String glob); 
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/DefaultMetricsSystem.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/DefaultMetricsSystem.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/DefaultMetricsSystem.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/DefaultMetricsSystem.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,281 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import static java.util.concurrent.TimeUnit.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.util.ReflectionUtils;
+
+/**
+ * A basic implementation which moves metrics data from sources to sinks.
+ */
+final class DefaultMetricsSystem implements MetricsSystem, Runnable{ 
+
+  public static final Log LOG = LogFactory.getLog(DefaultMetricsSystem.class);
+
+  private final String prefix;
+  private final ScheduledExecutorService sched;
+  private final AtomicBoolean state = new AtomicBoolean(false);
+  private final HamaConfiguration conf;
+  private final MetricsConfig config;
+  private final MetricsFilter sourceFilter;
+  private final long period;
+  private final AtomicReference<MetricsSourceAdaptor> sysSource = 
+    new AtomicReference<MetricsSourceAdaptor>();
+  private final ConcurrentMap<String, MetricsSourceAdaptor> sources = 
+    new ConcurrentHashMap<String, MetricsSourceAdaptor>(); 
+  private final ConcurrentMap<String, MetricsSinkAdaptor> sinks = 
+    new ConcurrentHashMap<String, MetricsSinkAdaptor>(); 
+  private final BlockingQueue<List<Pair>> queue;
+  public static final String MS_NAME = "MetricsSystem";
+  public static final String MS_STATS_NAME = MS_NAME +",sub=Stats";
+  public static final String MS_STATS_DESC = "Metrics system metrics";
+  
+
+  DefaultMetricsSystem(String prefix, HamaConfiguration conf){
+    this.prefix = prefix;
+    this.conf = conf;
+    this.config = MetricsConfig.create(this.conf);
+    this.sourceFilter = this.config.getFilter("default.source.filter");
+    this.period = conf.getInt("bsp.metrics.period", 3); 
+    this.queue = new LinkedBlockingQueue<List<Pair>>();
+    int workers = conf.getInt("bsp.metrics.threads_pool", 1);
+    this.sched = Executors.newScheduledThreadPool(workers);
+  }
+
+  public final String prefix(){
+    return this.prefix;
+  }
+
+  public final long period(){
+    return this.period;
+  } 
+
+  /**  
+   * Start MetricsSystem, scheduling with fix rate. The MetricsSystem 
+   * will periodically execute fetching metrics from source to sink.
+   */
+  public void start(){
+    if(true == state.get()){
+      LOG.warn("MetricsSystem has already been started.");
+      return;
+    }
+    configure();
+    state.set(true);
+    this.sched.scheduleAtFixedRate((Runnable)this, 3, period(), SECONDS);
+  }
+
+  private void configure(){
+    configureSinks();
+    configureSources();
+  }
+
+  private void configureSinks(){
+    Set<MetricsConfig.Entry<String, String>> sinkSubset = 
+      this.config.subset(prefix(), "sink");
+    for(MetricsConfig.Entry<String, String> entry: sinkSubset){
+      String key = entry.key();
+      String value = entry.value();
+      if(LOG.isDebugEnabled()){
+        LOG.debug("Configure sinks' key "+key+" value "+value);
+      }
+      if(key.endsWith("class")) {
+        try{
+          LOG.info("Sink name to be registered:"+value);
+          MetricsSink sink =  
+            ReflectionUtils.newInstance(value);
+          register(value, "sink registered from properties file.", (MetricsSink)sink);
+        }catch(ClassNotFoundException cnfe) {
+          LOG.warn("Class "+value+" not found.", cnfe); 
+        }   
+      } 
+    }
+  }
+
+  private void configureSources(){
+    registerSystemSource();
+  }
+
+  private void registerSystemSource(){
+    // sysSource not in sources list
+    MetricsSource src = MetricsFactory.createSource(SystemMetrics.class, 
+      new Class[]{ Map.class, Map.class, HamaConfiguration.class }, 
+      new Object[]{ sources, sinks, conf });
+    MetricsSourceAdaptor srcAdp = 
+      new MetricsSourceAdaptor(prefix(), MS_STATS_NAME, MS_STATS_DESC, src, 
+      (period()+1), conf);
+    sysSource.set(srcAdp);
+    sysSource.get().start();  
+  }
+
+  /**
+   * Stop MetricsSystem. First stop source, then sinks; evetually shutting down
+   * the thread scheduler.
+   */
+  public void stop(){
+    state.set(false);
+    stopSources();
+    stopSinks();
+    sched.shutdown();
+  }
+
+  void stopSources(){
+    for(MetricsSourceAdaptor adpt: sources.values()){
+      adpt.stop();
+    }
+    sysSource.get().stop();
+    sources.clear();
+  }
+
+  void stopSinks(){
+    for(MetricsSinkAdaptor adpt: sinks.values()){
+      adpt.stop();
+    }
+    sinks.clear();
+  }
+
+  /**
+   * Register source and starts as MBean service.
+   * @param name to be referenced to the source. 
+   * @param description describes the purpose.
+   * @param source of metrics.
+   */
+  public void register(String name, String description, MetricsSource source){
+    if(sources.containsKey(name)){
+      LOG.warn("Source with name `"+name+"' is already registered.");
+      return;
+    }
+    MetricsSourceAdaptor adpt = 
+      new MetricsSourceAdaptor(prefix(), name, description, source, period()+1, 
+      this.conf); 
+    sources.putIfAbsent(name, adpt);  
+    adpt.start(); // register as MBean
+    LOG.info("Registered source "+name+".");
+  }
+
+  /**
+   * Register sink with a specified name.
+   * @param name to be associated with sink. 
+   * @param description describes the purpose.
+   * @param sink is used to consumes metrics.
+   */
+  public void register(String name, String description, MetricsSink sink){
+    if(sinks.containsKey(name)){
+      LOG.warn("Sink with name `"+name+"' is already registered.");
+      return;
+    }
+    MetricsFilter srcFilter = this.config.getFilter(prefix()+".source.filter");
+    MetricsFilter recordFilter = this.config.getFilter(prefix()+".record.filter");
+    MetricsFilter metricFilter = this.config.getFilter(prefix()+".metric.filter");
+    MetricsSinkAdaptor adpt = 
+      new MetricsSinkAdaptor(name, description, sink, this.queue, this.conf, 
+      srcFilter, recordFilter, metricFilter);
+    sinks.putIfAbsent(name, adpt);  
+    adpt.start(); 
+    LOG.info("Registered sink "+name+".");
+  }
+
+  /**
+   * Periodically harvest metrics data.  
+   */
+  public void run() {
+    if(LOG.isDebugEnabled()){ LOG.debug(" Sinks size "+sinks.size()); }
+    if(0 < sinks.size()) publishMetrics(sample()); 
+  }
+
+  void publishMetrics(final List<Pair> collections){
+    for(MetricsSinkAdaptor adpt: sinks.values()){
+      if(LOG.isDebugEnabled()){ LOG.debug(" sink adptr name "+adpt.name()); }
+      adpt.putMetrics(collections); 
+    }
+  }
+
+  /**
+   * With corresponded source adaptor, a list of MetricsRecord are collected 
+   * and stored into a list of pair, which will be consumed later by sinks
+   * that has interest.
+   */
+  private List<Pair> sample(){
+    final List<Pair> collector = new ArrayList<Pair>(); 
+    for(Map.Entry<String, MetricsSourceAdaptor> entry:  sources.entrySet()){
+      if(null == this.sourceFilter || sourceFilter.accepts(entry.getKey())){
+        MetricsSourceAdaptor adpt = entry.getValue();
+        collector.add(new Pair(adpt.name(), snapshot(adpt)));
+      }
+    }
+    if(LOG.isDebugEnabled()){ 
+      LOG.debug(" Source adaptor size "+collector.size());
+    }
+    return collector;
+  }
+
+  /**
+   * Source adaptor harvests metrics information into a list of MetricsRecord. 
+   */
+  private List<MetricsRecord> snapshot(MetricsSourceAdaptor srcAdpt){
+    final List<MetricsRecord> collector = new ArrayList<MetricsRecord>(); 
+    srcAdpt.getMetrics(collector); 
+    return collector;
+  }
+
+  static String hostname() {
+    try {
+      return InetAddress.getLocalHost().getHostName();
+    }
+    catch (Exception e) {
+      LOG.error("Error getting localhost name. "+
+                "hostname() would return 'localhost'", e);
+    }
+    return "localhost";
+  }
+
+  @Override
+  public MetricsSink findSink(String name){
+    MetricsSinkAdaptor adpt = sinks.get(name);
+    if(null != adpt){
+      return adpt.sink(); 
+    }
+    return null;
+  }
+
+  @Override
+  public MetricsSource findSource(String name){
+    MetricsSourceAdaptor adpt = sources.get(name);
+    if(null != adpt){
+      return adpt.source();
+    }
+    return null;
+  }
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/GlobalFilter.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/GlobalFilter.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/GlobalFilter.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/GlobalFilter.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * This class provides an abstract implementation of the BSP interface.
+ */
+public final class GlobalFilter extends AbstractPatternFilter {
+
+  public static final Log LOG = LogFactory.getLog(GlobalFilter.class);
+  
+  private static final char BACKSLASH = '\\';
+
+  public GlobalFilter(Set<MetricsConfig.Entry<String, String>> set) { 
+    super(set);
+  }
+
+  /**
+   * Set and compile a glob pattern
+   * @param glob  the glob pattern string
+   */
+  @Override
+  public Pattern compile(String glob) {
+    StringBuilder regex = new StringBuilder();
+    int setOpen = 0;
+    int curlyOpen = 0;
+    int len = glob.length();
+    boolean hasWildcard = false;
+
+    for (int i = 0; i < len; i++) {
+      char c = glob.charAt(i);
+
+      switch (c) {
+        case BACKSLASH:
+          if (++i >= len) {
+            error("Missing escaped character", glob, i);
+          }
+          regex.append(c).append(glob.charAt(i));
+          continue;
+        case '.':
+        case '$':
+        case '(':
+        case ')':
+        case '|':
+        case '+':
+          // escape regex special chars that are not glob special chars
+          regex.append(BACKSLASH);
+          break;
+        case '*':
+          regex.append('.');
+          hasWildcard = true;
+          break;
+        case '?':
+          regex.append('.');
+          hasWildcard = true;
+          continue;
+        case '{': // start of a group
+          regex.append("(?:"); // non-capturing
+          curlyOpen++;
+          hasWildcard = true;
+          continue;
+        case ',':
+          regex.append(curlyOpen > 0 ? '|' : c);
+          continue;
+        case '}':
+          if (curlyOpen > 0) {
+            // end of a group
+            curlyOpen--;
+            regex.append(")");
+            continue;
+          }
+          break;
+        case '[':
+          if (setOpen > 0) {
+            error("Unclosed character class", glob, i);
+          }
+          setOpen++;
+          hasWildcard = true;
+          break;
+        case '^': // ^ inside [...] can be unescaped
+          if (setOpen == 0) {
+            regex.append(BACKSLASH);
+          }
+          break;
+        case '!': // [! needs to be translated to [^
+          regex.append(setOpen > 0 && '[' == glob.charAt(i - 1) ? '^' : '!');
+          continue;
+        case ']':
+          // Many set errors like [][] could not be easily detected here,
+          // as []], []-] and [-] are all valid POSIX glob and java regex.
+          // We'll just let the regex compiler do the real work.
+          setOpen = 0;
+          break;
+        default:
+      }
+      regex.append(c);
+    }
+
+    if (setOpen > 0) {
+      error("Unclosed character class", glob, len);
+    }
+    if (curlyOpen > 0) {
+      error("Unclosed group", glob, len);
+    }
+    return Pattern.compile(regex.toString());
+  }
+
+  private static void error(String message, String pattern, int pos) {
+    throw new PatternSyntaxException(message, pattern, pos);
+  }
+
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/JvmMetrics.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/JvmMetrics.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/JvmMetrics.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/JvmMetrics.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.ArrayList;
+import java.util.List;
+import static java.lang.Thread.State.*;
+import static org.apache.hama.metrics.JvmMetrics.Metrics.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+
+public final class JvmMetrics  implements MetricsSource {
+
+  public static final Log LOG = LogFactory.getLog(JvmMetrics.class);
+
+  private final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+  private final List<GarbageCollectorMXBean> gcBeans =
+      ManagementFactory.getGarbageCollectorMXBeans();
+  private final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+  private static final long M = 1024*1024;
+  private final String name, description, processName, sessionId;
+
+  public enum Metrics{ 
+    MemNonHeapUsedM("Non-heap memory used in MB"),
+    MemNonHeapCommittedM("Non-heap memory committed in MB"),
+    MemHeapUsedM("Heap memory used in MB"),
+    MemHeapCommittedM("Heap memory committed in MB"),
+    GcCount("Total GC count"),
+    GcTimeMillis("Total GC time in milliseconds"),
+    ThreadsNew("Number of new threads"),
+    ThreadsRunnable("Number of runnable threads"),
+    ThreadsBlocked("Number of blocked threads"),
+    ThreadsWaiting("Number of waiting threads"),
+    ThreadsTimedWaiting("Number of timed waiting threads"),
+    ThreadsTerminated("Number of terminated threads");
+  
+    private final String desc;
+  
+    Metrics(String desc) { this.desc = desc; }
+    public String description() { return desc; }
+
+    @Override public String toString(){
+      return "name "+name()+" description "+desc;
+    }
+  }
+
+  public JvmMetrics(String processName, String sessionId, HamaConfiguration conf) {
+    this.name = "JvmMetric";
+    this.description = "JVM metrics information.";
+    this.processName = processName;
+    this.sessionId = sessionId;
+  }
+
+  @Override
+  public final String name(){
+    return this.name;
+  }
+
+  @Override
+  public final String description(){
+    return this.description;
+  }
+
+  /**
+   * Record metrics data, and collect those information as records.   
+   * @param collector of records to be collected.
+   */
+  public void snapshot(final List<MetricsRecord> collector){
+     final MetricsRecord record = new MetricsRecord(name, description);
+     record.tag("ProcessName", processName);
+     record.tag("SessionId", sessionId);
+     memory(record);
+     gc(record);
+     threads(record);
+     collector.add(record);
+  }
+
+  private void memory(final MetricsRecord record){
+    MemoryUsage memNonHeap = memoryMXBean.getNonHeapMemoryUsage();
+    MemoryUsage memHeap = memoryMXBean.getHeapMemoryUsage();
+    record.add(new Metric(MemNonHeapUsedM, memNonHeap.getUsed() / M));
+    record.add(new Metric(MemNonHeapCommittedM, 
+                           memNonHeap.getCommitted() / M));
+    record.add(new Metric(MemHeapUsedM, memHeap.getUsed() / M));
+    record.add(new Metric(MemHeapCommittedM, memHeap.getCommitted() / M));
+  }
+
+  private void gc(final MetricsRecord record){
+    long count = 0;
+    long timeMillis = 0;
+    for (GarbageCollectorMXBean gcBean : gcBeans) {
+      long c = gcBean.getCollectionCount();
+      long t = gcBean.getCollectionTime();
+      String name = gcBean.getName();
+      record.add(new Metric("GcCount"+name, c));
+      record.add(new Metric("GcTimeMillis"+name, t));
+      count += c;
+      timeMillis += t;
+    }
+    record.add(new Metric(GcCount, count));
+    record.add(new Metric(GcTimeMillis, timeMillis));
+  }
+
+  private void threads(final MetricsRecord record){
+    int threadsNew = 0;
+    int threadsRunnable = 0;
+    int threadsBlocked = 0;
+    int threadsWaiting = 0;
+    int threadsTimedWaiting = 0;
+    int threadsTerminated = 0;
+    long threadIds[] = threadMXBean.getAllThreadIds();
+    for (ThreadInfo threadInfo : threadMXBean.getThreadInfo(threadIds, 0)) {
+      if (threadInfo == null) continue; // race protection
+      Thread.State state = threadInfo.getThreadState();
+      if(NEW.equals(state)){           
+        threadsNew++;           
+        break;
+      }else if(RUNNABLE.equals(state)){
+        threadsRunnable++;      
+        break;
+      }else if(BLOCKED.equals(state)){
+        threadsBlocked++;
+        break;
+      }else if(WAITING.equals(state)){
+        threadsWaiting++;
+        break;
+      }else if(TIMED_WAITING.equals(state)){ 
+        threadsTimedWaiting++;  
+        break;
+      }else if(TERMINATED.equals(state)){
+        threadsTerminated++;
+        break;
+      }
+    }
+    record.add(new Metric(ThreadsNew, threadsNew));
+    record.add(new Metric(ThreadsRunnable, threadsRunnable));
+    record.add(new Metric(ThreadsBlocked, threadsBlocked));
+    record.add(new Metric(ThreadsWaiting, threadsWaiting));
+    record.add(new Metric(ThreadsTimedWaiting, threadsTimedWaiting));
+    record.add(new Metric(ThreadsTerminated, threadsTerminated));
+  }
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Metric.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Metric.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Metric.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Metric.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+public final class Metric<T> {
+
+  private final String name;
+  private final String description;
+  private final T value;
+
+  public Metric(Enum name, T value){
+    this(name.toString(), name.toString()+" record", value);
+  }
+
+  public Metric(String name, T value){
+    this(name, name+" metric.", value);
+  }
+
+  public Metric(String name, String description, T value){
+    this.name = name;
+    this.description = description;
+    this.value = value;
+  }
+
+  public final String name(){
+    return this.name;
+  } 
+
+  public final String description(){
+    return this.description;
+  }
+
+  public final T value(){
+    return this.value;
+  }
+
+  @Override
+  public boolean equals(Object target){
+   if (target == this)
+      return true;
+    if (null == target)
+      return false;
+    if (getClass() != target.getClass())
+      return false;
+
+    Metric m = (Metric) target;
+    if(!name().equals(m.name))
+      return false;
+    if(!description().equals(m.description))
+      return false;
+
+    return true;
+  }
+
+  @Override 
+  public int hashCode(){
+    int result = 17;
+    result = 37 * result + name.hashCode();
+    result = 37 * result + description.hashCode();
+    return result;
+  }
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsConfig.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsConfig.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsConfig.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsConfig.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,218 @@
+package org.apache.hama.metrics;
+
+import java.io.InputStream;
+import java.io.IOException;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.util.ReflectionUtils;
+
+public final class MetricsConfig {
+
+  public static final Log LOG = LogFactory.getLog(MetricsConfig.class); 
+
+  public static final String DEFAULT_METRICS_CONFIG = "hama-metrics.properties";
+  private final Properties properties;
+
+  public static final class Entry<KEY, VALUE>{
+    private final KEY key;
+    private VALUE value;
+    Entry(KEY key, VALUE value){
+      this.key = key;
+      this.value = value;
+    } 
+    public KEY key(){
+      return this.key;
+    }
+    public VALUE value(){
+      return this.value;
+    }
+  }
+
+  /**
+   * Internal use only. The purpose is to filter out unnecessary key value pair.
+   */
+  abstract class Filter {
+    public abstract void doFilter(String key, String value);
+  }
+
+  MetricsConfig(Properties properties){
+    this.properties = properties;
+  }
+
+  /**
+   * Create MetricsConfig with HamaConfiguration file specified.
+   * Within HamaConfiguration file, `bsp.metrics.conf_files' can specify
+   * multiple config files with comma as dilemeter. It loads with whichever 
+   * succeeded first.
+   * @param conf file contains multiple metrics properties configuration files.
+   * @return MetricsConfig file which contains all properties.
+   */ 
+  public static MetricsConfig create(HamaConfiguration conf){
+     String configFiles = 
+       conf.get("bsp.metrics.conf_files", DEFAULT_METRICS_CONFIG);
+     return loadFirst(conf, configFiles.split(",")); 
+  }  
+
+  static MetricsConfig loadFirst(HamaConfiguration conf, String... fileNames) {
+    StringBuilder names = new StringBuilder();
+    for (String name : fileNames) {
+      InputStream stream = null;
+      ClassLoader loader = Thread.currentThread().getContextClassLoader();
+      if(null != loader) { 
+        stream = loader.getSystemResourceAsStream(name.trim());
+      }
+      if(null != stream) {
+        Properties prop = new Properties();
+        try{
+          prop.load(stream);
+        }catch(IOException ioe){
+          names.append(name+" "); 
+          LOG.warn("Can not load properties from file "+name, ioe); 
+        }
+        return new MetricsConfig(prop);
+      }else{
+        names.append(name+" "); 
+        LOG.warn("Can not load properties from file "+name);
+      }
+    }
+    throw new RuntimeException("Can not load configuration files "+
+          names.toString());
+  }
+
+  /**
+   * Return all properties stored as MetricsConfig.Entry. It is accessible 
+   * by calling key() and value() for obtaining key, value data respectively. 
+   * @return Set that contains all properties.
+   */
+  public Set<Entry<String, String>> entrySet(){
+    final Set<Entry<String, String>> set = new HashSet<Entry<String, String>>();
+    list(new Filter(){
+      public void doFilter(String key, String value){
+        set.add(new Entry(key, value));
+      }
+    });  
+    return set;
+  }
+
+  private void list(Filter filter){
+    for(Enumeration e = properties.propertyNames(); e.hasMoreElements(); ){
+      String key = (String)e.nextElement();
+      String value = properties.getProperty(key); 
+      filter.doFilter(key, value);
+    }
+  }
+
+  private Set<Entry<String, String>> filter(String pattern){
+    final Pattern p = Pattern.compile(pattern); 
+    final Set<Entry<String, String>> set = 
+      new HashSet<Entry<String, String>>();
+    list(new Filter(){
+      public void doFilter(String key, String value){
+        Matcher m = p.matcher(key);    
+        if(m.matches()){
+           set.add(new Entry(key, value));
+        }
+      }
+    }); 
+    return set;
+  }
+
+  /**
+   * Default delimeter set to `.'
+   */
+  public Set<Entry<String, String>> subset(String key) {
+    return subset(key, '.');
+  }
+  
+  public Set<Entry<String, String>> subset(String key, 
+      char delimeter){
+    String[] array = key.split("\\"+delimeter);
+    if(LOG.isDebugEnabled()){
+      LOG.debug("array length:"+array.length);
+    }
+    return subset(array);
+  }
+
+  public Set<Entry<String, String>> subset(String ... keys){
+    int pos = 0;
+    StringBuilder pattern = new StringBuilder();
+    for(String key: keys){
+      if(0==pos){ 
+        pattern.append("^("+key+")");
+      }else {
+        pattern.append("\\.("+key+")");
+      }
+      pos++;
+    }
+    pattern.append("\\..+");
+    if(LOG.isDebugEnabled()){
+      LOG.debug("Pattern string: "+pattern.toString());
+    }
+    return filter(pattern.toString()); 
+  }
+
+  /**
+   * List all keys without filter any key in properties file.
+   * @return Set that contains all keys.
+   */
+  public Set<String> keys(){
+    final Set<String> keys = new HashSet<String>();
+    list(new Filter(){
+      public void doFilter(String key, String value){
+        keys.add(key);
+      }
+    });
+    return keys;
+  }
+  
+  /**
+   * List all values without filter any value in properties file.
+   * @return Set that contains all values.
+   */
+  public Set<String> values(){
+    final Set<String> values = new HashSet<String>();
+    list(new Filter(){
+      public void doFilter(String key, String value){
+        values.add(value);
+      }
+    });
+    return values;
+  } 
+
+  /**
+   * Retrieve metrics filter according to the name provided.
+   * @param name name of the filter.  
+   * @return MetricsFilter to be obtained.
+   */
+  public MetricsFilter getFilter(String name){ // default.source.filter
+    if(null == name) return null;
+    if(name.isEmpty()) return null;
+    String[] array = name.split("\\.");
+    if(3 != array.length) {
+      LOG.warn("Name is not in a valid form of [prefix].[source|sink].[filter]"+
+               name);
+      return null;
+    }
+    Set<Entry<String, String>> sub = subset(array);
+    if(null == sub) return null;
+    for(Entry<String, String> e: sub){
+      if((e.key()).equals(name+".class")){
+         try {
+           return (MetricsFilter)ReflectionUtils.newInstance(e.value());
+         }catch(ClassNotFoundException cnfe){
+           LOG.warn("Can not load filter from properties file.", cnfe);
+         }
+      }
+    }
+    return new GlobalFilter(subset(array[0], array[1]));
+  }
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFactory.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFactory.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFactory.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFactory.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.util.ReflectionUtils;
+
+
+/**
+ * Entry point to create MetricsSystem, MetricsSource, and MetricsSink.
+ */
+public final class MetricsFactory {
+ 
+  public static final Log LOG = LogFactory.getLog(MetricsFactory.class);
+
+  private static final ConcurrentMap<Class<?>, Object> cache = 
+    new ConcurrentHashMap<Class<?>, Object>();
+
+  /**
+   * Create MetricsSystem.
+   * @param conf supplies configuration parameters to MetricsSystem. 
+   */
+  public static final MetricsSystem createMetricsSystem(String prefix,
+      final HamaConfiguration conf){
+     MetricsSystem mc = new DefaultMetricsSystem(prefix, conf);
+     cache.putIfAbsent(MetricsSystem.class, mc);
+     return mc;
+  }
+
+  /**
+   * Create MetricsSource.
+   * @param clazz is the MetricsSource to be created.
+   * @param values are an array as runtime parameters supplied to clazz.
+   */
+  public static final MetricsSource createSource(Class<?> clazz, 
+      Object[] values){
+    return createSource(clazz, values, false);
+  }
+
+  /**
+   * Create MetricsSource.
+   * @param clazz is the MetricsSource to be created.
+   * @param params are class type of object values. 
+   * @param values are an array as runtime parameters supplied to clazz.
+   */
+  public static final MetricsSource createSource(Class<?> clazz, Class[] params,
+      Object[] values){
+    return createSource(clazz, params, values, false);
+  }
+
+  /**
+   * Create MetricsSource.
+   * @param clazz is the MetricsSource to be created.
+   * @param values are an array as runtime parameters supplied to clazz.
+   * @param toCache indicates if the instance is to be cached. 
+   */
+  public static final MetricsSource createSource(Class<?> clazz, 
+      Object[] values, boolean toCache){
+    if(!MetricsSource.class.isAssignableFrom(clazz)){
+      throw new IllegalArgumentException("Unknown source "+clazz.getName());
+    }
+    if(toCache){
+      return (MetricsSource) cache.putIfAbsent(clazz, 
+             ReflectionUtils.newInstance(clazz, values));
+    }else{
+      return (MetricsSource)ReflectionUtils.newInstance(clazz, values);
+    }
+  }
+
+  /**
+   * Create MetricsSource.
+   * @param clazz is the MetricsSource to be created.
+   * @param params are class type of object values. 
+   * @param values are an array as runtime parameters supplied to clazz.
+   * @param toCache indicates if the instance is to be cached. 
+   */
+  public static final MetricsSource createSource(Class<?> clazz, Class[] params, 
+      Object[] values, boolean toCache){
+    if(!MetricsSource.class.isAssignableFrom(clazz)){
+      throw new IllegalArgumentException("Unknown source "+clazz.getName());
+    }
+    if(toCache){
+      return (MetricsSource) cache.putIfAbsent(clazz, 
+             ReflectionUtils.newInstance(clazz, params, values));
+    }else{
+      return (MetricsSource)ReflectionUtils.newInstance(clazz, params, values);
+    }
+  }
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFilter.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFilter.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFilter.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsFilter.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+/**
+ * The metrics filter. 
+ */
+public interface MetricsFilter {
+
+  /**
+   * Whether to accept the name
+   * @param name  to filter on
+   * @return  true to accept; false otherwise.
+   */
+  boolean accepts(String name);
+
+  /**
+   * Whether to accept the tags
+   * @param tags to filter on
+   * @return  true to accept; false otherwise
+   */
+  boolean accepts(Iterable<MetricsTag> tags);
+
+  /**
+   * Whether to accept the record
+   * @param record  to filter on
+   * @return  true to accept; false otherwise.
+   */
+  boolean accepts(MetricsRecord record); 
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsInfo.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsInfo.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsInfo.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsInfo.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+/**
+ * A class that holds basic Metrics Information, including name and description.
+ */
+public interface MetricsInfo {
+
+  /**
+   * Name of the MetricsSource.
+   * @return String name of MetricsSource.
+   */
+  String name();
+
+  /**
+   * Description of the MetricsSource.
+   * @return String that describes MetricsSource.
+   */
+  String description();
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsRecord.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsRecord.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsRecord.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsRecord.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.List;
+
+/**
+ * Represents a record containing multiple metrics. 
+ */
+public final class MetricsRecord  {
+
+  private final String name, description;
+  private final List<Metric<? extends Number>> metrics = 
+    new CopyOnWriteArrayList<Metric<? extends Number>>();
+  private final List<MetricsTag> tags = new CopyOnWriteArrayList<MetricsTag>();
+
+  public MetricsRecord(String name, String description){
+    this.name = name;
+    this.description = description;
+  }
+
+  public MetricsRecord(String name){
+    this(name, name+" record."); 
+  }
+
+  public final String name(){
+    return name;
+  }
+
+  public final String description(){
+    return description;
+  }
+
+  public final void add(Metric<? extends Number> metric){
+    metrics.add(metric);
+  }
+
+  public final void add(List<Metric<? extends Number>> metrics){
+    metrics.addAll(metrics);
+  }
+
+  public final void tag(Enum key, String value){
+    tags.add(new MetricsTag(key.toString(), value));
+  }
+
+  public final void tag(String key, String value){
+    tags.add(new MetricsTag(key, value));
+  }
+
+  public final Collection<MetricsTag> tags(){
+    return Collections.unmodifiableList(tags);
+  }
+
+  public final List<Metric<? extends Number>> metrics(){
+    return Collections.unmodifiableList(metrics);
+  }
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSink.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSink.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSink.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSink.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+public interface MetricsSink {
+
+  /**
+   * Put a metrics reocrd to sink.
+   * @param record to be stored in the sink.
+   */
+  void putMetrics(final MetricsRecord record);
+
+  /**
+   * Flush buffered metrics.
+   */
+  void flush();
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSinkAdaptor.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSinkAdaptor.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSinkAdaptor.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSinkAdaptor.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.List;
+import java.util.Queue;
+import static java.util.concurrent.TimeUnit.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+
+class MetricsSinkAdaptor {
+
+  public static final Log LOG = LogFactory.getLog(MetricsSinkAdaptor.class);
+
+  private final String name, description;
+  private final BlockingQueue<List<Pair>> queue; 
+  private final HamaConfiguration conf;
+  private final int MAX_QUEUE_SIZE;
+  private final Worker worker;
+  private final MetricsFilter sourceFilter, recordFilter, metricFilter;
+
+  /**
+   * Resolve and move pair data to corresponded sink. 
+   * Each MetricsSinkAdaptor instance will equipped with one
+   * worker consuming metrics data
+   */
+  private class Worker implements Callable {
+    private final ScheduledExecutorService sched;
+    private final BlockingQueue<List<Pair>> queue;
+    private final MetricsSink sink;
+    private final AtomicBoolean state = new AtomicBoolean(false);
+
+    Worker(BlockingQueue<List<Pair>> queue, MetricsSink sink){
+      this.queue = queue;
+      this.sink = sink;
+      this.sched = Executors.newScheduledThreadPool(1);
+    } 
+
+    void start(){
+      state.set(true);
+      sched.schedule(this, 0, SECONDS); 
+    }
+ 
+    MetricsSink sink(){
+      return this.sink;
+    }
+
+    /**
+     * Consume metrics with corresponded sink.
+     */
+    public Object call() throws Exception {
+      while(state.get()){
+        List<Pair> pairs = this.queue.take();
+        if(LOG.isDebugEnabled()){
+          LOG.debug(" Pairs (in queue) size "+((null!=pairs)?pairs.size():"0"));
+        }
+        for(Pair nameAndRecords: pairs){ // adpt.name & metricsRecords
+          if(LOG.isDebugEnabled()){
+            LOG.debug(" Pair's name "+nameAndRecords.name()+" record size "+
+            ((null!=nameAndRecords.records())?nameAndRecords.records().size():"0"));
+          }
+          if (sourceFilter == null || sourceFilter.accepts(nameAndRecords.name())) {
+            for(MetricsRecord record: nameAndRecords.records()){
+              if (recordFilter == null || recordFilter.accepts(record)) { // TODO: context
+                sink.putMetrics(record);
+              }
+            }// for
+          }
+        }
+      }
+      return null;
+    }
+
+    void stop(){
+      state.set(false);
+      sched.shutdown();
+    }
+  }
+
+  MetricsSinkAdaptor(String name, String description, MetricsSink sink, 
+      BlockingQueue<List<Pair>> queue, HamaConfiguration conf, 
+      MetricsFilter sourceFilter, MetricsFilter recordFilter, 
+      MetricsFilter metricFilter){
+    this.name = name;
+    this.description = description;
+    this.queue = queue; // contains a list of name and List<MetricsRecords> pair
+    this.conf = conf;
+    this.MAX_QUEUE_SIZE = conf.getInt("bsp.metrics.max_queue_size", 100);
+    worker = new Worker(queue, sink);
+    this.sourceFilter = sourceFilter;
+    this.recordFilter = recordFilter;
+    this.metricFilter = metricFilter;
+  }
+
+  /**
+   * Store metrics data into queue.
+   */
+  void putMetrics(final List<Pair> pairs){
+    try{
+      if(queue.size() == this.MAX_QUEUE_SIZE){
+        queue.take();
+      }
+      queue.put(pairs);
+    }catch(Exception e){
+      LOG.error("Fail to add data to designated queue.", e);
+    }
+  }
+
+  void start(){
+    worker.start(); 
+  } 
+
+  void stop(){
+    worker.stop();
+  }
+  
+  final String name(){
+    return this.name;
+  }
+
+  final String description(){
+    return this.description;
+  }
+
+  public MetricsSink sink(){
+    return worker.sink();
+  }
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSource.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSource.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSource.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSource.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.List;
+
+public interface MetricsSource extends MetricsInfo{
+
+   /**
+    * Collect metrics data.
+    * @param collector is used to collect metrics records.
+    */
+   void snapshot(final List<MetricsRecord> collector);
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSourceAdaptor.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSourceAdaptor.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSourceAdaptor.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSourceAdaptor.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import javax.management.Attribute;
+import javax.management.AttributeList;
+import javax.management.AttributeNotFoundException;
+import javax.management.DynamicMBean;
+import javax.management.InvalidAttributeValueException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanException;
+import javax.management.MBeanInfo;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.ReflectionException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+
+class MetricsSourceAdaptor implements DynamicMBean {
+
+  public static final Log LOG = LogFactory.getLog(MetricsSourceAdaptor.class);
+   
+  private final String prefix;
+  private final AtomicReference<List<MetricsRecord>> lastRecs =  
+    new AtomicReference<List<MetricsRecord>>();
+  private final AtomicReference<MBeanInfo> infoCache = 
+    new AtomicReference<MBeanInfo>();
+  private final ConcurrentMap<String, Attribute> attrCache;  
+  private final long jmxCacheTTL;
+  private final AtomicReference<ObjectName> mbeanName = 
+    new AtomicReference<ObjectName>();
+  private final AtomicBoolean startMBeans = new AtomicBoolean(false);
+  private final MetricsSource source;
+  private String name, description;
+  
+  MetricsSourceAdaptor(final String prefix, final String name, 
+      final String description, final MetricsSource source, 
+      final long jmxCacheTTL, final HamaConfiguration conf){
+    this.prefix = prefix;
+    this.attrCache = new ConcurrentHashMap<String, Attribute>();
+    this.name = name;
+    this.description = description;
+    this.source = source;
+    this.jmxCacheTTL = jmxCacheTTL;
+    startMBeans.set(conf.getBoolean("bsp.metrics.start_mbeans", true));
+  }
+
+  void start(){
+    if(null != mbeanName.get()) return;
+    if(startMBeans.get()) startMBeans();
+  }
+
+  void stop(){
+    stopMBeans();
+  }
+
+  public void stopMBeans(){
+    final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();  
+    if(null == mbeanName.get()) {
+      LOG.error("MBean object name not found!");
+      throw new NullPointerException("MBean object name not found!");
+    }
+    try {
+      mbs.unregisterMBean(mbeanName.get());
+    } catch (Exception e) {
+      LOG.warn("Error unregistering "+ mbeanName.get(), e);
+    }
+    mbeanName.set(null);
+  }
+  
+  private void startMBeans(){
+    final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+    String mbeanStr = "Hama:service="+ prefix +",name="+ name();//+",sub=Stats";
+  //String nameStr =  "Hadoop:service="+ serviceName +",name="+ nameName; -> haddop metrics2
+  //                   Hama:service=MetricsSystem,sub=Stats,name=MetricsSystem,sub=Stats,sub=Stats -> error in test
+    try{
+      this.mbeanName.set(new ObjectName(mbeanStr));
+    }catch(Exception e){
+      LOG.error("Unable to create MBean Object name "+mbeanStr, e);
+    }
+    try{
+      mbs.registerMBean(this, mbeanName.get());
+      LOG.info("Registered MBean "+mbeanName);
+    }catch(Exception e){
+      LOG.error("Unable to register MBean "+mbeanStr, e);
+    }
+  }
+
+  @Override
+  public synchronized Object getAttribute(String attribute)
+      throws AttributeNotFoundException, MBeanException, ReflectionException {
+    updateJmxCache();
+    Attribute a = attrCache.get(attribute);
+    if (a == null) {
+      throw new AttributeNotFoundException(attribute +" not found");
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(attribute +": "+ a);
+    }
+    return a.getValue();
+  }
+
+  @Override
+  public synchronized AttributeList getAttributes(String[] attributes) {
+    updateJmxCache();
+    AttributeList ret = new AttributeList();
+    for (String key : attributes) {
+      Attribute attr = attrCache.get(key);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(key +": "+ attr);
+      }
+      ret.add(attr);
+    }
+    return ret;
+  }
+
+  @Override
+  public MBeanInfo getMBeanInfo() {
+    updateJmxCache();
+    return infoCache.get();
+  }
+
+  private void updateJmxCache(){
+    long jmxCacheTS = 0;
+    if (System.currentTimeMillis() - jmxCacheTS >= jmxCacheTTL) {
+      if(null == lastRecs.get()){
+        final List<MetricsRecord> collector = new ArrayList<MetricsRecord>();
+        getMetrics(collector); 
+      }
+      int oldCacheSize = attrCache.size();
+      int newCacheSize = updateAttrCache();
+      if(oldCacheSize < newCacheSize){
+        updateInfoCache(); 
+      }
+      jmxCacheTS = System.currentTimeMillis(); 
+      lastRecs.set(null);
+    }
+  }
+
+  private void updateInfoCache(){
+    List<MBeanAttributeInfo> attrs = new ArrayList<MBeanAttributeInfo>();
+    for(MetricsRecord record: lastRecs.get()){ // tag 
+      for(MetricsTag t: record.tags()){
+        attrs.add(new MBeanAttributeInfo("tag."+t.name(), "java.lang.String", 
+                  t.name()+" record", true, false, false)); // read-only
+      }
+      for(Metric m: record.metrics()){ // metrics
+        attrs.add(new MBeanAttributeInfo(m.name(), m.value().getClass().toString(), 
+                  m.name(), true, false, false)); // read-only
+      }
+    }
+    MBeanAttributeInfo[] attrsArray = new MBeanAttributeInfo[attrs.size()];
+    infoCache.set(new MBeanInfo(name(), description(),  
+                  attrs.toArray(attrsArray), null, null, null));
+  }
+
+  public void getMetrics(final List<MetricsRecord> collector){
+    this.source.snapshot(collector);
+    lastRecs.set(collector);
+  }
+  
+  private int updateAttrCache() {
+    int recNo = 0;
+    int numMetrics = 0;
+    for(MetricsRecord record: lastRecs.get()){
+      for(MetricsTag t:  record.tags()){
+        setAttrCacheTag(t, recNo);  
+        ++numMetrics;
+      }
+      for(Metric m: record.metrics()){
+        setAttrCacheMetric(m, recNo);  
+        ++numMetrics;
+      }
+      ++recNo;
+    } 
+    return numMetrics;
+  }
+
+  private void setAttrCacheTag(MetricsTag t, int recNo){
+    String key = "tag."+ t.name() + ((0<recNo)?"."+recNo:"");
+    attrCache.put(key, new Attribute(key, t.value()));   
+  }
+
+  private void setAttrCacheMetric(Metric m, int recNo){
+    String key = m.name() + ((0<recNo)?"."+recNo:"");
+    attrCache.put(key, new Attribute(key, m.value()));   
+  }
+
+
+  @Override
+  public void setAttribute(Attribute attribute)
+      throws AttributeNotFoundException, InvalidAttributeValueException,
+             MBeanException, ReflectionException {
+    throw new UnsupportedOperationException("Metrics are read-only.");
+  }
+
+  @Override
+  public AttributeList setAttributes(AttributeList attributes) {
+    throw new UnsupportedOperationException("Metrics are read-only.");
+  }
+
+  @Override
+  public Object invoke(String actionName, Object[] params, String[] signature)
+      throws MBeanException, ReflectionException {
+    throw new UnsupportedOperationException("Not supported yet.");
+  }
+
+  String name(){
+    return this.name;
+  }
+
+  String description(){
+    return this.description;
+  }
+
+  public MetricsSource source(){
+    return this.source;
+  }
+  
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSystem.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSystem.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSystem.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsSystem.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+/** 
+ * A timer thread periodically collects metric information from MetricsSource,
+ * and then put data to MetricsSink.
+ */
+public interface MetricsSystem { 
+
+  /**
+   * Prefix value the current metrics system holds.
+   * @return String of prefix value.
+   */
+  String prefix();
+
+  /**
+   * A value the metrics system periodically harvests metrics information.
+   * @return long value the metrics system is configured.
+   */
+  long period();
+
+  /**
+   * Start the metric system.
+   */
+  void start();
+
+  /**
+   * Stop the metric system.
+   */
+  void stop();
+
+  /**
+   * Register the metric source with a name.
+   * @param name as identification for the source. 
+   * @param description describes the purpose. 
+   * @param source to be used for collecting metrics.
+   */
+  void register(String name, String description, MetricsSource source);
+
+  /**
+   * Register the metric sink with a name.
+   * @param name as identification for the sink. 
+   * @param description describes the purpose. 
+   * @param sink to be used for consuming metrics.
+   */
+  void register(String name, String description, MetricsSink sink);
+
+  /**
+   * Find MetricsSink with corresponded name.
+   * @param name of sink to be searched.
+   * @return MetricsSink in cache.
+   */
+  MetricsSink findSink(String name);
+
+  /**
+   * Find MetricsSource with corresponded name.
+   * @param name of source to be searched.
+   * @return MetricsSource in cache.
+   */
+  MetricsSource findSource(String name);
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsTag.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsTag.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsTag.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/MetricsTag.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+/**
+ * An information classifies MetricsRecord.
+ */
+public final class MetricsTag {
+  private final String name;
+  private final String value;
+
+  public MetricsTag(String name, String value){
+    this.name = name;
+    this.value = value;
+  }
+
+  public final String name(){
+    return this.name;
+  }
+
+  public final String value(){
+    return this.value;
+  }
+
+  @Override
+  public boolean equals(Object target){
+   if (target == this) return true;
+    if (null == target) return false;
+    if (getClass() != target.getClass()) return false;
+ 
+    MetricsTag t = (MetricsTag) target;
+    if(!t.name.equals(name)) return false;
+    if(!t.value.equals(value)) return false;
+    return true;
+  }
+
+  @Override
+  public int hashCode(){
+    int result = 17;
+    result = 37 * result + name().hashCode();
+    result = 37 * result + value().hashCode();
+    return result; 
+  }
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Pair.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Pair.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Pair.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/Pair.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.List;
+
+final class Pair {
+  private final String name;
+  private final List<MetricsRecord> records;
+
+  Pair(String name, List<MetricsRecord> records){
+    this.name = name;
+    this.records = records;
+  } 
+
+  final String name(){
+    return this.name;
+  }
+
+  final List<MetricsRecord> records(){
+    return this.records;
+  }
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/RegexFilter.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/RegexFilter.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/RegexFilter.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/RegexFilter.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * This class provides an abstract implementation of the BSP interface.
+ */
+public final class RegexFilter extends AbstractPatternFilter {
+
+  public static final Log LOG = LogFactory.getLog(RegexFilter.class);
+
+  public RegexFilter(Set<MetricsConfig.Entry<String, String>> set) { 
+    super(set);
+  }
+
+  @Override
+  public Pattern compile(String glob) {
+    return Pattern.compile(glob);
+  }
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMetrics.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMetrics.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMetrics.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMetrics.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import static org.apache.hama.metrics.SystemMetrics.Metrics.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hama.HamaConfiguration;
+
+public final class SystemMetrics implements MetricsSource {
+
+  public static final Log LOG = LogFactory.getLog(SystemMetrics.class);
+  private final String name, description;
+  private final Map<String, MetricsSource> sources;
+  private final Map<String, MetricsSink> sinks;
+
+  public enum Metrics{
+    NumActiveSources("Number of active metrics sources"),
+    NumActiveSinks("Number of active metrics sinks"),
+    Hostname("Metrics system hostname");
+
+    private final String desc;
+
+    Metrics(String desc) { this.desc = desc; }
+    public String description() { return desc; }
+
+    @Override public String toString(){
+      return "name "+name()+" description "+desc;
+    }
+  }
+
+  public SystemMetrics(final Map<String, MetricsSource> sources, 
+      final Map<String, MetricsSink> sinks, HamaConfiguration conf){
+    this.name = "SystemMetrics";
+    this.description = "System metrics information.";
+    this.sources = sources;
+    this.sinks = sinks;
+  }
+
+  @Override
+  public final String name(){
+    return this.name;
+  }
+
+  @Override
+  public final String description(){
+    return this.description;
+  }
+
+  /**
+   * Record metrics data, and collect those information as records.   
+   * @param collector of records to be collected.
+   */
+  public void snapshot(final List<MetricsRecord> collector){
+    final MetricsRecord record = new MetricsRecord(name(), description());
+    record.add(new Metric(NumActiveSources, sources.size()));
+    record.add(new Metric(NumActiveSinks, sinks.size()));
+    record.tag(Hostname, DefaultMetricsSystem.hostname());
+    collector.add(record);
+  }
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMonitorSink.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMonitorSink.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMonitorSink.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/metrics/SystemMonitorSink.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.metrics;
+
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Provide the latest metrics information to master server. 
+ */
+public final class SystemMonitorSink implements MetricsSink {
+
+  public static final Log LOG = LogFactory.getLog(SystemMonitorSink.class);
+
+  private final AtomicReference<MetricsRecord> holder =
+    new AtomicReference<MetricsRecord>();
+
+  @Override
+  public void putMetrics(final MetricsRecord record){
+    if(LOG.isDebugEnabled()){
+      LOG.debug("MetricsRecord size "+
+        (null!=record?record.metrics().size():"0"));
+    }
+    holder.set(record);
+  }
+
+  /**
+   * Obtain the latest metrics record.
+   * @return the latest metrics record.
+   */
+  public MetricsRecord get(){
+    return holder.get();
+  }
+
+  @Override
+  public void flush(){}
+
+
+}

Added: incubator/hama/trunk/core/src/main/java/org/apache/hama/util/ReflectionUtils.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/util/ReflectionUtils.java?rev=1207083&view=auto
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/util/ReflectionUtils.java (added)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/util/ReflectionUtils.java Mon Nov 28 11:21:27 2011
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.util;
+
+import java.lang.reflect.Constructor;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Refelction utility for BSP programmes.
+ */
+public class ReflectionUtils {
+
+  public static final Log LOG = LogFactory.getLog(ReflectionUtils.class);
+
+  private static final Map<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE = 
+    new ConcurrentHashMap<Class<?>, Constructor<?>>();
+
+  @SuppressWarnings("unchecked")
+  public static <T> T newInstance(String className) 
+      throws ClassNotFoundException {
+    T result;
+    try{
+      Class<T> theClass = (Class<T>)Class.forName(className);
+      Constructor<T> meth = (Constructor<T>) CONSTRUCTOR_CACHE.get(theClass);
+      if(null == meth) {
+        meth = theClass.getDeclaredConstructor(new Class[0]);
+        CONSTRUCTOR_CACHE.put(theClass, meth);
+      }
+      result = meth.newInstance();
+    }catch(Exception e){
+      throw new RuntimeException(e);
+    }
+    return result;
+  }
+
+  /**
+   * Create an instance using class literal name and object values supplied.
+   * @param className is the string name of the class to be created.
+   * @param values supplied in object array.
+   * @exception ClassNotFoundException
+   */
+  @SuppressWarnings("unchecked")
+  public static <T> T newInstance(String className, Object[] values) 
+      throws ClassNotFoundException{
+    return newInstance((Class<T>)Class.forName(className), values);
+  }
+
+  /**
+   * Create an instance with corresponded class and object values supplied. Constructor
+   * @param theClass supplies instance to be created.
+   * @param values are parameters applied when instance is created.
+   */
+  @SuppressWarnings("unchecked")
+  public static <T> T newInstance(Class<T> theClass, Object[] values) {
+    T result;
+    try{
+      Constructor<T> meth = (Constructor<T>) CONSTRUCTOR_CACHE.get(theClass);
+      if (null == meth) {
+        Class[] parameters = new Class[values.length];
+        int idx = 0;
+        for(Object value: values){
+          parameters[idx++] = value.getClass();
+        }     
+        meth = theClass.getDeclaredConstructor(parameters);
+        meth.setAccessible(true);
+        CONSTRUCTOR_CACHE.put(theClass, meth);
+      }
+      result = meth.newInstance(values);
+    }catch(Exception e){
+      throw new RuntimeException(e);
+    }
+    return result;
+
+  }
+
+  /**
+   * Create an instance with corresponded class and object values supplied. Constructor
+   * @param theClass supplies instance to be created.
+   * @param parameters are class type of object values supplied.
+   * @param values are parameters applied when instance is created.
+   */
+  @SuppressWarnings("unchecked")
+  public static <T> T newInstance(Class<T> theClass, Class[] parameters, 
+      Object[] values) {
+    T result;
+    try{
+      Constructor<T> meth = (Constructor<T>) CONSTRUCTOR_CACHE.get(theClass);
+      if (null == meth) {
+        meth = theClass.getDeclaredConstructor(parameters);
+        meth.setAccessible(true);
+        CONSTRUCTOR_CACHE.put(theClass, meth);
+      }
+      result = meth.newInstance(values);
+    }catch(Exception e){
+      throw new RuntimeException(e);
+    }
+    return result;
+  }
+
+}