You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2012/12/20 05:25:32 UTC

[22/51] [partial] GIRAPH-457: update module names (nitay)

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetric.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetric.java b/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetric.java
new file mode 100644
index 0000000..9d782c4
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetric.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+/**
+ * An aggregator over metrics from multiple hosts. Computes min, max, and mean.
+ */
+public class AggregatedMetric {
+  /** Minimum value seen with the host that it came from */
+  private ValueWithHostname min;
+  /** Maximum value seen with the host that it came from */
+  private ValueWithHostname max;
+  /** Total of all the values seen */
+  private long sum;
+  /** Number of values seen */
+  private long count;
+
+  /**
+   * Create new aggregated metric.
+   */
+  public AggregatedMetric() {
+    min = new ValueWithHostname(Long.MAX_VALUE);
+    max = new ValueWithHostname(Long.MIN_VALUE);
+  }
+
+  /**
+   * Add another item to the aggregation.
+   *
+   * @param value long value to add
+   * @param hostnamePartitionId String hostname it came from
+   */
+  public void addItem(long value, String hostnamePartitionId) {
+    if (value < min.getValue()) {
+      min.set(value, hostnamePartitionId);
+    }
+    if (value > max.getValue()) {
+      max.set(value, hostnamePartitionId);
+    }
+    sum += value;
+    count++;
+  }
+
+  /**
+   * Whether this AggregatedMetric has any data.
+   *
+   * @return true if we have any data.
+   */
+  public boolean hasData() {
+    return count > 0;
+  }
+
+  /**
+   * Get minimum value together with host it came from.
+   *
+   * @return ValueWithHostname for minimum
+   */
+  public ValueWithHostname min() {
+    return min;
+  }
+
+  /**
+   * Get maximum value together with host it came from.
+   *
+   * @return ValueWithHostname for maximum
+   */
+  public ValueWithHostname max() {
+    return max;
+  }
+
+  /**
+   * Get total of all the values seen
+   *
+   * @return long total of values seen
+   */
+  public long sum() {
+    return sum;
+  }
+
+  /**
+   * Get average of all the values
+   *
+   * @return computed average of all the values
+   */
+  public double mean() {
+    return sum / (double) count;
+  }
+
+  /**
+   * Get number of values seen
+   *
+   * @return count of values
+   */
+  public long count() {
+    return count;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java b/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java
new file mode 100644
index 0000000..f146820
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.graph.BspServiceWorker;
+import org.apache.giraph.graph.GraphMapper;
+
+import com.google.common.collect.Maps;
+
+import java.io.PrintStream;
+import java.util.Map;
+
+/**
+ * Map of a bunch of aggregated metrics
+ */
+public class AggregatedMetrics {
+  /** counter for user compute time in milliseconds */
+  public static final String USER_COMPUTE_MS = "user-compute-ms";
+
+  /** Mapping from name to aggregated metric */
+  private Map<String, AggregatedMetric> metrics = Maps.newHashMap();
+
+  /**
+   * Add value from hostname for a metric.
+   *
+   * @param name String name of metric
+   * @param value long value to track
+   * @param hostnamePartitionId String host it came from
+   * @return this
+   */
+  public AggregatedMetrics add(String name, long value,
+                               String hostnamePartitionId) {
+    AggregatedMetric aggregatedMetric = metrics.get(name);
+    if (aggregatedMetric == null) {
+      aggregatedMetric = new AggregatedMetric();
+      metrics.put(name, aggregatedMetric);
+    }
+    aggregatedMetric.addItem(value, hostnamePartitionId);
+    return this;
+  }
+
+  /**
+   * Add metrics from worker.
+   *
+   * @param workerMetrics WorkerSuperstepMetrics from work
+   * @param hostname String hostname of worker
+   * @return this
+   */
+  public AggregatedMetrics add(WorkerSuperstepMetrics workerMetrics,
+                               String hostname) {
+    add(GraphMapper.TIMER_COMMUNICATION_TIME,
+        workerMetrics.getCommTimer(), hostname);
+    add(GraphMapper.TIMER_COMPUTE_ALL,
+        workerMetrics.getComputeAllTimer(), hostname);
+    add(GraphMapper.TIMER_TIME_TO_FIRST_MSG,
+        workerMetrics.getTimeToFirstMsg(), hostname);
+    add(BspServiceWorker.TIMER_WAIT_REQUESTS,
+        workerMetrics.getWaitRequestsTimer(), hostname);
+    add(USER_COMPUTE_MS,
+        workerMetrics.getUserComputeTime(), hostname);
+    return this;
+  }
+
+  /**
+   * Print the aggregated metrics to the stream provided.
+   *
+   * @param superstep long number of superstep.
+   * @param out PrintStream to write to.
+   * @return this
+   */
+  public AggregatedMetrics print(long superstep, PrintStream out) {
+    AggregatedMetric commTime = get(GraphMapper.TIMER_COMMUNICATION_TIME);
+    AggregatedMetric computeAll = get(GraphMapper.TIMER_COMPUTE_ALL);
+    AggregatedMetric timeToFirstMsg =
+        get(GraphMapper.TIMER_TIME_TO_FIRST_MSG);
+    AggregatedMetric waitRequestsMicros = get(
+        BspServiceWorker.TIMER_WAIT_REQUESTS);
+    AggregatedMetric userComputeTime = get(USER_COMPUTE_MS);
+
+    out.println();
+    out.println("--- METRICS: superstep " + superstep + " ---");
+    printAggregatedMetric(out, "compute all partitions", "ms", computeAll);
+    printAggregatedMetric(out, "user compute time", "ms", userComputeTime);
+    printAggregatedMetric(out, "network communication time", "ms", commTime);
+    printAggregatedMetric(out, "time to first message", "us", timeToFirstMsg);
+    printAggregatedMetric(out, "wait requests time", "us", waitRequestsMicros);
+
+    return this;
+  }
+
+  /**
+   * Print the aggregated metrics to the stream provided.
+   *
+   * @param superstep long number of superstep.
+   * @return this
+   */
+  public AggregatedMetrics print(long superstep) {
+    return print(superstep, System.out);
+  }
+
+  /**
+   * Print batch of lines for AggregatedMetric
+   *
+   * @param out PrintStream to write to
+   * @param header String header to print.
+   * @param timeUnit String time unit of metric
+   * @param aggregatedMetric AggregatedMetric to write
+   */
+  private void printAggregatedMetric(PrintStream out, String header,
+                                     String timeUnit,
+                                     AggregatedMetric aggregatedMetric) {
+    if (aggregatedMetric.hasData()) {
+      out.println(header);
+      out.println("  mean: " + aggregatedMetric.mean() + " " + timeUnit);
+      printValueFromHost(out, "  slowest: ", timeUnit, aggregatedMetric.max());
+      printValueFromHost(out, "  fastest: ", timeUnit, aggregatedMetric.min());
+    } else {
+      out.println(header + ": NO DATA");
+    }
+  }
+
+  /**
+   * Print a line for a value with the host it came from.
+   *
+   * @param out PrintStream to write to
+   * @param prefix String to write at beginning
+   * @param timeUnit String timeUnit of metric
+   * @param vh ValueWithHostname to write
+   */
+  private void printValueFromHost(PrintStream out, String prefix,
+                                  String timeUnit, ValueWithHostname vh) {
+    out.println(prefix + vh.getValue() + ' ' + timeUnit +
+        " from " + vh.getHostname());
+  }
+
+  /**
+   * Get AggregatedMetric with given name.
+   *
+   * @param name String metric to lookup.
+   * @return AggregatedMetric for given metric name.
+   */
+  public AggregatedMetric get(String name) {
+    return metrics.get(name);
+  }
+
+  /**
+   * Get map of all aggregated metrics.
+   *
+   * @return Map of all the aggregated metrics.
+   */
+  public Map<String, AggregatedMetric> getAll() {
+    return metrics;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java
new file mode 100644
index 0000000..f30f199
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.graph.BspService;
+
+import com.google.common.collect.Lists;
+
+import java.io.PrintStream;
+import java.util.List;
+
+/**
+ * Top level metrics class for using Yammer's metrics in Giraph.
+ */
+public class GiraphMetrics {
+  /** Singleton instance for everyone to use */
+  private static GiraphMetrics INSTANCE = new GiraphMetrics();
+
+  /** registry for per-superstep metrics */
+  private final SuperstepMetricsRegistry perSuperstep;
+
+  /** registry for per-job metrics */
+  private final GiraphMetricsRegistry perJob;
+
+  /** observer for per-superstep metrics re-initialization */
+  private final List<ResetSuperstepMetricsObserver> observers =
+      Lists.newArrayList();
+
+  /**
+   * Initialize no-op registry that creates no-op metrics.
+   */
+  private GiraphMetrics() {
+    perJob = new GiraphMetricsRegistry();
+    perSuperstep = new SuperstepMetricsRegistry();
+  }
+
+  /**
+   * Initialize GiraphMetrics with Hadoop Context
+   *
+   * @param conf GiraphConfiguration to use.
+   */
+  private GiraphMetrics(GiraphConfiguration conf) {
+    perJob = new GiraphMetricsRegistry(conf, "giraph", "job");
+    perSuperstep = new SuperstepMetricsRegistry(conf,
+        BspService.INPUT_SUPERSTEP);
+  }
+
+  /**
+   * Get singleton instance of GiraphMetrics.
+   *
+   * @return GiraphMetrics singleton instance
+   */
+  public static GiraphMetrics get() {
+    return INSTANCE;
+  }
+
+  /**
+   * Initialize singleton instance of GiraphMetrics.
+   *
+   * @param conf GiraphConfiguration to use.
+   */
+  public static void init(GiraphConfiguration conf) {
+    INSTANCE = new GiraphMetrics(conf);
+  }
+
+  /**
+   * Get per-job metrics.
+   *
+   * @return per-job GiraphMetricsRegistry
+   */
+  public GiraphMetricsRegistry perJob() {
+    return perJob;
+  }
+
+  /**
+   * Get per-superstep metrics.
+   *
+   * @return per-superstep GiraphMetricsRegistry
+   */
+  public SuperstepMetricsRegistry perSuperstep() {
+    return perSuperstep;
+  }
+
+  /**
+   * Anyone using per-superstep counters needs to re-initialize their Metrics
+   * object on each new superstep. Otherwise they will always be updating just
+   * one counter. This method allows people to easily register a callback for
+   * when they should do the re-initializing.
+   *
+   * @param observer SuperstepObserver to watch
+   */
+  public void addSuperstepResetObserver(
+      ResetSuperstepMetricsObserver observer) {
+    observers.add(observer);
+  }
+
+  /**
+   * Reset the per-superstep MetricsRegistry
+   *
+   * @param superstep long number of superstep
+   */
+  public void resetSuperstepMetrics(long superstep) {
+    perSuperstep.setSuperstep(superstep);
+    for (ResetSuperstepMetricsObserver observer : observers) {
+      observer.newSuperstep(perSuperstep);
+    }
+  }
+
+  /**
+   * Dump all metrics to output stream provided.
+   *
+   * @param out PrintStream to dump to.
+   */
+  public void dumpToStream(PrintStream out) {
+    perJob.printToStream(out);
+    perSuperstep.printToStream(out);
+  }
+
+  /**
+   * Dump all metrics to stdout.
+   */
+  public void dumpToStdout() {
+    dumpToStream(System.out);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java
new file mode 100644
index 0000000..184c97e
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.conf.GiraphConfiguration;
+
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.Meter;
+import com.yammer.metrics.core.Metric;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricPredicate;
+import com.yammer.metrics.core.MetricsRegistry;
+import com.yammer.metrics.core.Timer;
+import com.yammer.metrics.reporting.ConsoleReporter;
+import com.yammer.metrics.reporting.JmxReporter;
+
+import java.io.PrintStream;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A holder for MetricsRegistry together with a JmxReporter.
+ */
+public class GiraphMetricsRegistry {
+  /** String name of group to use for metrics created */
+  private String groupName;
+  /** String type to use for metrics created */
+  private String type;
+  /** Internal Yammer registry used */
+  private final MetricsRegistry registry;
+  /** JmxReporter that send metrics to JMX */
+  private final JmxReporter jmxReporter;
+
+  /**
+   * Create no-op empty registry that makes no-op metrics.
+   */
+  public GiraphMetricsRegistry() {
+    registry = new NoOpMetricsRegistry();
+    jmxReporter = null;
+  }
+
+  /**
+   * Create registry with Hadoop Configuration and group to use for metrics.
+   *
+   * @param conf Hadoop Configuration to use.
+   * @param groupName String group to use for metrics.
+   * @param type String type to use for metrics.
+   */
+  public GiraphMetricsRegistry(GiraphConfiguration conf, String groupName,
+                               String type) {
+    this.groupName = groupName;
+    this.type = type;
+    if (conf.metricsEnabled()) {
+      registry = new MetricsRegistry();
+      jmxReporter = new JmxReporter(registry);
+      jmxReporter.start();
+    } else {
+      registry = new NoOpMetricsRegistry();
+      jmxReporter = null;
+    }
+  }
+
+  /**
+   * Get map of all metrics.
+   *
+   * @return Map of all metrics held.
+   */
+  public Map<MetricName, Metric> getAll() {
+    return registry.allMetrics();
+  }
+
+  /**
+   * Get group name used for metrics.
+   *
+   * @return String group name.
+   */
+  public String getGroupName() {
+    return groupName;
+  }
+
+  /**
+   * Set group name used by this MetricsRegistry. Used for incrementing
+   * superstep number to create a new hierarchy of metrics per superstep.
+   *
+   * @param groupName String group name to use.
+   */
+  protected void setGroupName(String groupName) {
+    this.groupName = groupName;
+  }
+
+  /**
+   * Get type used for new metrics created
+   *
+   * @return String type to use for metrics
+   */
+  public String getType() {
+    return type;
+  }
+
+  /**
+   * Set type to use for new metrics
+   *
+   * @param type String type to use
+   */
+  public void setType(String type) {
+    this.type = type;
+  }
+
+  /**
+   * Dump all the metrics to the PrintStream provided.
+   *
+   * @param out PrintStream to write metrics to.
+   */
+  public void printToStream(PrintStream out) {
+    out.println("");
+    new ConsoleReporter(registry, out, MetricPredicate.ALL).run();
+  }
+
+  /**
+   * Get internal MetricsRegistry used.
+   *
+   * @return MetricsRegistry being used.
+   */
+  protected MetricsRegistry getInternalRegistry() {
+    return registry;
+  }
+
+  /**
+   * Creates a new {@link com.yammer.metrics.core.Counter} and registers it
+   * under the given group and name.
+   *
+   * @param name the name of the metric
+   * @return a new {@link com.yammer.metrics.core.Counter}
+   */
+  public Counter getCounter(String name) {
+    return registry.newCounter(makeMetricName(name));
+  }
+
+  /**
+   * Given a new {@link com.yammer.metrics.core.Gauge}, registers it under the
+   * given group and name.
+   *
+   * @param name   the name of the metric
+   * @param metric the metric
+   * @param <T>    the type of the value returned by the metric
+   * @return {@code metric}
+   */
+  public <T> Gauge<T> getGauge(String name, Gauge<T> metric) {
+    return registry.newGauge(makeMetricName(name), metric);
+  }
+
+  /**
+   * Creates a new non-biased {@link com.yammer.metrics.core.Histogram} and
+   * registers it under the given group and name.
+   *
+   * @param name  the name of the metric
+   * @return a new {@link com.yammer.metrics.core.Histogram}
+   */
+  public Histogram getHistogram(String name) {
+    return registry.newHistogram(makeMetricName(name), false);
+  }
+
+  /**
+   * Creates a new {@link Histogram} and registers it under the given group
+   * and name.
+   *
+   * @param name   the name of the metric
+   * @param biased whether or not the histogram should be biased
+   * @return a new {@link Histogram}
+   */
+  public Histogram getHistogram(String name, boolean biased) {
+    return registry.newHistogram(makeMetricName(name), biased);
+  }
+
+  /**
+   * Creates a new {@link com.yammer.metrics.core.Meter} and registers it under
+   * the given group and name.
+   *
+   * @param name      the name of the metric
+   * @param eventType the plural name of the type of events the meter is
+   *                  measuring (e.g., {@code "requests"})
+   * @param timeUnit  the rate unit of the new meter
+   * @return a new {@link com.yammer.metrics.core.Meter}
+   */
+  public Meter getMeter(String name, String eventType, TimeUnit timeUnit) {
+    return registry.newMeter(makeMetricName(name), eventType, timeUnit);
+  }
+
+  /**
+   * Creates a new {@link com.yammer.metrics.core.Timer} and registers it under
+   * the given group and name, measuring elapsed time in milliseconds and
+   * invocations per second.
+   *
+   * @param name  the name of the metric
+   * @return a new {@link com.yammer.metrics.core.Timer}
+   */
+  public Timer getTimer(String name) {
+    return getTimer(name, TimeUnit.MILLISECONDS, TimeUnit.SECONDS);
+  }
+
+  /**
+   * Creates a new {@link Timer} and registers it under the given
+   * group and name.
+   *
+   * @param name         the name of the metric
+   * @param durationUnit the duration scale unit of the new timer
+   * @param rateUnit     the rate scale unit of the new timer
+   * @return a new {@link Timer}
+   */
+  public Timer getTimer(String name, TimeUnit durationUnit, TimeUnit rateUnit) {
+    return registry.newTimer(makeMetricName(name), durationUnit, rateUnit);
+  }
+
+  /**
+   * Get a Gauge that is already present in the MetricsRegistry
+   *
+   * @param name String name of Gauge
+   * @param <T> value type Gauge returns
+   * @return Gauge<T> from MetricsRegistry
+   */
+  public <T> Gauge<T> getExistingGauge(String name) {
+    Metric metric = registry.allMetrics().get(makeMetricName(name));
+    return metric instanceof Gauge ? (Gauge<T>) metric : null;
+  }
+
+  /**
+   * Create a MetricName using the job ID, group, and name.
+   *
+   * @param name String name given to metric
+   * @return MetricName for use with MetricsRegistry
+   */
+  protected MetricName makeMetricName(String name) {
+    return new MetricName(groupName, type, name);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimer.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimer.java b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimer.java
new file mode 100644
index 0000000..fa96480
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimer.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A timer to record duration of an event in a given TimeUnit.
+ * GiraphTimer is actually just a single-value Yammer Gauge that has some
+ * methods to make timing things easier.
+ */
+public class GiraphTimer extends ValueGauge<Long> {
+  /** Mapping from TimeUnit to abbreviation used for printing */
+  private static final ImmutableMap<TimeUnit, String> TIME_UNIT_TO_ABBREV =
+      ImmutableMap.<TimeUnit, String>builder().
+          put(TimeUnit.DAYS, "days").
+          put(TimeUnit.HOURS, "hours").
+          put(TimeUnit.MICROSECONDS, "us").
+          put(TimeUnit.MILLISECONDS, "ms").
+          put(TimeUnit.MINUTES, "mins").
+          put(TimeUnit.NANOSECONDS, "ns").
+          put(TimeUnit.SECONDS, "secs").build();
+
+  /** dimension to measure things by */
+  private TimeUnit timeUnit;
+
+  /**
+   * Create new timer, add it to the registry.
+   *
+   * @param registry GiraphMetricsRegistry to add timer to
+   * @param name String name of timer
+   * @param timeUnit TimeUnit to measure in
+   */
+  public GiraphTimer(GiraphMetricsRegistry registry, String name,
+                     TimeUnit timeUnit) {
+    super(registry, name);
+    this.timeUnit = timeUnit;
+    set(0L);
+  }
+
+  /**
+   * Get TimeUnit used.
+   *
+   * @return TimeUnit being used.
+   */
+  public TimeUnit getTimeUnit() {
+    return timeUnit;
+  }
+
+  /**
+   * Begin timing an event.
+   *
+   * @return GiraphTimerContext. Use stop() to end timing the event.
+   */
+  public GiraphTimerContext time() {
+    return new GiraphTimerContext(this);
+  }
+
+  /**
+   * Set value from a given TimeUnit, converting to our TimeUnit.
+   *
+   * @param value long measurement taken.
+   * @param valueTimeUnit TimeUnit measurement is in.
+   * @return this
+   */
+  public GiraphTimer set(long value, TimeUnit valueTimeUnit) {
+    set(timeUnit.convert(value, valueTimeUnit));
+    return this;
+  }
+
+  /**
+   * Get abbreviated string of TimeUnit.
+   *
+   * @return TimeUnit abbreviation.
+   */
+  public String getTimeUnitAbbrev() {
+    return TIME_UNIT_TO_ABBREV.get(timeUnit);
+  }
+
+  /**
+   * Get string representation of value
+   *
+   * @return String value and abbreviated time unit
+   */
+  public String valueAndUnit() {
+    return "" + value() + " " + getTimeUnitAbbrev();
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimerContext.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimerContext.java b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimerContext.java
new file mode 100644
index 0000000..5c7942a
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimerContext.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.utils.SystemTime;
+import org.apache.giraph.utils.Time;
+import org.apache.giraph.utils.Times;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Context to use with GiraphTimer for timing an event.
+ */
+public class GiraphTimerContext {
+  /** Object to use for time measurement */
+  private static Time TIME = SystemTime.get();
+
+  /** GiraphTimer to callback to with result of timing */
+  private GiraphTimer giraphTimer;
+  /** Beginning time to compare against */
+  private long startTime;
+
+  /**
+   * Create with GiraphTimer to callback to.
+   *
+   * @param giraphTimer GiraphTimer to use.
+   */
+  public GiraphTimerContext(GiraphTimer giraphTimer) {
+    this.giraphTimer = giraphTimer;
+    this.startTime = TIME.getNanoseconds();
+  }
+
+  /**
+   * End timing. Set time elapsed in GiraphTimer.
+   */
+  public void stop() {
+    giraphTimer.set(Times.getNanosecondsSince(TIME, startTime),
+                    TimeUnit.NANOSECONDS);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/LongAndTimeUnit.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/LongAndTimeUnit.java b/giraph-core/src/main/java/org/apache/giraph/metrics/LongAndTimeUnit.java
new file mode 100644
index 0000000..8034eb4
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/LongAndTimeUnit.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Pair of long,TimeUnit
+ */
+public class LongAndTimeUnit {
+  /** Mapping from TimeUnit to abbreviation used for printing */
+  private static final ImmutableMap<TimeUnit, String> TIME_UNIT_TO_ABBREV =
+      ImmutableMap.<TimeUnit, String>builder().
+          put(TimeUnit.DAYS, "days").
+          put(TimeUnit.HOURS, "hours").
+          put(TimeUnit.MICROSECONDS, "us").
+          put(TimeUnit.MILLISECONDS, "ms").
+          put(TimeUnit.MINUTES, "mins").
+          put(TimeUnit.NANOSECONDS, "ns").
+          put(TimeUnit.SECONDS, "secs").build();
+
+  /** value held */
+  private long value;
+  /** TimeUnit part */
+  private TimeUnit timeUnit;
+
+  @Override
+  public String toString() {
+    if (timeUnit == null) {
+      return String.valueOf(value);
+    } else {
+      return value + " " + TIME_UNIT_TO_ABBREV.get(timeUnit);
+    }
+  }
+
+  /**
+   * Get the value
+   *
+   * @return value
+   */
+  public long getValue() {
+    return value;
+  }
+
+  /**
+   * Set the value
+   *
+   * @param value to set
+   */
+  public void setValue(long value) {
+    this.value = value;
+  }
+
+  /**
+   * Get the TimeUnit
+   *
+   * @return TimeUnit
+   */
+  public TimeUnit getTimeUnit() {
+    return timeUnit;
+  }
+
+  /**
+   * Set the TimeUnit
+   * @param timeUnit to set
+   */
+  public void setTimeUnit(TimeUnit timeUnit) {
+    this.timeUnit = timeUnit;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/MetricsRegistryDebugger.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/MetricsRegistryDebugger.java b/giraph-core/src/main/java/org/apache/giraph/metrics/MetricsRegistryDebugger.java
new file mode 100644
index 0000000..9a5e73d
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/MetricsRegistryDebugger.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.log4j.Logger;
+
+import com.yammer.metrics.core.Metric;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricsRegistryListener;
+
+/**
+ * A simple MetricsRegistry listener that prints every Metric added/removed.
+ */
+public class MetricsRegistryDebugger implements MetricsRegistryListener {
+  /** logger instance */
+  private static final Logger LOG =
+      Logger.getLogger(MetricsRegistryDebugger.class);
+  /** static singleton instance */
+  private static final MetricsRegistryDebugger INSTANCE =
+      new MetricsRegistryDebugger();
+
+  /**
+   * Get singleton static instance of this class.
+   *
+   * @return singleton static instance.
+   */
+  public static MetricsRegistryDebugger get() {
+    return INSTANCE;
+  }
+
+  @Override
+  public void onMetricAdded(MetricName name, Metric metric) {
+    LOG.info("Adding metric " + name + " - " +
+        metric.getClass().getSimpleName());
+  }
+
+  @Override
+  public void onMetricRemoved(MetricName name) {
+    LOG.info("Removing metric " + name);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/NoOpMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/NoOpMetricsRegistry.java b/giraph-core/src/main/java/org/apache/giraph/metrics/NoOpMetricsRegistry.java
new file mode 100644
index 0000000..28558d5
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/NoOpMetricsRegistry.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import com.google.common.collect.Maps;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.NoOpCounter;
+import com.yammer.metrics.core.NoOpGuage;
+import com.yammer.metrics.core.NoOpHistogram;
+import com.yammer.metrics.core.NoOpMeter;
+import com.yammer.metrics.core.NoOpTimer;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.Meter;
+import com.yammer.metrics.core.Metric;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricPredicate;
+import com.yammer.metrics.core.MetricsRegistry;
+import com.yammer.metrics.core.MetricsRegistryListener;
+import com.yammer.metrics.core.Timer;
+
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An empty MetricsRegistry, used to easily turn off all metrics without
+ * affecting client code.
+ */
+public class NoOpMetricsRegistry extends MetricsRegistry {
+  @Override
+  public <T> Gauge<T> newGauge(Class<?> klass, String name, Gauge<T> metric) {
+    return NoOpGuage.INSTANCE;
+  }
+
+  @Override
+  public <T> Gauge<T> newGauge(Class<?> klass, String name, String scope,
+                               Gauge<T> metric) {
+    return NoOpGuage.INSTANCE;
+  }
+
+  @Override
+  public <T> Gauge<T> newGauge(MetricName metricName, Gauge<T> metric) {
+    return NoOpGuage.INSTANCE;
+  }
+
+  @Override
+  public Counter newCounter(Class<?> klass, String name) {
+    return NoOpCounter.INSTANCE;
+  }
+
+  @Override
+  public Counter newCounter(Class<?> klass, String name, String scope) {
+    return NoOpCounter.INSTANCE;
+  }
+
+  @Override
+  public Counter newCounter(MetricName metricName) {
+    return NoOpCounter.INSTANCE;
+  }
+
+  @Override
+  public Histogram newHistogram(Class<?> klass, String name, boolean biased) {
+    return NoOpHistogram.INSTANCE;
+  }
+
+  @Override
+  public Histogram newHistogram(Class<?> klass, String name, String scope,
+                                boolean biased) {
+    return NoOpHistogram.INSTANCE;
+  }
+
+  @Override
+  public Histogram newHistogram(Class<?> klass, String name) {
+    return NoOpHistogram.INSTANCE;
+  }
+
+  @Override
+  public Histogram newHistogram(Class<?> klass, String name, String scope) {
+    return NoOpHistogram.INSTANCE;
+  }
+
+  @Override
+  public Histogram newHistogram(MetricName metricName, boolean biased) {
+    return NoOpHistogram.INSTANCE;
+  }
+
+  @Override
+  public Meter newMeter(Class<?> klass, String name, String eventType,
+                        TimeUnit unit) {
+    return NoOpMeter.INSTANCE;
+  }
+
+  @Override
+  public Meter newMeter(
+    Class<?> klass, String name, String scope, String eventType, TimeUnit unit
+  ) {
+    return NoOpMeter.INSTANCE;
+  }
+
+  @Override
+  public Meter newMeter(MetricName metricName, String eventType,
+                        TimeUnit unit) {
+    return NoOpMeter.INSTANCE;
+  }
+
+  @Override
+  public Timer newTimer(Class<?> klass, String name) {
+    return NoOpTimer.INSTANCE;
+  }
+
+  @Override
+  public Timer newTimer(Class<?> klass, String name, TimeUnit durationUnit,
+                        TimeUnit rateUnit) {
+    return NoOpTimer.INSTANCE;
+  }
+
+  @Override
+  public Timer newTimer(Class<?> klass, String name, String scope) {
+    return NoOpTimer.INSTANCE;
+  }
+
+  @Override
+  public Timer newTimer(Class<?> klass, String name, String scope,
+                        TimeUnit durationUnit, TimeUnit rateUnit
+  ) {
+    return NoOpTimer.INSTANCE;
+  }
+
+  @Override
+  public Timer newTimer(MetricName metricName, TimeUnit durationUnit,
+                        TimeUnit rateUnit) {
+    return NoOpTimer.INSTANCE;
+  }
+
+  @Override
+  public Map<MetricName, Metric> allMetrics() {
+    return Maps.newHashMap();
+  }
+
+  @Override
+  public SortedMap<String, SortedMap<MetricName, Metric>> groupedMetrics() {
+    return Maps.newTreeMap();
+  }
+
+  @Override
+  public SortedMap<String, SortedMap<MetricName, Metric>>
+  groupedMetrics(MetricPredicate predicate) {
+    return Maps.newTreeMap();
+  }
+
+  @Override
+  public void shutdown() { }
+
+  @Override
+  public ScheduledExecutorService newScheduledThreadPool(int poolSize,
+                                                         String name) {
+    return null;
+  }
+
+  @Override
+  public void removeMetric(Class<?> klass, String name) { }
+
+  @Override
+  public void removeMetric(Class<?> klass, String name, String scope) { }
+
+  @Override
+  public void removeMetric(MetricName name) { }
+
+  @Override
+  public void addListener(MetricsRegistryListener listener) { }
+
+  @Override
+  public void removeListener(MetricsRegistryListener listener) { }
+
+  @Override
+  protected ConcurrentMap<MetricName, Metric> newMetricsMap() {
+    return Maps.newConcurrentMap();
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/ResetSuperstepMetricsObserver.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/ResetSuperstepMetricsObserver.java b/giraph-core/src/main/java/org/apache/giraph/metrics/ResetSuperstepMetricsObserver.java
new file mode 100644
index 0000000..178618a
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/ResetSuperstepMetricsObserver.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+/**
+ * Observer for users of per-superstep counters. Triggered with calls whenever
+ * we are starting a new superstep so that user can re-initialize their metrics.
+ */
+public interface ResetSuperstepMetricsObserver {
+  /**
+   * Starting a new superstep. Re-initialize your metrics.
+   *
+   * @param superstepMetrics SuperstepMetricsRegistry being used.
+   */
+  void newSuperstep(SuperstepMetricsRegistry superstepMetrics);
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/SuperstepMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/SuperstepMetricsRegistry.java b/giraph-core/src/main/java/org/apache/giraph/metrics/SuperstepMetricsRegistry.java
new file mode 100644
index 0000000..dd9eef7
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/SuperstepMetricsRegistry.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.graph.BspService;
+
+import java.io.PrintStream;
+
+/**
+ * Wrapper around MetricsRegistry for per-superstep metrics.
+ */
+public class SuperstepMetricsRegistry extends GiraphMetricsRegistry {
+  /** Number of superstep to use for group of metrics created */
+  private long superstep = BspService.INPUT_SUPERSTEP;
+
+  /**
+   * Create no-op registry that creates no-op metrics.
+   */
+  public SuperstepMetricsRegistry() {
+    super();
+  }
+
+  /**
+   * Create with Hadoop Configuration and superstep number.
+   *
+   * @param conf Hadoop Configuration to use.
+   * @param superstep number of superstep to use as group for metrics.
+   */
+  public SuperstepMetricsRegistry(GiraphConfiguration conf, long superstep) {
+    super(conf, "giraph.superstep", String.valueOf(superstep));
+    this.superstep = superstep;
+  }
+
+  /**
+   * Set superstep number used. Internally sets the group for metrics created.
+   *
+   * @param superstep long number of superstep to use.
+   */
+  public void setSuperstep(long superstep) {
+    super.setType(String.valueOf(superstep));
+    this.superstep = superstep;
+  }
+
+  /**
+   * Print human readable summary of superstep metrics.
+   *
+   * @param out PrintStream to write to.
+   */
+  public void printSummary(PrintStream out) {
+    new WorkerSuperstepMetrics().readFromRegistry().print(superstep, out);
+  }
+
+  /**
+   * Print human readable summary of superstep metrics.
+   */
+  public void printSummary() {
+    printSummary(System.out);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/ValueGauge.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/ValueGauge.java b/giraph-core/src/main/java/org/apache/giraph/metrics/ValueGauge.java
new file mode 100644
index 0000000..aa7ab30
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/ValueGauge.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import com.yammer.metrics.core.Gauge;
+
+/**
+ * A Gauge that holds a value.
+ *
+ * @param <T> type of value being held.
+ */
+public class ValueGauge<T extends Number> extends Gauge<T> {
+  /** value held by this class */
+  private T val;
+
+  /**
+   * Constructor that registers Gauge in MetricsRegistry.
+   *
+   * @param registry GiraphMetricsRegistry to use.
+   * @param name String name of Gauge.
+   */
+  public ValueGauge(GiraphMetricsRegistry registry, String name) {
+    registry.getGauge(name, this);
+  }
+
+  @Override
+  public T value() {
+    return val;
+  }
+
+  /**
+   * Get double representation of value held.
+   *
+   * @return double value
+   */
+  public double getDouble() {
+    return val != null ? val.doubleValue() : 0.0d;
+  }
+
+  /**
+   * Get long representation of value held.
+   *
+   * @return long value
+   */
+  public long getLong() {
+    return val != null ? val.longValue() : 0L;
+  }
+
+  /**
+   * Set value held by this object.
+   *
+   * @param value value to set.
+   * @return this
+   */
+  public ValueGauge<T> set(T value) {
+    this.val = value;
+    return this;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/ValueWithHostname.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/ValueWithHostname.java b/giraph-core/src/main/java/org/apache/giraph/metrics/ValueWithHostname.java
new file mode 100644
index 0000000..208c4c7
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/ValueWithHostname.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+/**
+ * Pair of value with host it came from.
+ */
+class ValueWithHostname {
+  /** long value we're holding */
+  private long value;
+  /** host associated with value */
+  private String hostname;
+
+  /**
+   * Create with initial value
+   *
+   * @param value long initial value to use
+   */
+  public ValueWithHostname(long value) {
+    this.value = value;
+    this.hostname = null;
+  }
+
+  /**
+   * @return long value
+   */
+  public long getValue() {
+    return value;
+  }
+
+  /**
+   * @return String hostname
+   */
+  public String getHostname() {
+    return hostname;
+  }
+
+  /**
+   * Check if there is any hostname. Used as a flag that we have any data.
+   *
+   * @return true if hostname is set
+   */
+  public boolean hasHostname() {
+    return hostname != null;
+  }
+
+  /**
+   * Set value and partition together.
+   * @param value long value to use.
+   * @param hostname String host it came from.
+   */
+  public void set(long value, String hostname) {
+    this.value = value;
+    this.hostname = hostname;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/WorkerSuperstepMetrics.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/WorkerSuperstepMetrics.java b/giraph-core/src/main/java/org/apache/giraph/metrics/WorkerSuperstepMetrics.java
new file mode 100644
index 0000000..9db5107
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/WorkerSuperstepMetrics.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.graph.BspServiceWorker;
+import org.apache.giraph.graph.ComputeCallable;
+import org.apache.giraph.graph.GraphMapper;
+import org.apache.hadoop.io.Writable;
+
+import com.yammer.metrics.core.Gauge;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Per-superstep metrics for a Worker.
+ */
+public class WorkerSuperstepMetrics implements Writable {
+  /** Total network communication time */
+  private LongAndTimeUnit commTimer;
+  /** Time for all compute calls to complete */
+  private LongAndTimeUnit computeAllTimer;
+  /** Time till first message gets flushed */
+  private LongAndTimeUnit timeToFirstMsg;
+  /** Total superstep time */
+  private LongAndTimeUnit superstepTimer;
+  /** Time spent waiting for other workers to finish */
+  private LongAndTimeUnit waitRequestsTimer;
+  /** Time spent in Vertex#compute */
+  private LongAndTimeUnit userComputeTime;
+
+  /**
+   * Constructor
+   */
+  public WorkerSuperstepMetrics() {
+    commTimer = new LongAndTimeUnit();
+    computeAllTimer = new LongAndTimeUnit();
+    timeToFirstMsg = new LongAndTimeUnit();
+    superstepTimer = new LongAndTimeUnit();
+    waitRequestsTimer = new LongAndTimeUnit();
+
+    // Note this one is not backed by a GiraphTimer, but rather a real Timer
+    userComputeTime = new LongAndTimeUnit();
+    userComputeTime.setTimeUnit(TimeUnit.MILLISECONDS);
+  }
+
+  /**
+   * Read metric values from global MetricsRegistry.
+   *
+   * @return this object, for chaining
+   */
+  public WorkerSuperstepMetrics readFromRegistry() {
+    SuperstepMetricsRegistry ssm = GiraphMetrics.get().perSuperstep();
+    readGiraphTimer(GraphMapper.TIMER_COMMUNICATION_TIME, commTimer);
+    readGiraphTimer(GraphMapper.TIMER_COMPUTE_ALL, computeAllTimer);
+    readGiraphTimer(GraphMapper.TIMER_TIME_TO_FIRST_MSG, timeToFirstMsg);
+    readGiraphTimer(GraphMapper.TIMER_SUPERSTEP_TIME, superstepTimer);
+    readGiraphTimer(BspServiceWorker.TIMER_WAIT_REQUESTS, waitRequestsTimer);
+    userComputeTime.setValue((long) ssm.getTimer(
+        ComputeCallable.TIMER_COMPUTE_ONE).sum());
+    return this;
+  }
+
+  /**
+   * Read data from GiraphTimer into a LongAndTimeUnit.
+   *
+   * @param name String name of Gauge to retrieve.
+   * @param data LongAndTimeUnit to read data into.
+   */
+  private void readGiraphTimer(String name, LongAndTimeUnit data) {
+    Gauge<Long> gauge = GiraphMetrics.get().perSuperstep().
+        getExistingGauge(name);
+    if (gauge instanceof GiraphTimer) {
+      GiraphTimer giraphTimer = (GiraphTimer) gauge;
+      data.setTimeUnit(giraphTimer.getTimeUnit());
+      data.setValue(giraphTimer.value());
+    } else if (gauge != null) {
+      throw new IllegalStateException(name + " is not a GiraphTimer");
+    }
+  }
+
+  /**
+   * Human readable dump of metrics stored here.
+   *
+   * @param superstep long number of superstep.
+   * @param out PrintStream to write to.
+   * @return this object, for chaining
+   */
+  public WorkerSuperstepMetrics print(long superstep, PrintStream out) {
+    out.println();
+    out.println("--- METRICS: superstep " + superstep + " ---");
+    out.println("  superstep time: " + superstepTimer);
+    out.println("  compute all partitions: " + computeAllTimer);
+    out.println("  user compute time: " + userComputeTime);
+    out.println("  network communication time: " + commTimer);
+    out.println("  time to first message: " + timeToFirstMsg);
+    out.println("  wait on requests time: " + waitRequestsTimer);
+    return this;
+  }
+
+  /**
+   * @return Communication timer
+   */
+  public long getCommTimer() {
+    return commTimer.getValue();
+  }
+
+  /**
+   * @return Total compute timer
+   */
+  public long getComputeAllTimer() {
+    return computeAllTimer.getValue();
+  }
+
+  /**
+   * @return timer between start time and first message flushed.
+   */
+  public long getTimeToFirstMsg() {
+    return timeToFirstMsg.getValue();
+  }
+
+  /**
+   * @return timer for superstep time
+   */
+  public long getSuperstepTimer() {
+    return superstepTimer.getValue();
+  }
+
+  /**
+   * @return timer waiting for other workers
+   */
+  public long getWaitRequestsTimer() {
+    return waitRequestsTimer.getValue();
+  }
+
+  /**
+   * @return milliseconds in user compute code
+   */
+  public long getUserComputeTime() {
+    return userComputeTime.getValue();
+  }
+
+  @Override
+  public void readFields(DataInput dataInput) throws IOException {
+    commTimer.setValue(dataInput.readLong());
+    computeAllTimer.setValue(dataInput.readLong());
+    timeToFirstMsg.setValue(dataInput.readLong());
+    superstepTimer.setValue(dataInput.readLong());
+    waitRequestsTimer.setValue(dataInput.readLong());
+    userComputeTime.setValue(dataInput.readLong());
+  }
+
+  @Override
+  public void write(DataOutput dataOutput) throws IOException {
+    dataOutput.writeLong(commTimer.getValue());
+    dataOutput.writeLong(computeAllTimer.getValue());
+    dataOutput.writeLong(timeToFirstMsg.getValue());
+    dataOutput.writeLong(superstepTimer.getValue());
+    dataOutput.writeLong(waitRequestsTimer.getValue());
+    dataOutput.writeLong(userComputeTime.getValue());
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/package-info.java b/giraph-core/src/main/java/org/apache/giraph/metrics/package-info.java
new file mode 100644
index 0000000..20f3785
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Package of all the Giraph metrics related data.
+ */
+package org.apache.giraph.metrics;

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/package-info.java b/giraph-core/src/main/java/org/apache/giraph/package-info.java
new file mode 100644
index 0000000..3c5f960
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Base giraph package.
+ */
+package org.apache.giraph;

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/utils/AnnotationUtils.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/AnnotationUtils.java b/giraph-core/src/main/java/org/apache/giraph/utils/AnnotationUtils.java
new file mode 100644
index 0000000..79bb1c6
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/AnnotationUtils.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.utils;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.jar.JarEntry;
+import java.util.jar.JarInputStream;
+
+/**
+ * Helper class to deal with annotations in runtime.
+ */
+public class AnnotationUtils {
+  /** Do not instantiate. */
+  private AnnotationUtils() {
+  }
+
+  /**
+   * Finds all classes within a package which are annotated with certain
+   * annotation.
+   *
+   * @param annotation  Annotation which we are looking for
+   * @param <T>         Annotation class
+   * @param packageName Package in which to search
+   * @return The list of annotated classes
+   */
+  public static <T extends Annotation> List<Class<?>> getAnnotatedClasses(
+      Class<T> annotation, String packageName) {
+    ArrayList<Class<?>> ret = new ArrayList<Class<?>>();
+    for (Iterator<Class<?>> it = getClassesIterator(packageName);
+         it.hasNext();) {
+      Class<?> clazz = it.next();
+      if (clazz.getAnnotation(annotation) != null) {
+        ret.add(clazz);
+      }
+    }
+    return ret;
+  }
+
+  /**
+   * @param packageName Package through which to iterate
+   * @return Iterator through the classes of this jar file (if executed from
+   *         jar) or through the classes of org package (if .class file is
+   *         executed)
+   */
+  public static Iterator<Class<?>> getClassesIterator(String packageName) {
+    if (isExecutedFromJar()) {
+      try {
+        return new JarClassesIterator(packageName);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    } else {
+      return new GeneralClassesIterator(packageName);
+    }
+  }
+
+  /**
+   * @return Whether or not the code is executed from jar file
+   */
+  private static boolean isExecutedFromJar() {
+    return AnnotationUtils.class.getResource("AnnotationUtils.class")
+        .getProtocol().equals("jar");
+  }
+
+  /**
+   * To be used when {@link #isExecutedFromJar()}  is true.
+   *
+   * @return If executed from jar file returns the path to the jar
+   *         (otherwise it will return the folder in which this .class file is
+   *         located)
+   */
+  private static String getCurrentJar() {
+    return AnnotationUtils.class.getProtectionDomain().getCodeSource()
+        .getLocation().getFile();
+  }
+
+  /**
+   * To be used when {@link #isExecutedFromJar()}  is true.
+   * <p/>
+   * Iterator through classes of this jar file.
+   */
+  private static class JarClassesIterator implements Iterator<Class<?>> {
+    /** Used to go through classes in current jar */
+    private final JarInputStream jarIn;
+    /** Are we positioned on the next class entry */
+    private boolean entryLoaded;
+    /** Next class entry */
+    private JarEntry currentEntry;
+    /** Folder in which to look */
+    private final String path;
+
+    /**
+     * @param packageName Package through which to iterate
+     * @throws IOException
+     */
+    public JarClassesIterator(String packageName) throws IOException {
+      jarIn = new JarInputStream(new FileInputStream(new File(
+          getCurrentJar())));
+      entryLoaded = false;
+      currentEntry = null;
+      path = packageName.replace(".", File.separator);
+    }
+
+    @Override
+    public boolean hasNext() {
+      loadNextEntry();
+      return currentEntry != null;
+    }
+
+    @Override
+    public Class<?> next() {
+      loadNextEntry();
+      if (currentEntry == null) {
+        throw new NoSuchElementException();
+      }
+      entryLoaded = false;
+
+      String className = currentEntry.getName().replace(".class",
+          "").replace(File.separator, ".");
+      return loadClass(className);
+    }
+
+    /**
+     * Sets position to next class entry
+     */
+    private void loadNextEntry() {
+      while (!entryLoaded) {
+        try {
+          currentEntry = jarIn.getNextJarEntry();
+          if (currentEntry == null || (currentEntry.getName().endsWith(
+              ".class") && (currentEntry.getName().startsWith(path)))) {
+            entryLoaded = true;
+          } else {
+            currentEntry = null;
+          }
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+      if (currentEntry == null) {
+        try {
+          jarIn.close();
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    }
+
+    @Override
+    public void remove() {
+      throw new UnsupportedOperationException(
+          "Can't remove from classes iterator");
+    }
+  }
+
+  /**
+   * To be used when {@link #isExecutedFromJar()}  is false.
+   * <p/>
+   * Iterator through classes of some package.
+   */
+  private static class GeneralClassesIterator implements Iterator<Class<?>> {
+    /** From which position in path the package name starts */
+    private final int stringPosition;
+    /** Recursive directory iterator */
+    private final Iterator<File> iterator;
+
+    /**
+     * @param packageName Package through which to iterate
+     */
+    public GeneralClassesIterator(String packageName) {
+      String mainPath = AnnotationUtils.class.getProtectionDomain()
+          .getCodeSource().getLocation().getFile();
+      String subPath = packageName.replace(".", File.separator);
+      File directory = new File(mainPath + subPath);
+      stringPosition = directory.getPath().length() - packageName.length();
+      List<File> files = new ArrayList<File>();
+      addAllClassFiles(directory, files);
+      iterator = files.iterator();
+    }
+
+    /**
+     * Recursively add all .class files from the directory to the list.
+     *
+     * @param directory Directory from which we are adding files
+     * @param files List we add files to
+     */
+    private void addAllClassFiles(File directory, List<File> files) {
+      for (File file : directory.listFiles()) {
+        if (file.isDirectory()) {
+          addAllClassFiles(file, files);
+        } else if (file.getName().endsWith(".class")) {
+          files.add(file);
+        }
+      }
+    }
+
+    @Override
+    public boolean hasNext() {
+      return iterator.hasNext();
+    }
+
+    @Override
+    public Class<?> next() {
+      String className = iterator.next().getPath().substring(stringPosition)
+          .replace(".class", "").replace(File.separator, ".");
+      return loadClass(className);
+    }
+
+    @Override
+    public void remove() {
+      throw new UnsupportedOperationException(
+          "Can't remove from classes iterator");
+    }
+  }
+
+  /**
+   * Loads the class with the specified name
+   *
+   * @param className Name of the class we are loading
+   * @return Class with the specified name
+   */
+  private static Class<?> loadClass(String className) {
+    try {
+      return Class.forName(className);
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException("Error loading class " + className, e);
+    } catch (NoClassDefFoundError e) {
+      throw new RuntimeException("Error loading class " + className, e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/utils/ArrayListWritable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/ArrayListWritable.java b/giraph-core/src/main/java/org/apache/giraph/utils/ArrayListWritable.java
new file mode 100644
index 0000000..9d8f4f1
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/ArrayListWritable.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.utils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * A Writable for ListArray containing instances of a class.
+ *
+ * @param <M> Message data
+ */
+public abstract class ArrayListWritable<M extends Writable> extends ArrayList<M>
+  implements Writable, Configurable {
+  /** Defining a layout version for a serializable class. */
+  private static final long serialVersionUID = 1L;
+  /** Used for instantiation */
+  private Class<M> refClass = null;
+
+  /** Configuration */
+  private Configuration conf;
+
+  /**
+   * Using the default constructor requires that the user implement
+   * setClass(), guaranteed to be invoked prior to instantiation in
+   * readFields()
+   */
+  public ArrayListWritable() {
+  }
+
+  /**
+   * Constructor with another {@link ArrayListWritable}.
+   *
+   * @param arrayListWritable Array list to be used internally.
+   */
+  public ArrayListWritable(ArrayListWritable<M> arrayListWritable) {
+    super(arrayListWritable);
+  }
+
+  /**
+   * This constructor allows setting the refClass during construction.
+   *
+   * @param refClass internal type class
+   */
+  public ArrayListWritable(Class<M> refClass) {
+    super();
+    this.refClass = refClass;
+  }
+
+  /**
+   * This is a one-time operation to set the class type
+   *
+   * @param refClass internal type class
+   */
+  public void setClass(Class<M> refClass) {
+    if (this.refClass != null) {
+      throw new RuntimeException(
+          "setClass: refClass is already set to " +
+              this.refClass.getName());
+    }
+    this.refClass = refClass;
+  }
+
+  /**
+   * Subclasses must set the class type appropriately and can use
+   * setClass(Class<M> refClass) to do it.
+   */
+  public abstract void setClass();
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    if (this.refClass == null) {
+      setClass();
+    }
+    int numValues = in.readInt();            // read number of values
+    ensureCapacity(numValues);
+    for (int i = 0; i < numValues; i++) {
+      M value = ReflectionUtils.newInstance(refClass, conf);
+      value.readFields(in);                // read a value
+      add(value);                          // store it in values
+    }
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    int numValues = size();
+    out.writeInt(numValues);                 // write number of values
+    for (int i = 0; i < numValues; i++) {
+      get(i).write(out);
+    }
+  }
+
+  @Override
+  public final Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  public final void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterable.java b/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterable.java
new file mode 100644
index 0000000..cf2c187
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterable.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.utils;
+
+import java.util.Iterator;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This iterable is designed to deserialize a byte array on the fly to
+ * provide new copies of writable objects when desired.  It does not reuse
+ * objects, and instead creates a new one for every next().
+ *
+ * @param <T> Type that extends Writable that will be iterated
+ */
+public abstract class ByteArrayIterable<T extends Writable> implements
+    Iterable<T> {
+  /** Configuration */
+  protected final ImmutableClassesGiraphConfiguration configuration;
+  /** Data input */
+  protected final byte[] buf;
+  /** Offset to start in buf */
+  protected final int off;
+  /** Length of buf */
+  protected final int length;
+
+  /**
+   * Constructor
+   *
+   * @param configuration Configuration
+   * @param buf Buffer
+   * @param off Offset to start in the buffer
+   * @param length Length of the buffer
+   */
+  public ByteArrayIterable(ImmutableClassesGiraphConfiguration configuration,
+                           byte[] buf, int off, int length) {
+    this.configuration = configuration;
+    this.buf = buf;
+    this.off = off;
+    this.length = length;
+  }
+
+  /**
+   * Must be able to create the writable object
+   *
+   * @return New writable
+   */
+  protected abstract T createWritable();
+
+  /**
+   * Iterator over the internal byte array
+   */
+  private class ByteArrayIterableIterator extends ByteArrayIterator<T> {
+    /**
+     * Constructor.
+     *
+     * @param buf Buffer to read from
+     * @param off Offset to read from in the buffer
+     * @param length Maximum length of the buffer
+     */
+    private ByteArrayIterableIterator(byte[] buf, int off, int length) {
+      super(configuration, buf, off, length);
+    }
+
+    @Override
+    protected T createWritable() {
+      return ByteArrayIterable.this.createWritable();
+    }
+  }
+
+  @Override
+  public Iterator<T> iterator() {
+    return new ByteArrayIterableIterator(buf, off, length);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterator.java b/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterator.java
new file mode 100644
index 0000000..76ed789
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterator.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.utils;
+
+import java.io.IOException;
+import java.util.Iterator;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This iterator is designed to deserialize a byte array on the fly to
+ * provide new copies of writable objects when desired.  It does not reuse
+ * objects, and instead creates a new one for every next().
+ *
+ * @param <T> Type that extends Writable that will be iterated
+ */
+public abstract class ByteArrayIterator<T extends Writable> implements
+    Iterator<T> {
+  /** Data input */
+  protected final ExtendedDataInput extendedDataInput;
+
+  /**
+   * Constructor
+   *
+   * @param configuration Configuration
+   * @param buf Buffer
+   * @param off Offset to start in the buffer
+   * @param length Length of the buffer
+   */
+  public ByteArrayIterator(
+      ImmutableClassesGiraphConfiguration configuration,
+      byte[] buf, int off, int length) {
+    extendedDataInput =
+        configuration.createExtendedDataInput(buf, off, length);
+  }
+
+  @Override
+  public boolean hasNext() {
+    return extendedDataInput.available() > 0;
+  }
+
+  @Override
+  public T next() {
+    T writable = createWritable();
+    try {
+      writable.readFields(extendedDataInput);
+    } catch (IOException e) {
+      throw new IllegalStateException("next: readFields got IOException", e);
+    }
+    return writable;
+  }
+
+  @Override
+  public void remove() {
+    throw new IllegalAccessError("remove: Not supported");
+  }
+
+  /**
+   * Must be able to create the writable object
+   *
+   * @return New writable
+   */
+  protected abstract T createWritable();
+}