You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2012/12/20 05:25:32 UTC
[22/51] [partial] GIRAPH-457: update module names (nitay)
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetric.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetric.java b/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetric.java
new file mode 100644
index 0000000..9d782c4
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetric.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+/**
+ * An aggregator over metrics from multiple hosts. Computes min, max, and mean.
+ */
+public class AggregatedMetric {
+ /** Minimum value seen with the host that it came from */
+ private ValueWithHostname min;
+ /** Maximum value seen with the host that it came from */
+ private ValueWithHostname max;
+ /** Total of all the values seen */
+ private long sum;
+ /** Number of values seen */
+ private long count;
+
+ /**
+ * Create new aggregated metric.
+ */
+ public AggregatedMetric() {
+ min = new ValueWithHostname(Long.MAX_VALUE);
+ max = new ValueWithHostname(Long.MIN_VALUE);
+ }
+
+ /**
+ * Add another item to the aggregation.
+ *
+ * @param value long value to add
+ * @param hostnamePartitionId String hostname it came from
+ */
+ public void addItem(long value, String hostnamePartitionId) {
+ if (value < min.getValue()) {
+ min.set(value, hostnamePartitionId);
+ }
+ if (value > max.getValue()) {
+ max.set(value, hostnamePartitionId);
+ }
+ sum += value;
+ count++;
+ }
+
+ /**
+ * Whether this AggregatedMetric has any data.
+ *
+ * @return true if we have any data.
+ */
+ public boolean hasData() {
+ return count > 0;
+ }
+
+ /**
+ * Get minimum value together with host it came from.
+ *
+ * @return ValueWithHostname for minimum
+ */
+ public ValueWithHostname min() {
+ return min;
+ }
+
+ /**
+ * Get maximum value together with host it came from.
+ *
+ * @return ValueWithHostname for maximum
+ */
+ public ValueWithHostname max() {
+ return max;
+ }
+
+ /**
+ * Get total of all the values seen
+ *
+ * @return long total of values seen
+ */
+ public long sum() {
+ return sum;
+ }
+
+ /**
+ * Get average of all the values
+ *
+ * @return computed average of all the values
+ */
+ public double mean() {
+ return sum / (double) count;
+ }
+
+ /**
+ * Get number of values seen
+ *
+ * @return count of values
+ */
+ public long count() {
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java b/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java
new file mode 100644
index 0000000..f146820
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.graph.BspServiceWorker;
+import org.apache.giraph.graph.GraphMapper;
+
+import com.google.common.collect.Maps;
+
+import java.io.PrintStream;
+import java.util.Map;
+
+/**
+ * Map of a bunch of aggregated metrics
+ */
+public class AggregatedMetrics {
+ /** counter for user compute time in milliseconds */
+ public static final String USER_COMPUTE_MS = "user-compute-ms";
+
+ /** Mapping from name to aggregated metric */
+ private Map<String, AggregatedMetric> metrics = Maps.newHashMap();
+
+ /**
+ * Add value from hostname for a metric.
+ *
+ * @param name String name of metric
+ * @param value long value to track
+ * @param hostnamePartitionId String host it came from
+ * @return this
+ */
+ public AggregatedMetrics add(String name, long value,
+ String hostnamePartitionId) {
+ AggregatedMetric aggregatedMetric = metrics.get(name);
+ if (aggregatedMetric == null) {
+ aggregatedMetric = new AggregatedMetric();
+ metrics.put(name, aggregatedMetric);
+ }
+ aggregatedMetric.addItem(value, hostnamePartitionId);
+ return this;
+ }
+
+ /**
+ * Add metrics from worker.
+ *
+ * @param workerMetrics WorkerSuperstepMetrics from work
+ * @param hostname String hostname of worker
+ * @return this
+ */
+ public AggregatedMetrics add(WorkerSuperstepMetrics workerMetrics,
+ String hostname) {
+ add(GraphMapper.TIMER_COMMUNICATION_TIME,
+ workerMetrics.getCommTimer(), hostname);
+ add(GraphMapper.TIMER_COMPUTE_ALL,
+ workerMetrics.getComputeAllTimer(), hostname);
+ add(GraphMapper.TIMER_TIME_TO_FIRST_MSG,
+ workerMetrics.getTimeToFirstMsg(), hostname);
+ add(BspServiceWorker.TIMER_WAIT_REQUESTS,
+ workerMetrics.getWaitRequestsTimer(), hostname);
+ add(USER_COMPUTE_MS,
+ workerMetrics.getUserComputeTime(), hostname);
+ return this;
+ }
+
+ /**
+ * Print the aggregated metrics to the stream provided.
+ *
+ * @param superstep long number of superstep.
+ * @param out PrintStream to write to.
+ * @return this
+ */
+ public AggregatedMetrics print(long superstep, PrintStream out) {
+ AggregatedMetric commTime = get(GraphMapper.TIMER_COMMUNICATION_TIME);
+ AggregatedMetric computeAll = get(GraphMapper.TIMER_COMPUTE_ALL);
+ AggregatedMetric timeToFirstMsg =
+ get(GraphMapper.TIMER_TIME_TO_FIRST_MSG);
+ AggregatedMetric waitRequestsMicros = get(
+ BspServiceWorker.TIMER_WAIT_REQUESTS);
+ AggregatedMetric userComputeTime = get(USER_COMPUTE_MS);
+
+ out.println();
+ out.println("--- METRICS: superstep " + superstep + " ---");
+ printAggregatedMetric(out, "compute all partitions", "ms", computeAll);
+ printAggregatedMetric(out, "user compute time", "ms", userComputeTime);
+ printAggregatedMetric(out, "network communication time", "ms", commTime);
+ printAggregatedMetric(out, "time to first message", "us", timeToFirstMsg);
+ printAggregatedMetric(out, "wait requests time", "us", waitRequestsMicros);
+
+ return this;
+ }
+
+ /**
+ * Print the aggregated metrics to the stream provided.
+ *
+ * @param superstep long number of superstep.
+ * @return this
+ */
+ public AggregatedMetrics print(long superstep) {
+ return print(superstep, System.out);
+ }
+
+ /**
+ * Print batch of lines for AggregatedMetric
+ *
+ * @param out PrintStream to write to
+ * @param header String header to print.
+ * @param timeUnit String time unit of metric
+ * @param aggregatedMetric AggregatedMetric to write
+ */
+ private void printAggregatedMetric(PrintStream out, String header,
+ String timeUnit,
+ AggregatedMetric aggregatedMetric) {
+ if (aggregatedMetric.hasData()) {
+ out.println(header);
+ out.println(" mean: " + aggregatedMetric.mean() + " " + timeUnit);
+ printValueFromHost(out, " slowest: ", timeUnit, aggregatedMetric.max());
+ printValueFromHost(out, " fastest: ", timeUnit, aggregatedMetric.min());
+ } else {
+ out.println(header + ": NO DATA");
+ }
+ }
+
+ /**
+ * Print a line for a value with the host it came from.
+ *
+ * @param out PrintStream to write to
+ * @param prefix String to write at beginning
+ * @param timeUnit String timeUnit of metric
+ * @param vh ValueWithHostname to write
+ */
+ private void printValueFromHost(PrintStream out, String prefix,
+ String timeUnit, ValueWithHostname vh) {
+ out.println(prefix + vh.getValue() + ' ' + timeUnit +
+ " from " + vh.getHostname());
+ }
+
+ /**
+ * Get AggregatedMetric with given name.
+ *
+ * @param name String metric to lookup.
+ * @return AggregatedMetric for given metric name.
+ */
+ public AggregatedMetric get(String name) {
+ return metrics.get(name);
+ }
+
+ /**
+ * Get map of all aggregated metrics.
+ *
+ * @return Map of all the aggregated metrics.
+ */
+ public Map<String, AggregatedMetric> getAll() {
+ return metrics;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java
new file mode 100644
index 0000000..f30f199
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.graph.BspService;
+
+import com.google.common.collect.Lists;
+
+import java.io.PrintStream;
+import java.util.List;
+
+/**
+ * Top level metrics class for using Yammer's metrics in Giraph.
+ */
+public class GiraphMetrics {
+ /** Singleton instance for everyone to use */
+ private static GiraphMetrics INSTANCE = new GiraphMetrics();
+
+ /** registry for per-superstep metrics */
+ private final SuperstepMetricsRegistry perSuperstep;
+
+ /** registry for per-job metrics */
+ private final GiraphMetricsRegistry perJob;
+
+ /** observer for per-superstep metrics re-initialization */
+ private final List<ResetSuperstepMetricsObserver> observers =
+ Lists.newArrayList();
+
+ /**
+ * Initialize no-op registry that creates no-op metrics.
+ */
+ private GiraphMetrics() {
+ perJob = new GiraphMetricsRegistry();
+ perSuperstep = new SuperstepMetricsRegistry();
+ }
+
+ /**
+ * Initialize GiraphMetrics with Hadoop Context
+ *
+ * @param conf GiraphConfiguration to use.
+ */
+ private GiraphMetrics(GiraphConfiguration conf) {
+ perJob = new GiraphMetricsRegistry(conf, "giraph", "job");
+ perSuperstep = new SuperstepMetricsRegistry(conf,
+ BspService.INPUT_SUPERSTEP);
+ }
+
+ /**
+ * Get singleton instance of GiraphMetrics.
+ *
+ * @return GiraphMetrics singleton instance
+ */
+ public static GiraphMetrics get() {
+ return INSTANCE;
+ }
+
+ /**
+ * Initialize singleton instance of GiraphMetrics.
+ *
+ * @param conf GiraphConfiguration to use.
+ */
+ public static void init(GiraphConfiguration conf) {
+ INSTANCE = new GiraphMetrics(conf);
+ }
+
+ /**
+ * Get per-job metrics.
+ *
+ * @return per-job GiraphMetricsRegistry
+ */
+ public GiraphMetricsRegistry perJob() {
+ return perJob;
+ }
+
+ /**
+ * Get per-superstep metrics.
+ *
+ * @return per-superstep GiraphMetricsRegistry
+ */
+ public SuperstepMetricsRegistry perSuperstep() {
+ return perSuperstep;
+ }
+
+ /**
+ * Anyone using per-superstep counters needs to re-initialize their Metrics
+ * object on each new superstep. Otherwise they will always be updating just
+ * one counter. This method allows people to easily register a callback for
+ * when they should do the re-initializing.
+ *
+ * @param observer SuperstepObserver to watch
+ */
+ public void addSuperstepResetObserver(
+ ResetSuperstepMetricsObserver observer) {
+ observers.add(observer);
+ }
+
+ /**
+ * Reset the per-superstep MetricsRegistry
+ *
+ * @param superstep long number of superstep
+ */
+ public void resetSuperstepMetrics(long superstep) {
+ perSuperstep.setSuperstep(superstep);
+ for (ResetSuperstepMetricsObserver observer : observers) {
+ observer.newSuperstep(perSuperstep);
+ }
+ }
+
+ /**
+ * Dump all metrics to output stream provided.
+ *
+ * @param out PrintStream to dump to.
+ */
+ public void dumpToStream(PrintStream out) {
+ perJob.printToStream(out);
+ perSuperstep.printToStream(out);
+ }
+
+ /**
+ * Dump all metrics to stdout.
+ */
+ public void dumpToStdout() {
+ dumpToStream(System.out);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java
new file mode 100644
index 0000000..184c97e
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.conf.GiraphConfiguration;
+
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.Meter;
+import com.yammer.metrics.core.Metric;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricPredicate;
+import com.yammer.metrics.core.MetricsRegistry;
+import com.yammer.metrics.core.Timer;
+import com.yammer.metrics.reporting.ConsoleReporter;
+import com.yammer.metrics.reporting.JmxReporter;
+
+import java.io.PrintStream;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A holder for MetricsRegistry together with a JmxReporter.
+ */
+public class GiraphMetricsRegistry {
+ /** String name of group to use for metrics created */
+ private String groupName;
+ /** String type to use for metrics created */
+ private String type;
+ /** Internal Yammer registry used */
+ private final MetricsRegistry registry;
+ /** JmxReporter that send metrics to JMX */
+ private final JmxReporter jmxReporter;
+
+ /**
+ * Create no-op empty registry that makes no-op metrics.
+ */
+ public GiraphMetricsRegistry() {
+ registry = new NoOpMetricsRegistry();
+ jmxReporter = null;
+ }
+
+ /**
+ * Create registry with Hadoop Configuration and group to use for metrics.
+ *
+ * @param conf Hadoop Configuration to use.
+ * @param groupName String group to use for metrics.
+ * @param type String type to use for metrics.
+ */
+ public GiraphMetricsRegistry(GiraphConfiguration conf, String groupName,
+ String type) {
+ this.groupName = groupName;
+ this.type = type;
+ if (conf.metricsEnabled()) {
+ registry = new MetricsRegistry();
+ jmxReporter = new JmxReporter(registry);
+ jmxReporter.start();
+ } else {
+ registry = new NoOpMetricsRegistry();
+ jmxReporter = null;
+ }
+ }
+
+ /**
+ * Get map of all metrics.
+ *
+ * @return Map of all metrics held.
+ */
+ public Map<MetricName, Metric> getAll() {
+ return registry.allMetrics();
+ }
+
+ /**
+ * Get group name used for metrics.
+ *
+ * @return String group name.
+ */
+ public String getGroupName() {
+ return groupName;
+ }
+
+ /**
+ * Set group name used by this MetricsRegistry. Used for incrementing
+ * superstep number to create a new hierarchy of metrics per superstep.
+ *
+ * @param groupName String group name to use.
+ */
+ protected void setGroupName(String groupName) {
+ this.groupName = groupName;
+ }
+
+ /**
+ * Get type used for new metrics created
+ *
+ * @return String type to use for metrics
+ */
+ public String getType() {
+ return type;
+ }
+
+ /**
+ * Set type to use for new metrics
+ *
+ * @param type String type to use
+ */
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ /**
+ * Dump all the metrics to the PrintStream provided.
+ *
+ * @param out PrintStream to write metrics to.
+ */
+ public void printToStream(PrintStream out) {
+ out.println("");
+ new ConsoleReporter(registry, out, MetricPredicate.ALL).run();
+ }
+
+ /**
+ * Get internal MetricsRegistry used.
+ *
+ * @return MetricsRegistry being used.
+ */
+ protected MetricsRegistry getInternalRegistry() {
+ return registry;
+ }
+
+ /**
+ * Creates a new {@link com.yammer.metrics.core.Counter} and registers it
+ * under the given group and name.
+ *
+ * @param name the name of the metric
+ * @return a new {@link com.yammer.metrics.core.Counter}
+ */
+ public Counter getCounter(String name) {
+ return registry.newCounter(makeMetricName(name));
+ }
+
+ /**
+ * Given a new {@link com.yammer.metrics.core.Gauge}, registers it under the
+ * given group and name.
+ *
+ * @param name the name of the metric
+ * @param metric the metric
+ * @param <T> the type of the value returned by the metric
+ * @return {@code metric}
+ */
+ public <T> Gauge<T> getGauge(String name, Gauge<T> metric) {
+ return registry.newGauge(makeMetricName(name), metric);
+ }
+
+ /**
+ * Creates a new non-biased {@link com.yammer.metrics.core.Histogram} and
+ * registers it under the given group and name.
+ *
+ * @param name the name of the metric
+ * @return a new {@link com.yammer.metrics.core.Histogram}
+ */
+ public Histogram getHistogram(String name) {
+ return registry.newHistogram(makeMetricName(name), false);
+ }
+
+ /**
+ * Creates a new {@link Histogram} and registers it under the given group
+ * and name.
+ *
+ * @param name the name of the metric
+ * @param biased whether or not the histogram should be biased
+ * @return a new {@link Histogram}
+ */
+ public Histogram getHistogram(String name, boolean biased) {
+ return registry.newHistogram(makeMetricName(name), biased);
+ }
+
+ /**
+ * Creates a new {@link com.yammer.metrics.core.Meter} and registers it under
+ * the given group and name.
+ *
+ * @param name the name of the metric
+ * @param eventType the plural name of the type of events the meter is
+ * measuring (e.g., {@code "requests"})
+ * @param timeUnit the rate unit of the new meter
+ * @return a new {@link com.yammer.metrics.core.Meter}
+ */
+ public Meter getMeter(String name, String eventType, TimeUnit timeUnit) {
+ return registry.newMeter(makeMetricName(name), eventType, timeUnit);
+ }
+
+ /**
+ * Creates a new {@link com.yammer.metrics.core.Timer} and registers it under
+ * the given group and name, measuring elapsed time in milliseconds and
+ * invocations per second.
+ *
+ * @param name the name of the metric
+ * @return a new {@link com.yammer.metrics.core.Timer}
+ */
+ public Timer getTimer(String name) {
+ return getTimer(name, TimeUnit.MILLISECONDS, TimeUnit.SECONDS);
+ }
+
+ /**
+ * Creates a new {@link Timer} and registers it under the given
+ * group and name.
+ *
+ * @param name the name of the metric
+ * @param durationUnit the duration scale unit of the new timer
+ * @param rateUnit the rate scale unit of the new timer
+ * @return a new {@link Timer}
+ */
+ public Timer getTimer(String name, TimeUnit durationUnit, TimeUnit rateUnit) {
+ return registry.newTimer(makeMetricName(name), durationUnit, rateUnit);
+ }
+
+ /**
+ * Get a Gauge that is already present in the MetricsRegistry
+ *
+ * @param name String name of Gauge
+ * @param <T> value type Gauge returns
+ * @return Gauge<T> from MetricsRegistry
+ */
+ public <T> Gauge<T> getExistingGauge(String name) {
+ Metric metric = registry.allMetrics().get(makeMetricName(name));
+ return metric instanceof Gauge ? (Gauge<T>) metric : null;
+ }
+
+ /**
+ * Create a MetricName using the job ID, group, and name.
+ *
+ * @param name String name given to metric
+ * @return MetricName for use with MetricsRegistry
+ */
+ protected MetricName makeMetricName(String name) {
+ return new MetricName(groupName, type, name);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimer.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimer.java b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimer.java
new file mode 100644
index 0000000..fa96480
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimer.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A timer to record duration of an event in a given TimeUnit.
+ * GiraphTimer is actually just a single-value Yammer Gauge that has some
+ * methods to make timing things easier.
+ */
+public class GiraphTimer extends ValueGauge<Long> {
+ /** Mapping from TimeUnit to abbreviation used for printing */
+ private static final ImmutableMap<TimeUnit, String> TIME_UNIT_TO_ABBREV =
+ ImmutableMap.<TimeUnit, String>builder().
+ put(TimeUnit.DAYS, "days").
+ put(TimeUnit.HOURS, "hours").
+ put(TimeUnit.MICROSECONDS, "us").
+ put(TimeUnit.MILLISECONDS, "ms").
+ put(TimeUnit.MINUTES, "mins").
+ put(TimeUnit.NANOSECONDS, "ns").
+ put(TimeUnit.SECONDS, "secs").build();
+
+ /** dimension to measure things by */
+ private TimeUnit timeUnit;
+
+ /**
+ * Create new timer, add it to the registry.
+ *
+ * @param registry GiraphMetricsRegistry to add timer to
+ * @param name String name of timer
+ * @param timeUnit TimeUnit to measure in
+ */
+ public GiraphTimer(GiraphMetricsRegistry registry, String name,
+ TimeUnit timeUnit) {
+ super(registry, name);
+ this.timeUnit = timeUnit;
+ set(0L);
+ }
+
+ /**
+ * Get TimeUnit used.
+ *
+ * @return TimeUnit being used.
+ */
+ public TimeUnit getTimeUnit() {
+ return timeUnit;
+ }
+
+ /**
+ * Begin timing an event.
+ *
+ * @return GiraphTimerContext. Use stop() to end timing the event.
+ */
+ public GiraphTimerContext time() {
+ return new GiraphTimerContext(this);
+ }
+
+ /**
+ * Set value from a given TimeUnit, converting to our TimeUnit.
+ *
+ * @param value long measurement taken.
+ * @param valueTimeUnit TimeUnit measurement is in.
+ * @return this
+ */
+ public GiraphTimer set(long value, TimeUnit valueTimeUnit) {
+ set(timeUnit.convert(value, valueTimeUnit));
+ return this;
+ }
+
+ /**
+ * Get abbreviated string of TimeUnit.
+ *
+ * @return TimeUnit abbreviation.
+ */
+ public String getTimeUnitAbbrev() {
+ return TIME_UNIT_TO_ABBREV.get(timeUnit);
+ }
+
+ /**
+ * Get string representation of value
+ *
+ * @return String value and abbreviated time unit
+ */
+ public String valueAndUnit() {
+ return "" + value() + " " + getTimeUnitAbbrev();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimerContext.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimerContext.java b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimerContext.java
new file mode 100644
index 0000000..5c7942a
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphTimerContext.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.utils.SystemTime;
+import org.apache.giraph.utils.Time;
+import org.apache.giraph.utils.Times;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Context to use with GiraphTimer for timing an event.
+ */
+public class GiraphTimerContext {
+ /** Object to use for time measurement */
+ private static Time TIME = SystemTime.get();
+
+ /** GiraphTimer to callback to with result of timing */
+ private GiraphTimer giraphTimer;
+ /** Beginning time to compare against */
+ private long startTime;
+
+ /**
+ * Create with GiraphTimer to callback to.
+ *
+ * @param giraphTimer GiraphTimer to use.
+ */
+ public GiraphTimerContext(GiraphTimer giraphTimer) {
+ this.giraphTimer = giraphTimer;
+ this.startTime = TIME.getNanoseconds();
+ }
+
+ /**
+ * End timing. Set time elapsed in GiraphTimer.
+ */
+ public void stop() {
+ giraphTimer.set(Times.getNanosecondsSince(TIME, startTime),
+ TimeUnit.NANOSECONDS);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/LongAndTimeUnit.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/LongAndTimeUnit.java b/giraph-core/src/main/java/org/apache/giraph/metrics/LongAndTimeUnit.java
new file mode 100644
index 0000000..8034eb4
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/LongAndTimeUnit.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Pair of long,TimeUnit
+ */
+public class LongAndTimeUnit {
+ /** Mapping from TimeUnit to abbreviation used for printing */
+ private static final ImmutableMap<TimeUnit, String> TIME_UNIT_TO_ABBREV =
+ ImmutableMap.<TimeUnit, String>builder().
+ put(TimeUnit.DAYS, "days").
+ put(TimeUnit.HOURS, "hours").
+ put(TimeUnit.MICROSECONDS, "us").
+ put(TimeUnit.MILLISECONDS, "ms").
+ put(TimeUnit.MINUTES, "mins").
+ put(TimeUnit.NANOSECONDS, "ns").
+ put(TimeUnit.SECONDS, "secs").build();
+
+ /** value held */
+ private long value;
+ /** TimeUnit part */
+ private TimeUnit timeUnit;
+
+ @Override
+ public String toString() {
+ if (timeUnit == null) {
+ return String.valueOf(value);
+ } else {
+ return value + " " + TIME_UNIT_TO_ABBREV.get(timeUnit);
+ }
+ }
+
+ /**
+ * Get the value
+ *
+ * @return value
+ */
+ public long getValue() {
+ return value;
+ }
+
+ /**
+ * Set the value
+ *
+ * @param value to set
+ */
+ public void setValue(long value) {
+ this.value = value;
+ }
+
+ /**
+ * Get the TimeUnit
+ *
+ * @return TimeUnit
+ */
+ public TimeUnit getTimeUnit() {
+ return timeUnit;
+ }
+
+ /**
+ * Set the TimeUnit
+ * @param timeUnit to set
+ */
+ public void setTimeUnit(TimeUnit timeUnit) {
+ this.timeUnit = timeUnit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/MetricsRegistryDebugger.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/MetricsRegistryDebugger.java b/giraph-core/src/main/java/org/apache/giraph/metrics/MetricsRegistryDebugger.java
new file mode 100644
index 0000000..9a5e73d
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/MetricsRegistryDebugger.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.log4j.Logger;
+
+import com.yammer.metrics.core.Metric;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricsRegistryListener;
+
+/**
+ * A simple MetricsRegistry listener that prints every Metric added/removed.
+ */
+public class MetricsRegistryDebugger implements MetricsRegistryListener {
+ /** logger instance */
+ private static final Logger LOG =
+ Logger.getLogger(MetricsRegistryDebugger.class);
+ /** static singleton instance */
+ private static final MetricsRegistryDebugger INSTANCE =
+ new MetricsRegistryDebugger();
+
+ /**
+ * Get singleton static instance of this class.
+ *
+ * @return singleton static instance.
+ */
+ public static MetricsRegistryDebugger get() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void onMetricAdded(MetricName name, Metric metric) {
+ LOG.info("Adding metric " + name + " - " +
+ metric.getClass().getSimpleName());
+ }
+
+ @Override
+ public void onMetricRemoved(MetricName name) {
+ LOG.info("Removing metric " + name);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/NoOpMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/NoOpMetricsRegistry.java b/giraph-core/src/main/java/org/apache/giraph/metrics/NoOpMetricsRegistry.java
new file mode 100644
index 0000000..28558d5
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/NoOpMetricsRegistry.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import com.google.common.collect.Maps;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.NoOpCounter;
+import com.yammer.metrics.core.NoOpGuage;
+import com.yammer.metrics.core.NoOpHistogram;
+import com.yammer.metrics.core.NoOpMeter;
+import com.yammer.metrics.core.NoOpTimer;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.Meter;
+import com.yammer.metrics.core.Metric;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricPredicate;
+import com.yammer.metrics.core.MetricsRegistry;
+import com.yammer.metrics.core.MetricsRegistryListener;
+import com.yammer.metrics.core.Timer;
+
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An empty MetricsRegistry, used to easily turn off all metrics without
+ * affecting client code.
+ */
+public class NoOpMetricsRegistry extends MetricsRegistry {
+ @Override
+ public <T> Gauge<T> newGauge(Class<?> klass, String name, Gauge<T> metric) {
+ return NoOpGuage.INSTANCE;
+ }
+
+ @Override
+ public <T> Gauge<T> newGauge(Class<?> klass, String name, String scope,
+ Gauge<T> metric) {
+ return NoOpGuage.INSTANCE;
+ }
+
+ @Override
+ public <T> Gauge<T> newGauge(MetricName metricName, Gauge<T> metric) {
+ return NoOpGuage.INSTANCE;
+ }
+
+ @Override
+ public Counter newCounter(Class<?> klass, String name) {
+ return NoOpCounter.INSTANCE;
+ }
+
+ @Override
+ public Counter newCounter(Class<?> klass, String name, String scope) {
+ return NoOpCounter.INSTANCE;
+ }
+
+ @Override
+ public Counter newCounter(MetricName metricName) {
+ return NoOpCounter.INSTANCE;
+ }
+
+ @Override
+ public Histogram newHistogram(Class<?> klass, String name, boolean biased) {
+ return NoOpHistogram.INSTANCE;
+ }
+
+ @Override
+ public Histogram newHistogram(Class<?> klass, String name, String scope,
+ boolean biased) {
+ return NoOpHistogram.INSTANCE;
+ }
+
+ @Override
+ public Histogram newHistogram(Class<?> klass, String name) {
+ return NoOpHistogram.INSTANCE;
+ }
+
+ @Override
+ public Histogram newHistogram(Class<?> klass, String name, String scope) {
+ return NoOpHistogram.INSTANCE;
+ }
+
+ @Override
+ public Histogram newHistogram(MetricName metricName, boolean biased) {
+ return NoOpHistogram.INSTANCE;
+ }
+
+ @Override
+ public Meter newMeter(Class<?> klass, String name, String eventType,
+ TimeUnit unit) {
+ return NoOpMeter.INSTANCE;
+ }
+
+ @Override
+ public Meter newMeter(
+ Class<?> klass, String name, String scope, String eventType, TimeUnit unit
+ ) {
+ return NoOpMeter.INSTANCE;
+ }
+
+ @Override
+ public Meter newMeter(MetricName metricName, String eventType,
+ TimeUnit unit) {
+ return NoOpMeter.INSTANCE;
+ }
+
+ @Override
+ public Timer newTimer(Class<?> klass, String name) {
+ return NoOpTimer.INSTANCE;
+ }
+
+ @Override
+ public Timer newTimer(Class<?> klass, String name, TimeUnit durationUnit,
+ TimeUnit rateUnit) {
+ return NoOpTimer.INSTANCE;
+ }
+
+ @Override
+ public Timer newTimer(Class<?> klass, String name, String scope) {
+ return NoOpTimer.INSTANCE;
+ }
+
+ @Override
+ public Timer newTimer(Class<?> klass, String name, String scope,
+ TimeUnit durationUnit, TimeUnit rateUnit
+ ) {
+ return NoOpTimer.INSTANCE;
+ }
+
+ @Override
+ public Timer newTimer(MetricName metricName, TimeUnit durationUnit,
+ TimeUnit rateUnit) {
+ return NoOpTimer.INSTANCE;
+ }
+
+ @Override
+ public Map<MetricName, Metric> allMetrics() {
+ return Maps.newHashMap();
+ }
+
+ @Override
+ public SortedMap<String, SortedMap<MetricName, Metric>> groupedMetrics() {
+ return Maps.newTreeMap();
+ }
+
+ @Override
+ public SortedMap<String, SortedMap<MetricName, Metric>>
+ groupedMetrics(MetricPredicate predicate) {
+ return Maps.newTreeMap();
+ }
+
+ @Override
+ public void shutdown() { }
+
+ @Override
+ public ScheduledExecutorService newScheduledThreadPool(int poolSize,
+ String name) {
+ return null;
+ }
+
+ @Override
+ public void removeMetric(Class<?> klass, String name) { }
+
+ @Override
+ public void removeMetric(Class<?> klass, String name, String scope) { }
+
+ @Override
+ public void removeMetric(MetricName name) { }
+
+ @Override
+ public void addListener(MetricsRegistryListener listener) { }
+
+ @Override
+ public void removeListener(MetricsRegistryListener listener) { }
+
+ @Override
+ protected ConcurrentMap<MetricName, Metric> newMetricsMap() {
+ return Maps.newConcurrentMap();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/ResetSuperstepMetricsObserver.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/ResetSuperstepMetricsObserver.java b/giraph-core/src/main/java/org/apache/giraph/metrics/ResetSuperstepMetricsObserver.java
new file mode 100644
index 0000000..178618a
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/ResetSuperstepMetricsObserver.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+/**
+ * Observer for users of per-superstep counters. Triggered with calls whenever
+ * we are starting a new superstep so that user can re-initialize their metrics.
+ */
+public interface ResetSuperstepMetricsObserver {
+ /**
+ * Starting a new superstep. Re-initialize your metrics.
+ *
+ * @param superstepMetrics SuperstepMetricsRegistry being used.
+ */
+ void newSuperstep(SuperstepMetricsRegistry superstepMetrics);
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/SuperstepMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/SuperstepMetricsRegistry.java b/giraph-core/src/main/java/org/apache/giraph/metrics/SuperstepMetricsRegistry.java
new file mode 100644
index 0000000..dd9eef7
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/SuperstepMetricsRegistry.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.graph.BspService;
+
+import java.io.PrintStream;
+
+/**
+ * Wrapper around MetricsRegistry for per-superstep metrics.
+ */
+public class SuperstepMetricsRegistry extends GiraphMetricsRegistry {
+ /** Number of superstep to use for group of metrics created */
+ private long superstep = BspService.INPUT_SUPERSTEP;
+
+ /**
+ * Create no-op registry that creates no-op metrics.
+ */
+ public SuperstepMetricsRegistry() {
+ super();
+ }
+
+ /**
+ * Create with Hadoop Configuration and superstep number.
+ *
+ * @param conf Hadoop Configuration to use.
+ * @param superstep number of superstep to use as group for metrics.
+ */
+ public SuperstepMetricsRegistry(GiraphConfiguration conf, long superstep) {
+ super(conf, "giraph.superstep", String.valueOf(superstep));
+ this.superstep = superstep;
+ }
+
+ /**
+ * Set superstep number used. Internally sets the group for metrics created.
+ *
+ * @param superstep long number of superstep to use.
+ */
+ public void setSuperstep(long superstep) {
+ super.setType(String.valueOf(superstep));
+ this.superstep = superstep;
+ }
+
+ /**
+ * Print human readable summary of superstep metrics.
+ *
+ * @param out PrintStream to write to.
+ */
+ public void printSummary(PrintStream out) {
+ new WorkerSuperstepMetrics().readFromRegistry().print(superstep, out);
+ }
+
+ /**
+ * Print human readable summary of superstep metrics.
+ */
+ public void printSummary() {
+ printSummary(System.out);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/ValueGauge.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/ValueGauge.java b/giraph-core/src/main/java/org/apache/giraph/metrics/ValueGauge.java
new file mode 100644
index 0000000..aa7ab30
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/ValueGauge.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import com.yammer.metrics.core.Gauge;
+
+/**
+ * A Gauge that holds a value.
+ *
+ * @param <T> type of value being held.
+ */
+public class ValueGauge<T extends Number> extends Gauge<T> {
+ /** value held by this class */
+ private T val;
+
+ /**
+ * Constructor that registers Gauge in MetricsRegistry.
+ *
+ * @param registry GiraphMetricsRegistry to use.
+ * @param name String name of Gauge.
+ */
+ public ValueGauge(GiraphMetricsRegistry registry, String name) {
+ registry.getGauge(name, this);
+ }
+
+ @Override
+ public T value() {
+ return val;
+ }
+
+ /**
+ * Get double representation of value held.
+ *
+ * @return double value
+ */
+ public double getDouble() {
+ return val != null ? val.doubleValue() : 0.0d;
+ }
+
+ /**
+ * Get long representation of value held.
+ *
+ * @return long value
+ */
+ public long getLong() {
+ return val != null ? val.longValue() : 0L;
+ }
+
+ /**
+ * Set value held by this object.
+ *
+ * @param value value to set.
+ * @return this
+ */
+ public ValueGauge<T> set(T value) {
+ this.val = value;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/ValueWithHostname.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/ValueWithHostname.java b/giraph-core/src/main/java/org/apache/giraph/metrics/ValueWithHostname.java
new file mode 100644
index 0000000..208c4c7
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/ValueWithHostname.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+/**
+ * Pair of value with host it came from.
+ */
+class ValueWithHostname {
+ /** long value we're holding */
+ private long value;
+ /** host associated with value */
+ private String hostname;
+
+ /**
+ * Create with initial value
+ *
+ * @param value long initial value to use
+ */
+ public ValueWithHostname(long value) {
+ this.value = value;
+ this.hostname = null;
+ }
+
+ /**
+ * @return long value
+ */
+ public long getValue() {
+ return value;
+ }
+
+ /**
+ * @return String hostname
+ */
+ public String getHostname() {
+ return hostname;
+ }
+
+ /**
+ * Check if there is any hostname. Used as a flag that we have any data.
+ *
+ * @return true if hostname is set
+ */
+ public boolean hasHostname() {
+ return hostname != null;
+ }
+
+ /**
+ * Set value and partition together.
+ * @param value long value to use.
+ * @param hostname String host it came from.
+ */
+ public void set(long value, String hostname) {
+ this.value = value;
+ this.hostname = hostname;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/WorkerSuperstepMetrics.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/WorkerSuperstepMetrics.java b/giraph-core/src/main/java/org/apache/giraph/metrics/WorkerSuperstepMetrics.java
new file mode 100644
index 0000000..9db5107
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/WorkerSuperstepMetrics.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.metrics;
+
+import org.apache.giraph.graph.BspServiceWorker;
+import org.apache.giraph.graph.ComputeCallable;
+import org.apache.giraph.graph.GraphMapper;
+import org.apache.hadoop.io.Writable;
+
+import com.yammer.metrics.core.Gauge;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Per-superstep metrics for a Worker.
+ */
+public class WorkerSuperstepMetrics implements Writable {
+ /** Total network communication time */
+ private LongAndTimeUnit commTimer;
+ /** Time for all compute calls to complete */
+ private LongAndTimeUnit computeAllTimer;
+ /** Time till first message gets flushed */
+ private LongAndTimeUnit timeToFirstMsg;
+ /** Total superstep time */
+ private LongAndTimeUnit superstepTimer;
+ /** Time spent waiting for other workers to finish */
+ private LongAndTimeUnit waitRequestsTimer;
+ /** Time spent in Vertex#compute */
+ private LongAndTimeUnit userComputeTime;
+
+ /**
+ * Constructor
+ */
+ public WorkerSuperstepMetrics() {
+ commTimer = new LongAndTimeUnit();
+ computeAllTimer = new LongAndTimeUnit();
+ timeToFirstMsg = new LongAndTimeUnit();
+ superstepTimer = new LongAndTimeUnit();
+ waitRequestsTimer = new LongAndTimeUnit();
+
+ // Note this one is not backed by a GiraphTimer, but rather a real Timer
+ userComputeTime = new LongAndTimeUnit();
+ userComputeTime.setTimeUnit(TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Read metric values from global MetricsRegistry.
+ *
+ * @return this object, for chaining
+ */
+ public WorkerSuperstepMetrics readFromRegistry() {
+ SuperstepMetricsRegistry ssm = GiraphMetrics.get().perSuperstep();
+ readGiraphTimer(GraphMapper.TIMER_COMMUNICATION_TIME, commTimer);
+ readGiraphTimer(GraphMapper.TIMER_COMPUTE_ALL, computeAllTimer);
+ readGiraphTimer(GraphMapper.TIMER_TIME_TO_FIRST_MSG, timeToFirstMsg);
+ readGiraphTimer(GraphMapper.TIMER_SUPERSTEP_TIME, superstepTimer);
+ readGiraphTimer(BspServiceWorker.TIMER_WAIT_REQUESTS, waitRequestsTimer);
+ userComputeTime.setValue((long) ssm.getTimer(
+ ComputeCallable.TIMER_COMPUTE_ONE).sum());
+ return this;
+ }
+
+ /**
+ * Read data from GiraphTimer into a LongAndTimeUnit.
+ *
+ * @param name String name of Gauge to retrieve.
+ * @param data LongAndTimeUnit to read data into.
+ */
+ private void readGiraphTimer(String name, LongAndTimeUnit data) {
+ Gauge<Long> gauge = GiraphMetrics.get().perSuperstep().
+ getExistingGauge(name);
+ if (gauge instanceof GiraphTimer) {
+ GiraphTimer giraphTimer = (GiraphTimer) gauge;
+ data.setTimeUnit(giraphTimer.getTimeUnit());
+ data.setValue(giraphTimer.value());
+ } else if (gauge != null) {
+ throw new IllegalStateException(name + " is not a GiraphTimer");
+ }
+ }
+
+ /**
+ * Human readable dump of metrics stored here.
+ *
+ * @param superstep long number of superstep.
+ * @param out PrintStream to write to.
+ * @return this object, for chaining
+ */
+ public WorkerSuperstepMetrics print(long superstep, PrintStream out) {
+ out.println();
+ out.println("--- METRICS: superstep " + superstep + " ---");
+ out.println(" superstep time: " + superstepTimer);
+ out.println(" compute all partitions: " + computeAllTimer);
+ out.println(" user compute time: " + userComputeTime);
+ out.println(" network communication time: " + commTimer);
+ out.println(" time to first message: " + timeToFirstMsg);
+ out.println(" wait on requests time: " + waitRequestsTimer);
+ return this;
+ }
+
+ /**
+ * @return Communication timer
+ */
+ public long getCommTimer() {
+ return commTimer.getValue();
+ }
+
+ /**
+ * @return Total compute timer
+ */
+ public long getComputeAllTimer() {
+ return computeAllTimer.getValue();
+ }
+
+ /**
+ * @return timer between start time and first message flushed.
+ */
+ public long getTimeToFirstMsg() {
+ return timeToFirstMsg.getValue();
+ }
+
+ /**
+ * @return timer for superstep time
+ */
+ public long getSuperstepTimer() {
+ return superstepTimer.getValue();
+ }
+
+ /**
+ * @return timer waiting for other workers
+ */
+ public long getWaitRequestsTimer() {
+ return waitRequestsTimer.getValue();
+ }
+
+ /**
+ * @return milliseconds in user compute code
+ */
+ public long getUserComputeTime() {
+ return userComputeTime.getValue();
+ }
+
+ @Override
+ public void readFields(DataInput dataInput) throws IOException {
+ commTimer.setValue(dataInput.readLong());
+ computeAllTimer.setValue(dataInput.readLong());
+ timeToFirstMsg.setValue(dataInput.readLong());
+ superstepTimer.setValue(dataInput.readLong());
+ waitRequestsTimer.setValue(dataInput.readLong());
+ userComputeTime.setValue(dataInput.readLong());
+ }
+
+ @Override
+ public void write(DataOutput dataOutput) throws IOException {
+ dataOutput.writeLong(commTimer.getValue());
+ dataOutput.writeLong(computeAllTimer.getValue());
+ dataOutput.writeLong(timeToFirstMsg.getValue());
+ dataOutput.writeLong(superstepTimer.getValue());
+ dataOutput.writeLong(waitRequestsTimer.getValue());
+ dataOutput.writeLong(userComputeTime.getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/metrics/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/package-info.java b/giraph-core/src/main/java/org/apache/giraph/metrics/package-info.java
new file mode 100644
index 0000000..20f3785
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/metrics/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Package of all the Giraph metrics related data.
+ */
+package org.apache.giraph.metrics;
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/package-info.java b/giraph-core/src/main/java/org/apache/giraph/package-info.java
new file mode 100644
index 0000000..3c5f960
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Base giraph package.
+ */
+package org.apache.giraph;
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/utils/AnnotationUtils.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/AnnotationUtils.java b/giraph-core/src/main/java/org/apache/giraph/utils/AnnotationUtils.java
new file mode 100644
index 0000000..79bb1c6
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/AnnotationUtils.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.utils;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.jar.JarEntry;
+import java.util.jar.JarInputStream;
+
+/**
+ * Helper class to deal with annotations in runtime.
+ */
+public class AnnotationUtils {
+ /** Do not instantiate. */
+ private AnnotationUtils() {
+ }
+
+ /**
+ * Finds all classes within a package which are annotated with certain
+ * annotation.
+ *
+ * @param annotation Annotation which we are looking for
+ * @param <T> Annotation class
+ * @param packageName Package in which to search
+ * @return The list of annotated classes
+ */
+ public static <T extends Annotation> List<Class<?>> getAnnotatedClasses(
+ Class<T> annotation, String packageName) {
+ ArrayList<Class<?>> ret = new ArrayList<Class<?>>();
+ for (Iterator<Class<?>> it = getClassesIterator(packageName);
+ it.hasNext();) {
+ Class<?> clazz = it.next();
+ if (clazz.getAnnotation(annotation) != null) {
+ ret.add(clazz);
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * @param packageName Package through which to iterate
+ * @return Iterator through the classes of this jar file (if executed from
+ * jar) or through the classes of org package (if .class file is
+ * executed)
+ */
+ public static Iterator<Class<?>> getClassesIterator(String packageName) {
+ if (isExecutedFromJar()) {
+ try {
+ return new JarClassesIterator(packageName);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ return new GeneralClassesIterator(packageName);
+ }
+ }
+
+ /**
+ * @return Whether or not the code is executed from jar file
+ */
+ private static boolean isExecutedFromJar() {
+ return AnnotationUtils.class.getResource("AnnotationUtils.class")
+ .getProtocol().equals("jar");
+ }
+
+ /**
+ * To be used when {@link #isExecutedFromJar()} is true.
+ *
+ * @return If executed from jar file returns the path to the jar
+ * (otherwise it will return the folder in which this .class file is
+ * located)
+ */
+ private static String getCurrentJar() {
+ return AnnotationUtils.class.getProtectionDomain().getCodeSource()
+ .getLocation().getFile();
+ }
+
+ /**
+ * To be used when {@link #isExecutedFromJar()} is true.
+ * <p/>
+ * Iterator through classes of this jar file.
+ */
+ private static class JarClassesIterator implements Iterator<Class<?>> {
+ /** Used to go through classes in current jar */
+ private final JarInputStream jarIn;
+ /** Are we positioned on the next class entry */
+ private boolean entryLoaded;
+ /** Next class entry */
+ private JarEntry currentEntry;
+ /** Folder in which to look */
+ private final String path;
+
+ /**
+ * @param packageName Package through which to iterate
+ * @throws IOException
+ */
+ public JarClassesIterator(String packageName) throws IOException {
+ jarIn = new JarInputStream(new FileInputStream(new File(
+ getCurrentJar())));
+ entryLoaded = false;
+ currentEntry = null;
+ path = packageName.replace(".", File.separator);
+ }
+
+ @Override
+ public boolean hasNext() {
+ loadNextEntry();
+ return currentEntry != null;
+ }
+
+ @Override
+ public Class<?> next() {
+ loadNextEntry();
+ if (currentEntry == null) {
+ throw new NoSuchElementException();
+ }
+ entryLoaded = false;
+
+ String className = currentEntry.getName().replace(".class",
+ "").replace(File.separator, ".");
+ return loadClass(className);
+ }
+
+ /**
+ * Sets position to next class entry
+ */
+ private void loadNextEntry() {
+ while (!entryLoaded) {
+ try {
+ currentEntry = jarIn.getNextJarEntry();
+ if (currentEntry == null || (currentEntry.getName().endsWith(
+ ".class") && (currentEntry.getName().startsWith(path)))) {
+ entryLoaded = true;
+ } else {
+ currentEntry = null;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ if (currentEntry == null) {
+ try {
+ jarIn.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException(
+ "Can't remove from classes iterator");
+ }
+ }
+
+ /**
+ * To be used when {@link #isExecutedFromJar()} is false.
+ * <p/>
+ * Iterator through classes of some package.
+ */
+ private static class GeneralClassesIterator implements Iterator<Class<?>> {
+ /** From which position in path the package name starts */
+ private final int stringPosition;
+ /** Recursive directory iterator */
+ private final Iterator<File> iterator;
+
+ /**
+ * @param packageName Package through which to iterate
+ */
+ public GeneralClassesIterator(String packageName) {
+ String mainPath = AnnotationUtils.class.getProtectionDomain()
+ .getCodeSource().getLocation().getFile();
+ String subPath = packageName.replace(".", File.separator);
+ File directory = new File(mainPath + subPath);
+ stringPosition = directory.getPath().length() - packageName.length();
+ List<File> files = new ArrayList<File>();
+ addAllClassFiles(directory, files);
+ iterator = files.iterator();
+ }
+
+ /**
+ * Recursively add all .class files from the directory to the list.
+ *
+ * @param directory Directory from which we are adding files
+ * @param files List we add files to
+ */
+ private void addAllClassFiles(File directory, List<File> files) {
+ for (File file : directory.listFiles()) {
+ if (file.isDirectory()) {
+ addAllClassFiles(file, files);
+ } else if (file.getName().endsWith(".class")) {
+ files.add(file);
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public Class<?> next() {
+ String className = iterator.next().getPath().substring(stringPosition)
+ .replace(".class", "").replace(File.separator, ".");
+ return loadClass(className);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException(
+ "Can't remove from classes iterator");
+ }
+ }
+
+ /**
+ * Loads the class with the specified name
+ *
+ * @param className Name of the class we are loading
+ * @return Class with the specified name
+ */
+ private static Class<?> loadClass(String className) {
+ try {
+ return Class.forName(className);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Error loading class " + className, e);
+ } catch (NoClassDefFoundError e) {
+ throw new RuntimeException("Error loading class " + className, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/utils/ArrayListWritable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/ArrayListWritable.java b/giraph-core/src/main/java/org/apache/giraph/utils/ArrayListWritable.java
new file mode 100644
index 0000000..9d8f4f1
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/ArrayListWritable.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.utils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * A Writable for ListArray containing instances of a class.
+ *
+ * @param <M> Message data
+ */
+public abstract class ArrayListWritable<M extends Writable> extends ArrayList<M>
+ implements Writable, Configurable {
+ /** Defining a layout version for a serializable class. */
+ private static final long serialVersionUID = 1L;
+ /** Used for instantiation */
+ private Class<M> refClass = null;
+
+ /** Configuration */
+ private Configuration conf;
+
+ /**
+ * Using the default constructor requires that the user implement
+ * setClass(), guaranteed to be invoked prior to instantiation in
+ * readFields()
+ */
+ public ArrayListWritable() {
+ }
+
+ /**
+ * Constructor with another {@link ArrayListWritable}.
+ *
+ * @param arrayListWritable Array list to be used internally.
+ */
+ public ArrayListWritable(ArrayListWritable<M> arrayListWritable) {
+ super(arrayListWritable);
+ }
+
+ /**
+ * This constructor allows setting the refClass during construction.
+ *
+ * @param refClass internal type class
+ */
+ public ArrayListWritable(Class<M> refClass) {
+ super();
+ this.refClass = refClass;
+ }
+
+ /**
+ * This is a one-time operation to set the class type
+ *
+ * @param refClass internal type class
+ */
+ public void setClass(Class<M> refClass) {
+ if (this.refClass != null) {
+ throw new RuntimeException(
+ "setClass: refClass is already set to " +
+ this.refClass.getName());
+ }
+ this.refClass = refClass;
+ }
+
+ /**
+ * Subclasses must set the class type appropriately and can use
+ * setClass(Class<M> refClass) to do it.
+ */
+ public abstract void setClass();
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ if (this.refClass == null) {
+ setClass();
+ }
+ int numValues = in.readInt(); // read number of values
+ ensureCapacity(numValues);
+ for (int i = 0; i < numValues; i++) {
+ M value = ReflectionUtils.newInstance(refClass, conf);
+ value.readFields(in); // read a value
+ add(value); // store it in values
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ int numValues = size();
+ out.writeInt(numValues); // write number of values
+ for (int i = 0; i < numValues; i++) {
+ get(i).write(out);
+ }
+ }
+
+ @Override
+ public final Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public final void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterable.java b/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterable.java
new file mode 100644
index 0000000..cf2c187
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterable.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.utils;
+
+import java.util.Iterator;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This iterable is designed to deserialize a byte array on the fly to
+ * provide new copies of writable objects when desired. It does not reuse
+ * objects, and instead creates a new one for every next().
+ *
+ * @param <T> Type that extends Writable that will be iterated
+ */
+public abstract class ByteArrayIterable<T extends Writable> implements
+ Iterable<T> {
+ /** Configuration */
+ protected final ImmutableClassesGiraphConfiguration configuration;
+ /** Data input */
+ protected final byte[] buf;
+ /** Offset to start in buf */
+ protected final int off;
+ /** Length of buf */
+ protected final int length;
+
+ /**
+ * Constructor
+ *
+ * @param configuration Configuration
+ * @param buf Buffer
+ * @param off Offset to start in the buffer
+ * @param length Length of the buffer
+ */
+ public ByteArrayIterable(ImmutableClassesGiraphConfiguration configuration,
+ byte[] buf, int off, int length) {
+ this.configuration = configuration;
+ this.buf = buf;
+ this.off = off;
+ this.length = length;
+ }
+
+ /**
+ * Must be able to create the writable object
+ *
+ * @return New writable
+ */
+ protected abstract T createWritable();
+
+ /**
+ * Iterator over the internal byte array
+ */
+ private class ByteArrayIterableIterator extends ByteArrayIterator<T> {
+ /**
+ * Constructor.
+ *
+ * @param buf Buffer to read from
+ * @param off Offset to read from in the buffer
+ * @param length Maximum length of the buffer
+ */
+ private ByteArrayIterableIterator(byte[] buf, int off, int length) {
+ super(configuration, buf, off, length);
+ }
+
+ @Override
+ protected T createWritable() {
+ return ByteArrayIterable.this.createWritable();
+ }
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ return new ByteArrayIterableIterator(buf, off, length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterator.java b/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterator.java
new file mode 100644
index 0000000..76ed789
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/ByteArrayIterator.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.utils;
+
+import java.io.IOException;
+import java.util.Iterator;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This iterator is designed to deserialize a byte array on the fly to
+ * provide new copies of writable objects when desired. It does not reuse
+ * objects, and instead creates a new one for every next().
+ *
+ * @param <T> Type that extends Writable that will be iterated
+ */
+public abstract class ByteArrayIterator<T extends Writable> implements
+ Iterator<T> {
+ /** Data input */
+ protected final ExtendedDataInput extendedDataInput;
+
+ /**
+ * Constructor
+ *
+ * @param configuration Configuration
+ * @param buf Buffer
+ * @param off Offset to start in the buffer
+ * @param length Length of the buffer
+ */
+ public ByteArrayIterator(
+ ImmutableClassesGiraphConfiguration configuration,
+ byte[] buf, int off, int length) {
+ extendedDataInput =
+ configuration.createExtendedDataInput(buf, off, length);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return extendedDataInput.available() > 0;
+ }
+
+ @Override
+ public T next() {
+ T writable = createWritable();
+ try {
+ writable.readFields(extendedDataInput);
+ } catch (IOException e) {
+ throw new IllegalStateException("next: readFields got IOException", e);
+ }
+ return writable;
+ }
+
+ @Override
+ public void remove() {
+ throw new IllegalAccessError("remove: Not supported");
+ }
+
+ /**
+ * Must be able to create the writable object
+ *
+ * @return New writable
+ */
+ protected abstract T createWritable();
+}