You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by cl...@apache.org on 2013/06/26 00:00:07 UTC
git commit: updated refs/heads/trunk to f8a3c77
Updated Branches:
refs/heads/trunk 7ea62d9c1 -> f8a3c777e
GIRAPH-696: Should be able to spill giraph metrics to a specified directory on HDFS
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/f8a3c777
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/f8a3c777
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/f8a3c777
Branch: refs/heads/trunk
Commit: f8a3c777e1fc1902a6a965c1d2a33e7ce3cc1643
Parents: 7ea62d9
Author: Claudio Martella <cl...@apache.org>
Authored: Tue Jun 25 23:59:32 2013 +0200
Committer: Claudio Martella <cl...@apache.org>
Committed: Tue Jun 25 23:59:32 2013 +0200
----------------------------------------------------------------------
CHANGELOG | 3 ++
.../org/apache/giraph/conf/GiraphConstants.java | 5 +++
.../apache/giraph/master/BspServiceMaster.java | 41 +++++++++++++++++++-
3 files changed, 48 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/f8a3c777/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 36693ab..0f2758e 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,9 @@
Giraph Change Log
Release 1.1.0 - unreleased
+ GIRAPH-696: Should be able to spill giraph metrics to a specified
+ directory on HDFS (claudio)
+
GIRAPH-695: Add getCombiner and getComputation methods for
MasterCompute (aching)
http://git-wip-us.apache.org/repos/asf/giraph/blob/f8a3c777/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
index 80afab9..c65b5f0 100644
--- a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
+++ b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
@@ -315,6 +315,11 @@ public interface GiraphConstants {
new BooleanConfOption("giraph.metrics.enable", false,
"Enable the Metrics system");
+ /** Directory in HDFS to write master metrics to, instead of stderr */
+ StrConfOption METRICS_DIRECTORY =
+ new StrConfOption("giraph.metrics.directory", "",
+ "Directory in HDFS to write master metrics to, instead of stderr");
+
/**
* ZooKeeper comma-separated list (if not set,
* will start up ZooKeeper locally)
http://git-wip-us.apache.org/repos/asf/giraph/blob/f8a3c777/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java b/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java
index 0d266a6..3558887 100644
--- a/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java
+++ b/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java
@@ -91,6 +91,7 @@ import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -974,7 +975,11 @@ public class BspServiceMaster<I extends WritableComparable,
}
if (conf.metricsEnabled()) {
- aggregatedMetrics.print(superstep, System.err);
+ if (GiraphConstants.METRICS_DIRECTORY.isDefaultValue(conf)) {
+ aggregatedMetrics.print(superstep, System.err);
+ } else {
+ printAggregatedMetricsToHDFS(superstep, aggregatedMetrics);
+ }
}
if (LOG.isInfoEnabled()) {
@@ -985,6 +990,40 @@ public class BspServiceMaster<I extends WritableComparable,
}
/**
+ * Write superstep metrics to own file in HDFS
+ * @param superstep the current superstep
+ * @param aggregatedMetrics the aggregated metrics to write
+ */
+ private void printAggregatedMetricsToHDFS(
+ long superstep, AggregatedMetrics aggregatedMetrics) {
+ ImmutableClassesGiraphConfiguration conf = getConfiguration();
+ PrintStream out = null;
+ Path dir = new Path(GiraphConstants.METRICS_DIRECTORY.get(conf));
+ Path outFile = new Path(GiraphConstants.METRICS_DIRECTORY.get(conf) +
+ Path.SEPARATOR_CHAR + "superstep_" + superstep + ".metrics");
+ try {
+ FileSystem fs;
+ fs = FileSystem.get(conf);
+ if (!fs.exists(dir)) {
+ fs.mkdirs(dir);
+ }
+ if (fs.exists(outFile)) {
+ throw new RuntimeException(
+ "printAggregatedMetricsToHDFS: metrics file exists");
+ }
+ out = new PrintStream(fs.create(outFile));
+ aggregatedMetrics.print(superstep, out);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "printAggregatedMetricsToHDFS: error creating metrics file", e);
+ } finally {
+ if (out != null) {
+ out.close();
+ }
+ }
+ }
+
+ /**
* Finalize the checkpoint file prefixes by taking the chosen workers and
* writing them to a finalized file. Also write out the master
* aggregated aggregator array from the previous superstep.