You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by jg...@apache.org on 2012/02/10 21:16:24 UTC
svn commit: r1242909 - in /incubator/giraph/trunk: ./
src/main/java/org/apache/giraph/benchmark/ src/test/java/org/apache/giraph/
Author: jghoman
Date: Fri Feb 10 20:16:23 2012
New Revision: 1242909
URL: http://svn.apache.org/viewvc?rev=1242909&view=rev
Log:
GIRAPH-139: Change PageRankBenchmark to be accessible via bin/giraph.
Added:
incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java
incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java
Modified:
incubator/giraph/trunk/CHANGELOG
incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
incubator/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java
Modified: incubator/giraph/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/CHANGELOG?rev=1242909&r1=1242908&r2=1242909&view=diff
==============================================================================
--- incubator/giraph/trunk/CHANGELOG (original)
+++ incubator/giraph/trunk/CHANGELOG Fri Feb 10 20:16:23 2012
@@ -2,6 +2,9 @@ Giraph Change Log
Release 0.2.0 - unreleased
+ GIRAPH-139: Change PageRankBenchmark to be accessible via bin/giraph.
+ (jghoman)
+
GIRAPH-143: Add support for giraph to have a conf file. (jghoman)
GIRAPH-142: _hadoopBsp should be prefixable via configuration. (jghoman)
Added: incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java?rev=1242909&view=auto
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java (added)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java Fri Feb 10 20:16:23 2012
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.benchmark;
+
+import org.apache.giraph.graph.HashMapVertex;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.LongWritable;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * Same benchmark code as {@link PageRankBenchmark}, but uses Hashmap-backed Vertex
+ * implementation rather than {@link org.apache.giraph.graph.EdgeListVertex}
+ */
+public class HashMapVertexPageRankBenchmark extends HashMapVertex<
+ LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> {
+ public final static String SUPERSTEP_COUNT = PageRankComputation.SUPERSTEP_COUNT;
+
+ @Override
+ public void compute(Iterator<DoubleWritable> msgIterator) throws IOException {
+ PageRankComputation.computePageRank(this, msgIterator);
+ }
+}
Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java?rev=1242909&r1=1242908&r2=1242909&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java (original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java Fri Feb 10 20:16:23 2012
@@ -25,65 +25,28 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.giraph.graph.EdgeListVertex;
import org.apache.giraph.graph.GiraphJob;
-import org.apache.giraph.graph.HashMapVertex;
-import org.apache.giraph.graph.MutableVertex;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import java.io.IOException;
import java.util.Iterator;
/**
- * Benchmark based on the basic Pregel PageRank implementation.
+ * Default Pregel-style PageRank computation using a {@link EdgeListVertex}.
*/
-public class PageRankBenchmark implements Tool {
+public class PageRankBenchmark extends EdgeListVertex<
+ LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> implements Tool {
+ public final static String SUPERSTEP_COUNT = PageRankComputation.SUPERSTEP_COUNT;
+
/** Configuration from Configurable */
private Configuration conf;
- /** How many supersteps to run */
- public static String SUPERSTEP_COUNT = "PageRankBenchmark.superstepCount";
-
- private static void computePageRank(
- MutableVertex
- <LongWritable, DoubleWritable, DoubleWritable, DoubleWritable>
- vertex,
- Iterator<DoubleWritable> msgIterator) {
- if (vertex.getSuperstep() >= 1) {
- double sum = 0;
- while (msgIterator.hasNext()) {
- sum += msgIterator.next().get();
- }
- DoubleWritable vertexValue =
- new DoubleWritable((0.15f / vertex.getNumVertices()) + 0.85f *
- sum);
- vertex.setVertexValue(vertexValue);
- }
-
- if (vertex.getSuperstep() < vertex.getConf().getInt(SUPERSTEP_COUNT, -1)) {
- long edges = vertex.getNumOutEdges();
- vertex.sendMsgToAllEdges(
- new DoubleWritable(vertex.getVertexValue().get() / edges));
- } else {
- vertex.voteToHalt();
- }
- }
-
- public static class PageRankHashMapVertex extends HashMapVertex<
- LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> {
- @Override
- public void compute(Iterator<DoubleWritable> msgIterator) {
- PageRankBenchmark.computePageRank(this, msgIterator);
- }
- }
-
- public static class PageRankEdgeListVertex extends EdgeListVertex<
- LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> {
- @Override
- public void compute(Iterator<DoubleWritable> msgIterator) {
- PageRankBenchmark.computePageRank(this, msgIterator);
- }
+ @Override
+ public void compute(Iterator<DoubleWritable> msgIterator) throws IOException {
+ PageRankComputation.computePageRank(this, msgIterator);
}
@Override
@@ -120,7 +83,7 @@ public class PageRankBenchmark implement
options.addOption("c",
"vertexClass",
true,
- "Vertex class (0 for Vertex, 1 for EdgeListVertex)");
+ "Vertex class (0 for HashMapVertex, 1 for EdgeListVertex)");
HelpFormatter formatter = new HelpFormatter();
if (args.length == 0) {
formatter.printHelp(getClass().getName(), options, true);
@@ -155,12 +118,12 @@ public class PageRankBenchmark implement
if (!cmd.hasOption('c') ||
(Integer.parseInt(cmd.getOptionValue('c')) == 0)) {
System.out.println("Using " +
- PageRankHashMapVertex.class.getName());
- job.setVertexClass(PageRankHashMapVertex.class);
+ HashMapVertexPageRankBenchmark.class.getName());
+ job.setVertexClass(HashMapVertexPageRankBenchmark.class);
} else {
System.out.println("Using " +
- PageRankEdgeListVertex.class.getName());
- job.setVertexClass(PageRankEdgeListVertex.class);
+ PageRankBenchmark.class.getName());
+ job.setVertexClass(PageRankBenchmark.class);
}
job.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class);
job.setWorkerConfiguration(workers, workers, 100.0f);
Added: incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java?rev=1242909&view=auto
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java (added)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java Fri Feb 10 20:16:23 2012
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.benchmark;
+
+import org.apache.giraph.graph.MutableVertex;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.LongWritable;
+
+import java.util.Iterator;
+
+/**
+ * Shared computation of class Pregel-style PageRank computation for benchmark
+ * classes.
+ */
+public class PageRankComputation {
+ public final static String SUPERSTEP_COUNT = "PageRankBenchmark.superstepCount";
+
+ public static void computePageRank(
+ MutableVertex<LongWritable, DoubleWritable, DoubleWritable, DoubleWritable>
+ vertex, Iterator<DoubleWritable> msgIterator) {
+ if (vertex.getSuperstep() >= 1) {
+ double sum = 0;
+ while (msgIterator.hasNext()) {
+ sum += msgIterator.next().get();
+ }
+ DoubleWritable vertexValue =
+ new DoubleWritable((0.15f / vertex.getNumVertices()) + 0.85f * sum);
+ vertex.setVertexValue(vertexValue);
+ }
+
+ if (vertex.getSuperstep() < vertex.getConf().getInt(SUPERSTEP_COUNT, -1)) {
+ long edges = vertex.getNumOutEdges();
+ vertex.sendMsgToAllEdges(
+ new DoubleWritable(vertex.getVertexValue().get() / edges));
+ } else {
+ vertex.voteToHalt();
+ }
+ }
+}
Modified: incubator/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java?rev=1242909&r1=1242908&r2=1242909&view=diff
==============================================================================
--- incubator/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java (original)
+++ incubator/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java Fri Feb 10 20:16:23 2012
@@ -65,7 +65,7 @@ public class TestJsonBase64Format extend
throws IOException, InterruptedException, ClassNotFoundException {
GiraphJob job = new GiraphJob(getCallingMethodName());
setupConfiguration(job);
- job.setVertexClass(PageRankBenchmark.PageRankEdgeListVertex.class);
+ job.setVertexClass(PageRankBenchmark.class);
job.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class);
job.setVertexOutputFormatClass(JsonBase64VertexOutputFormat.class);
job.getConfiguration().setLong(
@@ -79,7 +79,7 @@ public class TestJsonBase64Format extend
job = new GiraphJob(getCallingMethodName());
setupConfiguration(job);
- job.setVertexClass(PageRankBenchmark.PageRankEdgeListVertex.class);
+ job.setVertexClass(PageRankBenchmark.class);
job.setVertexInputFormatClass(JsonBase64VertexInputFormat.class);
job.setVertexOutputFormatClass(JsonBase64VertexOutputFormat.class);
job.getConfiguration().setInt(PageRankBenchmark.SUPERSTEP_COUNT, 3);
@@ -95,7 +95,7 @@ public class TestJsonBase64Format extend
job = new GiraphJob(getCallingMethodName());
setupConfiguration(job);
- job.setVertexClass(PageRankBenchmark.PageRankEdgeListVertex.class);
+ job.setVertexClass(PageRankBenchmark.class);
job.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class);
job.setVertexOutputFormatClass(JsonBase64VertexOutputFormat.class);
job.getConfiguration().setLong(