You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2013/01/24 06:20:39 UTC
svn commit: r1437860 - in /hama/trunk: ./ core/
examples/src/main/java/org/apache/hama/examples/
examples/src/main/java/org/apache/hama/examples/util/
examples/src/test/java/org/apache/hama/examples/
Author: edwardyoon
Date: Thu Jan 24 05:20:38 2013
New Revision: 1437860
URL: http://svn.apache.org/viewvc?rev=1437860&view=rev
Log:
HAMA-714: Align format consistency between examples and generators
Removed:
hama/trunk/examples/src/main/java/org/apache/hama/examples/util/VertexInputGen.java
Modified:
hama/trunk/CHANGES.txt
hama/trunk/core/pom.xml
hama/trunk/examples/src/main/java/org/apache/hama/examples/PageRank.java
hama/trunk/examples/src/test/java/org/apache/hama/examples/PageRankTest.java
hama/trunk/pom.xml
Modified: hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hama/trunk/CHANGES.txt?rev=1437860&r1=1437859&r2=1437860&view=diff
==============================================================================
--- hama/trunk/CHANGES.txt (original)
+++ hama/trunk/CHANGES.txt Thu Jan 24 05:20:38 2013
@@ -16,6 +16,7 @@ Release 0.7 (unreleased changes)
IMPROVEMENTS
+ HAMA-714: Align format consistency between examples and generators (edwardyoon)
HAMA-531: Reimplementation of partitioner (edwardyoon)
Release 0.6 - November 28, 2012
Modified: hama/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/core/pom.xml?rev=1437860&r1=1437859&r2=1437860&view=diff
==============================================================================
--- hama/trunk/core/pom.xml (original)
+++ hama/trunk/core/pom.xml Thu Jan 24 05:20:38 2013
@@ -109,18 +109,6 @@
<artifactId>hadoop-test</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </dependency>
- <dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</dependency>
Modified: hama/trunk/examples/src/main/java/org/apache/hama/examples/PageRank.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/PageRank.java?rev=1437860&r1=1437859&r2=1437860&view=diff
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/PageRank.java (original)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/PageRank.java Thu Jan 24 05:20:38 2013
@@ -25,12 +25,13 @@ import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
import org.apache.hama.HamaConfiguration;
import org.apache.hama.bsp.HashPartitioner;
import org.apache.hama.bsp.SequenceFileInputFormat;
+import org.apache.hama.bsp.TextArrayWritable;
import org.apache.hama.bsp.TextOutputFormat;
import org.apache.hama.graph.AbstractAggregator;
import org.apache.hama.graph.AverageAggregator;
@@ -44,34 +45,6 @@ import org.apache.hama.graph.VertexInput
*/
public class PageRank {
- public static class PagerankTextReader extends
- VertexInputReader<LongWritable, Text, Text, NullWritable, DoubleWritable> {
-
- /**
- * The text file essentially should look like: <br/>
- * VERTEX_ID\t(n-tab separated VERTEX_IDs)<br/>
- * E.G:<br/>
- * 1\t2\t3\t4<br/>
- * 2\t3\t1<br/>
- * etc.
- */
- @Override
- public boolean parseVertex(LongWritable key, Text value,
- Vertex<Text, NullWritable, DoubleWritable> vertex) throws Exception {
- String[] split = value.toString().split("\t");
- for (int i = 0; i < split.length; i++) {
- if (i == 0) {
- vertex.setVertexID(new Text(split[i]));
- } else {
- vertex
- .addEdge(new Edge<Text, NullWritable>(new Text(split[i]), null));
- }
- }
- return true;
- }
-
- }
-
public static class PageRankVertex extends
Vertex<Text, NullWritable, DoubleWritable> {
@@ -153,24 +126,44 @@ public class PageRank {
}
- private static void printUsage() {
- System.out.println("Usage: <input> <output> [tasks]");
- System.exit(-1);
+ public static class DanglingNodeAggregator
+ extends
+ AbstractAggregator<DoubleWritable, Vertex<Text, NullWritable, DoubleWritable>> {
+
+ double danglingNodeSum;
+
+ @Override
+ public void aggregate(Vertex<Text, NullWritable, DoubleWritable> vertex,
+ DoubleWritable value) {
+ if (vertex != null) {
+ if (vertex.getEdges().size() == 0) {
+ danglingNodeSum += value.get();
+ }
+ } else {
+ danglingNodeSum += value.get();
+ }
+ }
+
+ @Override
+ public DoubleWritable getValue() {
+ return new DoubleWritable(danglingNodeSum);
+ }
+
}
- public static void main(String[] args) throws IOException,
- InterruptedException, ClassNotFoundException {
- if (args.length < 2)
- printUsage();
+ public static class PagerankSeqReader
+ extends
+ VertexInputReader<Text, TextArrayWritable, Text, NullWritable, DoubleWritable> {
+ @Override
+ public boolean parseVertex(Text key, TextArrayWritable value,
+ Vertex<Text, NullWritable, DoubleWritable> vertex) throws Exception {
+ vertex.setVertexID(key);
- HamaConfiguration conf = new HamaConfiguration(new Configuration());
- GraphJob pageJob = createJob(args, conf);
- pageJob.setVertexInputReaderClass(PagerankTextReader.class);
+ for (Writable v : value.get()) {
+ vertex.addEdge(new Edge<Text, NullWritable>((Text) v, null));
+ }
- long startTime = System.currentTimeMillis();
- if (pageJob.waitForCompletion(true)) {
- System.out.println("Job Finished in "
- + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
+ return true;
}
}
@@ -179,7 +172,7 @@ public class PageRank {
throws IOException {
GraphJob pageJob = new GraphJob(conf, PageRank.class);
pageJob.setJobName("Pagerank");
-
+
pageJob.setVertexClass(PageRankVertex.class);
pageJob.setInputPath(new Path(args[0]));
pageJob.setOutputPath(new Path(args[1]));
@@ -197,6 +190,9 @@ public class PageRank {
pageJob.setAggregatorClass(AverageAggregator.class,
DanglingNodeAggregator.class);
+ // Vertex reader
+ pageJob.setVertexInputReaderClass(PagerankSeqReader.class);
+
pageJob.setVertexIDClass(Text.class);
pageJob.setVertexValueClass(DoubleWritable.class);
pageJob.setEdgeValueClass(NullWritable.class);
@@ -210,28 +206,23 @@ public class PageRank {
return pageJob;
}
- public static class DanglingNodeAggregator
- extends
- AbstractAggregator<DoubleWritable, Vertex<Text, NullWritable, DoubleWritable>> {
+ private static void printUsage() {
+ System.out.println("Usage: <input> <output> [tasks]");
+ System.exit(-1);
+ }
- double danglingNodeSum;
+ public static void main(String[] args) throws IOException,
+ InterruptedException, ClassNotFoundException {
+ if (args.length < 2)
+ printUsage();
- @Override
- public void aggregate(Vertex<Text, NullWritable, DoubleWritable> vertex,
- DoubleWritable value) {
- if (vertex != null) {
- if (vertex.getEdges().size() == 0) {
- danglingNodeSum += value.get();
- }
- } else {
- danglingNodeSum += value.get();
- }
- }
+ HamaConfiguration conf = new HamaConfiguration(new Configuration());
+ GraphJob pageJob = createJob(args, conf);
- @Override
- public DoubleWritable getValue() {
- return new DoubleWritable(danglingNodeSum);
+ long startTime = System.currentTimeMillis();
+ if (pageJob.waitForCompletion(true)) {
+ System.out.println("Job Finished in "
+ + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
}
-
}
}
Modified: hama/trunk/examples/src/test/java/org/apache/hama/examples/PageRankTest.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/test/java/org/apache/hama/examples/PageRankTest.java?rev=1437860&r1=1437859&r2=1437860&view=diff
==============================================================================
--- hama/trunk/examples/src/test/java/org/apache/hama/examples/PageRankTest.java (original)
+++ hama/trunk/examples/src/test/java/org/apache/hama/examples/PageRankTest.java Thu Jan 24 05:20:38 2013
@@ -28,8 +28,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hama.HamaConfiguration;
-import org.apache.hama.examples.PageRank.PageRankVertex;
-import org.apache.hama.examples.util.VertexInputGen;
+import org.apache.hama.examples.util.SymmetricMatrixGen;
import org.apache.hama.graph.GraphJob;
import org.apache.hama.graph.GraphJobRunner;
@@ -87,11 +86,7 @@ public class PageRankTest extends TestCa
private void generateTestData() throws ClassNotFoundException,
InterruptedException, IOException {
- HamaConfiguration conf = new HamaConfiguration();
- conf.setInt(VertexInputGen.SIZE_OF_MATRIX, 40);
- conf.setInt(VertexInputGen.DENSITY, 10);
- conf.setInt("hama.test.vertexcreatorid", 1);
- VertexInputGen.runJob(conf, 3, INPUT, PageRankVertex.class);
+ SymmetricMatrixGen.main(new String[] { "40", "10", INPUT, "2" });
}
private void deleteTempDirs() {
Modified: hama/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/pom.xml?rev=1437860&r1=1437859&r2=1437860&view=diff
==============================================================================
--- hama/trunk/pom.xml (original)
+++ hama/trunk/pom.xml Thu Jan 24 05:20:38 2013
@@ -214,21 +214,6 @@
<version>${zookeeper.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <version>1.6.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc</artifactId>
- <version>1.6.0</version>
- </dependency>
- <dependency>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.2.6.Final</version>
- </dependency>
- <dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<version>1.9.2</version>