You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2012/06/27 10:03:35 UTC
svn commit: r1354358 - in /hama/trunk: core/pom.xml dist/pom.xml
examples/pom.xml graph/pom.xml
graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java ml/pom.xml
pom.xml yarn/pom.xml
Author: edwardyoon
Date: Wed Jun 27 08:03:33 2012
New Revision: 1354358
URL: http://svn.apache.org/viewvc?rev=1354358&view=rev
Log:
To avoid a lot of data exchanges among peers
Modified:
hama/trunk/core/pom.xml
hama/trunk/dist/pom.xml
hama/trunk/examples/pom.xml
hama/trunk/graph/pom.xml
hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java
hama/trunk/ml/pom.xml
hama/trunk/pom.xml
hama/trunk/yarn/pom.xml
Modified: hama/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/core/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/core/pom.xml (original)
+++ hama/trunk/core/pom.xml Wed Jun 27 08:03:33 2012
@@ -20,14 +20,14 @@
<parent>
<groupId>org.apache.hama</groupId>
<artifactId>hama-parent</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hama</groupId>
<artifactId>hama-core</artifactId>
<name>core</name>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
<packaging>jar</packaging>
<dependencies>
Modified: hama/trunk/dist/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/dist/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/dist/pom.xml (original)
+++ hama/trunk/dist/pom.xml Wed Jun 27 08:03:33 2012
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.hama</groupId>
<artifactId>hama-parent</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>hama-dist</artifactId>
Modified: hama/trunk/examples/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/examples/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/examples/pom.xml (original)
+++ hama/trunk/examples/pom.xml Wed Jun 27 08:03:33 2012
@@ -20,14 +20,14 @@
<parent>
<groupId>org.apache.hama</groupId>
<artifactId>hama-parent</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hama</groupId>
<artifactId>hama-examples</artifactId>
<name>examples</name>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
<packaging>jar</packaging>
<dependencies>
Modified: hama/trunk/graph/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/graph/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/graph/pom.xml (original)
+++ hama/trunk/graph/pom.xml Wed Jun 27 08:03:33 2012
@@ -20,14 +20,14 @@
<parent>
<groupId>org.apache.hama</groupId>
<artifactId>hama-parent</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hama</groupId>
<artifactId>hama-graph</artifactId>
<name>graph</name>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
<packaging>jar</packaging>
<dependencies>
Modified: hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java
URL: http://svn.apache.org/viewvc/hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java (original)
+++ hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java Wed Jun 27 08:03:33 2012
@@ -391,16 +391,16 @@ public final class GraphJobRunner<V exte
Vertex<V, E, M> vertex = newVertexInstance(vertexClass, conf);
vertex.setPeer(peer);
vertex.runner = this;
- while (true) {
- KeyValuePair<Writable, Writable> next = peer.readNext();
- if (next == null) {
- break;
- }
+
+ KeyValuePair<Writable, Writable> next = null;
+ int lines = 0;
+ while ((next = peer.readNext()) != null) {
boolean vertexFinished = reader.parseVertex(next.getKey(),
next.getValue(), vertex);
if (!vertexFinished) {
continue;
}
+
if (vertex.getEdges() == null) {
vertex.setEdges(new ArrayList<Edge<V, E>>(0));
}
@@ -420,12 +420,26 @@ public final class GraphJobRunner<V exte
}
peer.send(peer.getPeerName(partition), new GraphJobMessage(vertex));
} else {
+ // FIXME need to set destination names
vertex.setup(conf);
vertices.put(vertex.getVertexID(), vertex);
}
vertex = newVertexInstance(vertexClass, conf);
vertex.setPeer(peer);
vertex.runner = this;
+
+ lines++;
+ if((lines % 50000) == 0) {
+ peer.sync();
+ GraphJobMessage msg = null;
+ while ((msg = peer.getCurrentMessage()) != null) {
+ Vertex<V, E, M> messagedVertex = (Vertex<V, E, M>) msg.getVertex();
+ messagedVertex.setPeer(peer);
+ messagedVertex.runner = this;
+ messagedVertex.setup(conf);
+ vertices.put(messagedVertex.getVertexID(), messagedVertex);
+ }
+ }
}
if (runtimePartitioning) {
@@ -440,6 +454,8 @@ public final class GraphJobRunner<V exte
}
}
+ LOG.info("Loading finished at " + peer.getSuperstepCount() + " steps.");
+
/*
* If the user want to repair the graph, it should traverse through that
* local chunk of adjancency list and message the corresponding peer to
Modified: hama/trunk/ml/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/ml/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/ml/pom.xml (original)
+++ hama/trunk/ml/pom.xml Wed Jun 27 08:03:33 2012
@@ -20,14 +20,14 @@
<parent>
<groupId>org.apache.hama</groupId>
<artifactId>hama-parent</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hama</groupId>
<artifactId>hama-ml</artifactId>
<name>machine learning</name>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
<packaging>jar</packaging>
<dependencies>
Modified: hama/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/pom.xml (original)
+++ hama/trunk/pom.xml Wed Jun 27 08:03:33 2012
@@ -27,7 +27,7 @@
<groupId>org.apache.hama</groupId>
<artifactId>hama-parent</artifactId>
<name>Apache Hama parent POM</name>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
<url>http://hama.apache.org</url>
<packaging>pom</packaging>
<prerequisites>
Modified: hama/trunk/yarn/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/yarn/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/yarn/pom.xml (original)
+++ hama/trunk/yarn/pom.xml Wed Jun 27 08:03:33 2012
@@ -19,14 +19,14 @@
<parent>
<groupId>org.apache.hama</groupId>
<artifactId>hama-parent</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hama</groupId>
<artifactId>hama-yarn</artifactId>
<name>yarn</name>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.5.0-SNAPSHOT</version>
<packaging>jar</packaging>
<dependencies>