You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ap...@apache.org on 2013/03/07 06:37:41 UTC
[8/8] git commit: GIRAPH-528: Decouple vertex implementation from
edge storage (apresta)
Updated Branches:
refs/heads/trunk 8cdcf541e -> 3f5009aea
GIRAPH-528: Decouple vertex implementation from edge storage (apresta)
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/3f5009ae
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/3f5009ae
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/3f5009ae
Branch: refs/heads/trunk
Commit: 3f5009aea0797563304dd8ba1aa6fb06ebaab83b
Parents: 8cdcf54
Author: Alessandro Presta <al...@fb.com>
Authored: Mon Feb 25 16:12:50 2013 -0800
Committer: Alessandro Presta <al...@fb.com>
Committed: Wed Mar 6 21:37:08 2013 -0800
----------------------------------------------------------------------
CHANGELOG | 2 +
.../io/accumulo/TestAccumuloVertexFormat.java | 6 +-
.../edgemarker/AccumuloEdgeInputFormat.java | 9 +-
.../edgemarker/AccumuloEdgeOutputFormat.java | 2 +-
giraph-core/pom.xml | 6 +-
.../giraph/benchmark/AggregatorsBenchmark.java | 9 +-
.../ByteArrayVertexPageRankBenchmark.java | 39 --
.../benchmark/EdgeListVertexPageRankBenchmark.java | 38 --
.../benchmark/HashMapVertexPageRankBenchmark.java | 38 --
.../HashMapVertexShortestPathsBenchmark.java | 39 --
...MultiGraphByteArrayVertexPageRankBenchmark.java | 39 --
.../MultiGraphEdgeListVertexPageRankBenchmark.java | 38 --
...GraphRepresentativeVertexPageRankBenchmark.java | 39 --
.../apache/giraph/benchmark/PageRankBenchmark.java | 90 ++--
.../giraph/benchmark/PageRankComputation.java | 65 --
.../apache/giraph/benchmark/PageRankVertex.java | 53 ++
.../giraph/benchmark/RandomMessageBenchmark.java | 10 +-
.../RepresentativeVertexPageRankBenchmark.java | 39 --
.../giraph/benchmark/ShortestPathsBenchmark.java | 34 +-
.../giraph/benchmark/ShortestPathsComputation.java | 81 ---
.../giraph/benchmark/ShortestPathsVertex.java | 64 ++
.../org/apache/giraph/bsp/BspOutputFormat.java | 21 +-
.../main/java/org/apache/giraph/bsp/BspUtils.java | 397 ------------
.../main/java/org/apache/giraph/comm/MsgList.java | 56 --
.../java/org/apache/giraph/comm/SendEdgeCache.java | 2 +-
.../org/apache/giraph/comm/SendMutationsCache.java | 4 +-
.../org/apache/giraph/comm/SendPartitionCache.java | 2 +-
.../java/org/apache/giraph/comm/ServerData.java | 2 +-
.../giraph/comm/WorkerClientRequestProcessor.java | 4 +-
.../DiskBackedMessageStoreByPartition.java | 13 +-
.../comm/messages/SequentialFileMessageStore.java | 2 +-
.../netty/NettyWorkerClientRequestProcessor.java | 4 +-
.../giraph/comm/netty/NettyWorkerServer.java | 2 +-
.../comm/requests/SendWorkerEdgesRequest.java | 2 +-
.../java/org/apache/giraph/conf/GiraphClasses.java | 73 ++-
.../apache/giraph/conf/GiraphConfiguration.java | 13 +-
.../org/apache/giraph/conf/GiraphConstants.java | 2 +
.../conf/ImmutableClassesGiraphConfiguration.java | 102 +++-
.../org/apache/giraph/edge/ArrayListEdges.java | 112 ++++
.../org/apache/giraph/edge/ByteArrayEdges.java | 194 ++++++
.../giraph/edge/ConfigurableVertexEdges.java | 49 ++
.../java/org/apache/giraph/edge/DefaultEdge.java | 80 +++
.../src/main/java/org/apache/giraph/edge/Edge.java | 45 ++
.../java/org/apache/giraph/edge/EdgeFactory.java | 89 +++
.../java/org/apache/giraph/edge/EdgeNoValue.java | 70 +++
.../java/org/apache/giraph/edge/EdgeStore.java | 172 +++++
.../java/org/apache/giraph/edge/HashMapEdges.java | 142 +++++
.../org/apache/giraph/edge/HashMultimapEdges.java | 159 +++++
.../apache/giraph/edge/LongDoubleArrayEdges.java | 178 ++++++
.../apache/giraph/edge/LongDoubleHashMapEdges.java | 148 +++++
.../org/apache/giraph/edge/LongNullArrayEdges.java | 163 +++++
.../apache/giraph/edge/LongNullHashSetEdges.java | 135 ++++
.../giraph/edge/MultiRandomAccessVertexEdges.java | 41 ++
.../java/org/apache/giraph/edge/MutableEdge.java | 47 ++
.../giraph/edge/ReuseObjectsVertexEdges.java | 34 +
.../giraph/edge/StrictRandomAccessVertexEdges.java | 42 ++
.../java/org/apache/giraph/edge/VertexEdges.java | 84 +++
.../java/org/apache/giraph/edge/package-info.java | 21 +
.../org/apache/giraph/graph/ComputeCallable.java | 4 +-
.../java/org/apache/giraph/graph/DefaultEdge.java | 107 ----
.../apache/giraph/graph/DefaultVertexResolver.java | 9 +-
.../main/java/org/apache/giraph/graph/Edge.java | 46 --
.../java/org/apache/giraph/graph/EdgeFactory.java | 89 ---
.../java/org/apache/giraph/graph/EdgeNoValue.java | 90 ---
.../java/org/apache/giraph/graph/EdgeStore.java | 176 ------
.../giraph/graph/GiraphTransferRegulator.java | 1 -
.../org/apache/giraph/graph/GraphTaskManager.java | 11 +-
.../java/org/apache/giraph/graph/MutableEdge.java | 47 --
.../apache/giraph/graph/ReverseEdgeDuplicator.java | 115 ----
.../main/java/org/apache/giraph/graph/Vertex.java | 480 +++++++++++++++
.../org/apache/giraph/graph/VertexChanges.java | 2 +-
.../org/apache/giraph/graph/VertexMutations.java | 2 +-
.../org/apache/giraph/graph/VertexResolver.java | 1 -
.../main/java/org/apache/giraph/io/EdgeReader.java | 2 +-
.../apache/giraph/io/ReverseEdgeDuplicator.java | 116 ++++
.../java/org/apache/giraph/io/VertexReader.java | 2 +-
.../org/apache/giraph/io/VertexValueReader.java | 2 +-
.../java/org/apache/giraph/io/VertexWriter.java | 2 +-
.../AdjacencyListTextVertexInputFormat.java | 5 +-
.../AdjacencyListTextVertexOutputFormat.java | 4 +-
.../io/formats/IdWithValueTextOutputFormat.java | 2 +-
.../io/formats/IntIntNullIntTextInputFormat.java | 7 +-
.../io/formats/IntNullNullNullTextInputFormat.java | 5 +-
.../formats/IntNullReverseTextEdgeInputFormat.java | 2 +-
.../io/formats/JsonBase64VertexInputFormat.java | 9 +-
.../io/formats/JsonBase64VertexOutputFormat.java | 7 +-
...JsonLongDoubleFloatDoubleVertexInputFormat.java | 9 +-
...sonLongDoubleFloatDoubleVertexOutputFormat.java | 4 +-
...DoubleDoubleAdjacencyListVertexInputFormat.java | 4 +-
.../io/formats/PseudoRandomEdgeInputFormat.java | 15 +-
.../io/formats/PseudoRandomVertexInputFormat.java | 17 +-
.../io/formats/SequenceFileVertexInputFormat.java | 2 +-
.../io/formats/SequenceFileVertexOutputFormat.java | 2 +-
...DoubleDoubleAdjacencyListVertexInputFormat.java | 4 +-
.../giraph/io/formats/TextEdgeInputFormat.java | 4 +-
.../giraph/io/formats/TextVertexInputFormat.java | 4 +-
.../giraph/io/formats/TextVertexOutputFormat.java | 2 +-
.../giraph/job/GiraphConfigurationValidator.java | 62 ++-
.../giraph/partition/ByteArrayPartition.java | 2 +-
.../giraph/partition/DiskBackedPartitionStore.java | 2 +-
.../org/apache/giraph/partition/Partition.java | 2 +-
.../apache/giraph/partition/SimplePartition.java | 2 +-
.../org/apache/giraph/utils/ByteArrayEdges.java | 290 ---------
.../apache/giraph/utils/ByteArrayVertexIdData.java | 11 +
.../giraph/utils/ByteArrayVertexIdEdges.java | 16 +-
.../giraph/utils/ByteArrayVertexIdMessages.java | 5 +
.../org/apache/giraph/utils/ComparisonUtils.java | 62 --
.../apache/giraph/utils/ConfigurationUtils.java | 17 +-
.../org/apache/giraph/utils/EdgeComparator.java | 70 +++
.../org/apache/giraph/utils/EdgeIterables.java | 104 ++-
.../org/apache/giraph/utils/EmptyIterable.java | 37 +-
.../giraph/utils/ExtendedByteArrayDataOutput.java | 3 +-
.../apache/giraph/utils/InternalVertexRunner.java | 2 +
.../org/apache/giraph/utils/ReflectionUtils.java | 15 +-
.../utils/UnmodifiableDoubleArrayIterator.java | 53 --
.../giraph/utils/UnmodifiableIntArrayIterator.java | 53 --
.../utils/UnmodifiableLongArrayIterator.java | 53 --
.../UnmodifiableLongFloatEdgeArrayIterable.java | 73 ---
.../UnmodifiableLongNullEdgeArrayIterable.java | 67 --
.../giraph/utils/UnsafeByteArrayOutputStream.java | 2 +-
.../org/apache/giraph/utils/WritableUtils.java | 2 +-
.../org/apache/giraph/vertex/ByteArrayVertex.java | 63 --
.../apache/giraph/vertex/ByteArrayVertexBase.java | 126 ----
.../org/apache/giraph/vertex/EdgeListVertex.java | 73 ---
.../apache/giraph/vertex/EdgeListVertexBase.java | 109 ----
.../org/apache/giraph/vertex/HashMapVertex.java | 163 -----
.../apache/giraph/vertex/IntIntNullIntVertex.java | 100 ---
.../giraph/vertex/IntNullNullNullVertex.java | 61 --
.../LongDoubleFloatDoubleEdgeListVertex.java | 184 ------
.../giraph/vertex/LongDoubleFloatDoubleVertex.java | 219 -------
.../giraph/vertex/LongDoubleNullDoubleVertex.java | 139 -----
.../giraph/vertex/MultiGraphByteArrayVertex.java | 49 --
.../giraph/vertex/MultiGraphEdgeListVertex.java | 62 --
.../vertex/MultiGraphRepresentativeVertex.java | 49 --
.../org/apache/giraph/vertex/MutableVertex.java | 119 ----
.../apache/giraph/vertex/RepresentativeVertex.java | 69 --
.../giraph/vertex/RepresentativeVertexBase.java | 293 ---------
.../apache/giraph/vertex/SimpleMutableVertex.java | 132 ----
.../org/apache/giraph/vertex/SimpleVertex.java | 106 ----
.../main/java/org/apache/giraph/vertex/Vertex.java | 422 -------------
.../org/apache/giraph/vertex/package-info.java | 21 -
.../org/apache/giraph/worker/BspServiceWorker.java | 2 +-
.../apache/giraph/worker/DefaultWorkerContext.java | 2 +-
.../giraph/worker/EdgeInputSplitsCallable.java | 2 +-
.../giraph/worker/VertexInputSplitsCallable.java | 2 +-
.../src/test/java/org/apache/giraph/BspCase.java | 10 +-
.../java/org/apache/giraph/bsp/BspUtilsTest.java | 195 ------
.../org/apache/giraph/comm/ConnectionTest.java | 21 +-
.../org/apache/giraph/comm/RequestFailureTest.java | 11 +-
.../java/org/apache/giraph/comm/RequestTest.java | 20 +-
.../org/apache/giraph/comm/SaslConnectionTest.java | 11 +-
.../org/apache/giraph/comm/TestMessageStores.java | 41 +-
.../org/apache/giraph/conf/TestObjectCreation.java | 191 ++++++
.../apache/giraph/edge/TestMultiGraphEdges.java | 86 +++
.../giraph/edge/TestMultiRandomAccessEdges.java | 81 +++
.../org/apache/giraph/edge/TestNullValueEdges.java | 74 +++
.../apache/giraph/edge/TestStrictGraphEdges.java | 79 +++
.../giraph/edge/TestStrictRandomAccessEdges.java | 78 +++
.../apache/giraph/graph/TestVertexAndEdges.java | 348 +++++++++++
.../TestAdjacencyListTextVertexOutputFormat.java | 12 +-
.../java/org/apache/giraph/io/TestEdgeInput.java | 18 +-
.../giraph/io/TestIdWithValueTextOutputFormat.java | 11 +-
.../org/apache/giraph/io/TestJsonBase64Format.java | 20 +-
...DoubleDoubleAdjacencyListVertexInputFormat.java | 24 +-
...DoubleDoubleAdjacencyListVertexInputFormat.java | 67 +--
.../apache/giraph/master/TestMasterObserver.java | 11 +-
.../partition/TestGiraphTransferRegulator.java | 27 +-
.../giraph/partition/TestPartitionStores.java | 58 +-
.../apache/giraph/utils/ComparisonUtilsTest.java | 75 ---
.../java/org/apache/giraph/utils/MockUtils.java | 17 +-
.../giraph/vertex/TestIntIntNullIntVertex.java | 78 ---
.../apache/giraph/vertex/TestMultiGraphVertex.java | 122 ----
.../apache/giraph/vertex/TestMutableVertex.java | 476 --------------
giraph-examples/pom.xml | 6 +-
.../giraph/examples/AggregatorsTestVertex.java | 4 +-
.../giraph/examples/ConnectedComponentsVertex.java | 19 +-
.../org/apache/giraph/examples/IdentityVertex.java | 4 +-
.../LongDoubleDoubleDoubleTextInputFormat.java | 108 ++++
.../LongDoubleFloatDoubleTextInputFormat.java | 93 ---
...izingLongDoubleDoubleDoubleTextInputFormat.java | 139 +++++
...lizingLongDoubleFloatDoubleTextInputFormat.java | 125 ----
.../examples/PartitionContextTestVertex.java | 4 +-
.../apache/giraph/examples/RandomWalkVertex.java | 12 +-
.../giraph/examples/SimpleCheckpointVertex.java | 8 +-
.../giraph/examples/SimpleCombinerVertex.java | 4 +-
.../apache/giraph/examples/SimpleFailVertex.java | 4 +-
.../giraph/examples/SimpleInDegreeCountVertex.java | 6 +-
.../giraph/examples/SimpleMasterComputeVertex.java | 7 +-
.../apache/giraph/examples/SimpleMsgVertex.java | 4 +-
.../giraph/examples/SimpleMutateGraphVertex.java | 6 +-
.../examples/SimpleOutDegreeCountVertex.java | 4 +-
.../giraph/examples/SimplePageRankVertex.java | 13 +-
.../giraph/examples/SimpleShortestPathsVertex.java | 10 +-
.../giraph/examples/SimpleSuperstepVertex.java | 14 +-
.../examples/SimpleTextVertexOutputFormat.java | 2 +-
.../examples/SimpleTriangleClosingVertex.java | 6 +-
.../examples/SimpleVertexWithWorkerContext.java | 4 +-
.../org/apache/giraph/examples/VerifyMessage.java | 8 +-
...xWithDoubleValueDoubleEdgeTextOutputFormat.java | 56 ++
...exWithDoubleValueFloatEdgeTextOutputFormat.java | 57 --
.../test/java/org/apache/giraph/TestBspBasic.java | 11 +-
.../examples/ConnectedComponentsVertexTest.java | 13 +-
.../examples/RandomWalkWithRestartVertexTest.java | 21 +-
.../examples/SimpleShortestPathsVertexTest.java | 35 +-
.../examples/SimpleTriangleClosingVertexTest.java | 16 +-
.../examples/TryMultiIpcBindingPortsTest.java | 2 +
.../org/apache/giraph/vertex/TestVertexTypes.java | 37 +-
.../io/hbase/TestHBaseRootMarkerVertextFormat.java | 7 +-
.../io/hbase/edgemarker/TableEdgeInputFormat.java | 9 +-
.../io/hbase/edgemarker/TableEdgeOutputFormat.java | 2 +-
.../giraph/io/hcatalog/HCatGiraphRunner.java | 2 +-
.../io/hcatalog/HCatalogEdgeInputFormat.java | 4 +-
.../io/hcatalog/HCatalogVertexInputFormat.java | 9 +-
.../io/hcatalog/HCatalogVertexOutputFormat.java | 2 +-
.../org/apache/giraph/hive/HiveGiraphRunner.java | 2 +-
.../giraph/hive/input/edge/HiveEdgeReader.java | 11 +-
.../giraph/hive/input/vertex/HiveToVertex.java | 2 +-
.../giraph/hive/input/vertex/HiveVertexReader.java | 2 +-
.../giraph/hive/output/HiveVertexWriter.java | 2 +-
.../apache/giraph/hive/output/VertexToHive.java | 2 +-
pom.xml | 10 +-
221 files changed, 4914 insertions(+), 6706 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index d93bf55..7f5a113 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
Giraph Change Log
Release 0.2.0 - unreleased
+ GIRAPH-528: Decouple vertex implementation from edge storage (apresta)
+
GIRAPH-553: Cleanup HCatalogVertexOutputFormat (majakabiljo)
GIRAPH-545: Improve Facebook Hadoop dependency (nitay)
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/TestAccumuloVertexFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/TestAccumuloVertexFormat.java b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/TestAccumuloVertexFormat.java
index 8894199..6698c9a 100644
--- a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/TestAccumuloVertexFormat.java
+++ b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/TestAccumuloVertexFormat.java
@@ -35,18 +35,18 @@ import org.apache.giraph.BspCase;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.io.accumulo.edgemarker.AccumuloEdgeInputFormat;
import org.apache.giraph.io.accumulo.edgemarker.AccumuloEdgeOutputFormat;
-import org.apache.giraph.vertex.EdgeListVertex;
import org.apache.giraph.job.GiraphJob;
+import org.apache.giraph.graph.Vertex;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
+import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Map;
-import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -170,7 +170,7 @@ public class TestAccumuloVertexFormat extends BspCase{
The test set only has a 1-1 parent-to-child ratio for this unit test.
*/
public static class EdgeNotification
- extends EdgeListVertex<Text, Text, Text, Text> {
+ extends Vertex<Text, Text, Text, Text> {
@Override
public void compute(Iterable<Text> messages) throws IOException {
for (Text message : messages) {
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
index 30d40f6..4cbbc89 100644
--- a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
+++ b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
@@ -17,21 +17,20 @@
*/
package org.apache.giraph.io.accumulo.edgemarker;
+import com.google.common.collect.Lists;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
-import org.apache.giraph.graph.Edge;
-import org.apache.giraph.graph.EdgeFactory;
import org.apache.giraph.io.VertexReader;
import org.apache.giraph.io.accumulo.AccumuloVertexInputFormat;
-import org.apache.giraph.vertex.Vertex;
+import org.apache.giraph.edge.Edge;
+import org.apache.giraph.edge.EdgeFactory;
+import org.apache.giraph.graph.Vertex;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import com.google.common.collect.Lists;
-
import java.io.IOException;
import java.util.List;
import java.util.regex.Pattern;
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
index 70288f6..0937b84 100644
--- a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
+++ b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
@@ -20,7 +20,7 @@ package org.apache.giraph.io.accumulo.edgemarker;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.giraph.io.accumulo.AccumuloVertexOutputFormat;
-import org.apache.giraph.vertex.Vertex;
+import org.apache.giraph.graph.Vertex;
import org.apache.giraph.io.VertexWriter;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-core/pom.xml b/giraph-core/pom.xml
index 5ce3eaa..2a500d8 100644
--- a/giraph-core/pom.xml
+++ b/giraph-core/pom.xml
@@ -69,7 +69,7 @@ under the License.
<configLocation>checkstyle.xml</configLocation>
<enableRulesSummary>false</enableRulesSummary>
<headerLocation>license-header.txt</headerLocation>
- <failOnError>true</failOnError>
+ <failOnViolation>true</failOnViolation>
<includeTestSourceDirectory>false</includeTestSourceDirectory>
</configuration>
<executions>
@@ -291,8 +291,8 @@ under the License.
<artifactId>base64</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.mahout</groupId>
- <artifactId>mahout-collections</artifactId>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil</artifactId>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/AggregatorsBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/AggregatorsBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/AggregatorsBenchmark.java
index 4e47042..7043338 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/AggregatorsBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/AggregatorsBenchmark.java
@@ -25,11 +25,11 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.giraph.aggregators.LongSumAggregator;
import org.apache.giraph.io.formats.PseudoRandomInputFormatConstants;
+import org.apache.giraph.io.formats.PseudoRandomVertexInputFormat;
+import org.apache.giraph.job.GiraphJob;
import org.apache.giraph.master.DefaultMasterCompute;
+import org.apache.giraph.graph.Vertex;
import org.apache.giraph.worker.DefaultWorkerContext;
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.giraph.job.GiraphJob;
-import org.apache.giraph.io.formats.PseudoRandomVertexInputFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
@@ -55,8 +55,7 @@ public class AggregatorsBenchmark implements Tool {
* Vertex class for AggregatorsBenchmark
*/
public static class AggregatorsBenchmarkVertex extends
- EdgeListVertex<LongWritable, DoubleWritable, DoubleWritable,
- DoubleWritable> {
+ Vertex<LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> {
@Override
public void compute(Iterable<DoubleWritable> messages) throws IOException {
int n = getNumAggregators(getConf());
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/ByteArrayVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/ByteArrayVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/ByteArrayVertexPageRankBenchmark.java
deleted file mode 100644
index 7e51c26..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/ByteArrayVertexPageRankBenchmark.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.benchmark;
-
-import java.io.IOException;
-import org.apache.giraph.vertex.ByteArrayVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * Same benchmark code as {@link PageRankBenchmark}, but uses
- * {@link org.apache.giraph.vertex.ByteArrayVertex}
- * implementation.
- */
-public class ByteArrayVertexPageRankBenchmark extends
- ByteArrayVertex<LongWritable, DoubleWritable,
- DoubleWritable, DoubleWritable> {
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws
- IOException {
- PageRankComputation.computePageRank(this, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/EdgeListVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/EdgeListVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/EdgeListVertexPageRankBenchmark.java
deleted file mode 100644
index fe1e346..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/EdgeListVertexPageRankBenchmark.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.benchmark;
-
-import java.io.IOException;
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * Same benchmark code as {@link PageRankBenchmark}, but uses
- * {@link org.apache.giraph.vertex.EdgeListVertex} implementation rather than
- * {@link org.apache.giraph.vertex.HashMapVertex}
- */
-public class EdgeListVertexPageRankBenchmark extends EdgeListVertex<
- LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> {
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws
- IOException {
- PageRankComputation.computePageRank(this, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java
deleted file mode 100644
index ef37540..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.benchmark;
-
-import org.apache.giraph.vertex.HashMapVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.io.IOException;
-
-/**
- * Same benchmark code as {@link PageRankBenchmark}, but uses
- * {@link HashMapVertex} implementation rather than
- * {@link org.apache.giraph.vertex.EdgeListVertex}
- */
-public class HashMapVertexPageRankBenchmark extends HashMapVertex<
- LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> {
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws
- IOException {
- PageRankComputation.computePageRank(this, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexShortestPathsBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexShortestPathsBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexShortestPathsBenchmark.java
deleted file mode 100644
index a9d6deb..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexShortestPathsBenchmark.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.benchmark;
-
-import org.apache.giraph.vertex.HashMapVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.io.IOException;
-
-/**
- * Same benchmark code as {@link ShortestPathsBenchmark}, but uses
- * {@link HashMapVertex} implementation rather than
- * {@link org.apache.giraph.vertex.EdgeListVertex}
- */
-public class HashMapVertexShortestPathsBenchmark extends HashMapVertex<
- LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> {
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws
- IOException {
- ShortestPathsComputation.computeShortestPaths(this, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphByteArrayVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphByteArrayVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphByteArrayVertexPageRankBenchmark.java
deleted file mode 100644
index 9144641..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphByteArrayVertexPageRankBenchmark.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.benchmark;
-
-import org.apache.giraph.vertex.MultiGraphByteArrayVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.io.IOException;
-
-/**
- * Vertex for PageRank benchmark based on {@link
- * org.apache.giraph.vertex.MultiGraphByteArrayVertex}
- */
-public class MultiGraphByteArrayVertexPageRankBenchmark
- extends MultiGraphByteArrayVertex<LongWritable, DoubleWritable,
- DoubleWritable, DoubleWritable> {
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws
- IOException {
- PageRankComputation.computePageRank(this, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphEdgeListVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphEdgeListVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphEdgeListVertexPageRankBenchmark.java
deleted file mode 100644
index 712ca99..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphEdgeListVertexPageRankBenchmark.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.benchmark;
-
-import org.apache.giraph.vertex.MultiGraphEdgeListVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.io.IOException;
-
-/**
- * Vertex for PageRank benchmark based on {@link MultiGraphEdgeListVertex}
- */
-public class MultiGraphEdgeListVertexPageRankBenchmark
- extends MultiGraphEdgeListVertex<LongWritable, DoubleWritable,
- DoubleWritable, DoubleWritable> {
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws
- IOException {
- PageRankComputation.computePageRank(this, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphRepresentativeVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphRepresentativeVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphRepresentativeVertexPageRankBenchmark.java
deleted file mode 100644
index 9628832..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphRepresentativeVertexPageRankBenchmark.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.benchmark;
-
-import org.apache.giraph.vertex.MultiGraphRepresentativeVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.io.IOException;
-
-/**
- * Vertex for PageRank benchmark based on {@link
- * MultiGraphRepresentativeVertex}
- */
-public class MultiGraphRepresentativeVertexPageRankBenchmark
- extends MultiGraphRepresentativeVertex<LongWritable, DoubleWritable,
- DoubleWritable, DoubleWritable> {
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws
- IOException {
- PageRankComputation.computePageRank(this, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
index 06ee80c..2902fa9 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
@@ -25,6 +25,10 @@ import org.apache.commons.cli.PosixParser;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.combiner.DoubleSumCombiner;
+import org.apache.giraph.edge.ArrayListEdges;
+import org.apache.giraph.edge.ByteArrayEdges;
+import org.apache.giraph.edge.HashMapEdges;
+import org.apache.giraph.edge.LongDoubleArrayEdges;
import org.apache.giraph.io.formats.PseudoRandomInputFormatConstants;
import org.apache.giraph.job.GiraphJob;
import org.apache.giraph.io.formats.JsonBase64VertexOutputFormat;
@@ -81,17 +85,17 @@ public class PageRankBenchmark implements Tool {
true,
"Edges per vertex");
options.addOption("c",
- "vertexClass",
+ "edgesClass",
true,
- "Vertex class (0 for HashMapVertex, 1 for EdgeListVertex, " +
- "2 for ByteArrayVertex, " +
- "3 for ByteArrayVertex with unsafe, " +
- "4 for HashMapVertex (using EdgeInputFormat), " +
- "5 for MultiGraphEdgeListVertex (using EdgeInputFormat), " +
- "6 for MultiGraphByteArrayVertex (using " +
- "EdgeInputFormat), " +
- "7 for MultiGraphByteArrayVertex with unsafe (using " +
- "EdgeInputFormat))");
+ "Vertex edges class (0 for LongDoubleArrayEdges," +
+ "1 for ByteArrayEdges, " +
+ "2 for ByteArrayEdges with unsafe serialization, " +
+ "3 for ArrayListEdges, " +
+ "4 for HashMapVertex");
+ options.addOption("ei",
+ "edgeInput",
+ false,
+ "Use edge-based input instead of vertex-based input.");
options.addOption("l",
"localEdgesMinRatio",
true,
@@ -155,7 +159,7 @@ public class PageRankBenchmark implements Tool {
configuration.setWorkerConfiguration(workers, workers, 100.0f);
configuration.setInt(
- PageRankComputation.SUPERSTEP_COUNT,
+ PageRankVertex.SUPERSTEP_COUNT,
Integer.parseInt(cmd.getOptionValue('s')));
boolean isVerbose = false;
@@ -170,7 +174,7 @@ public class PageRankBenchmark implements Tool {
}
/**
- * Set vertex, input format, partitioner classes and related parameters
+ * Set vertex edges, input format, partitioner classes and related parameters
* based on command-line arguments.
*
* @param cmd Command line arguments
@@ -178,48 +182,46 @@ public class PageRankBenchmark implements Tool {
*/
protected void setClassesAndParameters(
CommandLine cmd, GiraphConfiguration configuration) {
- int vertexClassOption = cmd.hasOption('c') ? Integer.parseInt(
+ configuration.setVertexClass(PageRankVertex.class);
+ int edgesClassOption = cmd.hasOption('c') ? Integer.parseInt(
cmd.getOptionValue('c')) : 1;
- if (vertexClassOption == 1) {
- configuration.setVertexClass(
- EdgeListVertexPageRankBenchmark.class);
- } else if (vertexClassOption == 0 || vertexClassOption == 4) {
- configuration.setVertexClass(
- HashMapVertexPageRankBenchmark.class);
- } else if (vertexClassOption == 2) {
- configuration.setVertexClass(
- ByteArrayVertexPageRankBenchmark.class);
- configuration.useUnsafeSerialization(false);
- } else if (vertexClassOption == 3) {
- configuration.setVertexClass(
- ByteArrayVertexPageRankBenchmark.class);
- configuration.useUnsafeSerialization(true);
- } else if (vertexClassOption == 5) {
- configuration.setVertexClass(
- MultiGraphEdgeListVertexPageRankBenchmark.class);
- } else if (vertexClassOption == 6) {
- configuration.setVertexClass(
- MultiGraphByteArrayVertexPageRankBenchmark.class);
- configuration.useUnsafeSerialization(false);
- } else if (vertexClassOption == 7) {
- configuration.setVertexClass(
- MultiGraphByteArrayVertexPageRankBenchmark.class);
+ switch (edgesClassOption) {
+ case 0:
+ configuration.setVertexEdgesClass(LongDoubleArrayEdges.class);
+ break;
+ case 1:
+ configuration.setVertexEdgesClass(ByteArrayEdges.class);
+ break;
+ case 2:
+ configuration.setVertexEdgesClass(ByteArrayEdges.class);
configuration.useUnsafeSerialization(true);
+ break;
+ case 3:
+ configuration.setVertexEdgesClass(ArrayListEdges.class);
+ break;
+ case 4:
+ configuration.setVertexEdgesClass(HashMapEdges.class);
+ break;
+ default:
+ LOG.info("Unknown VertexEdges class, " +
+ "defaulting to LongDoubleArrayEdges");
+ configuration.setVertexEdgesClass(LongDoubleArrayEdges.class);
}
- LOG.info("Using vertex class " +
- configuration.get(GiraphConstants.VERTEX_CLASS));
+
+ LOG.info("Using edges class " +
+ configuration.get(GiraphConstants.VERTEX_EDGES_CLASS));
if (!cmd.hasOption('t') ||
(Integer.parseInt(cmd.getOptionValue('t')) == 1)) {
- configuration.setVertexCombinerClass(
- DoubleSumCombiner.class);
+ configuration.setVertexCombinerClass(DoubleSumCombiner.class);
}
- if (vertexClassOption <= 3) {
+ if (cmd.hasOption("ei")) {
+ configuration.setEdgeInputFormatClass(PseudoRandomEdgeInputFormat.class);
+ } else {
configuration.setVertexInputFormatClass(
PseudoRandomVertexInputFormat.class);
- } else {
- configuration.setEdgeInputFormatClass(PseudoRandomEdgeInputFormat.class);
}
+
configuration.setLong(
PseudoRandomInputFormatConstants.AGGREGATE_VERTICES,
Long.parseLong(cmd.getOptionValue('V')));
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java
deleted file mode 100644
index 57cc201..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.benchmark;
-
-import org.apache.giraph.vertex.MutableVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * Shared computation of class Pregel-style PageRank computation for benchmark
- * classes.
- */
-public class PageRankComputation {
- /** Number of supersteps */
- public static final String SUPERSTEP_COUNT =
- "PageRankBenchmark.superstepCount";
-
- /**
- * Do not construct.
- */
- private PageRankComputation() { }
-
- /**
- * Generic page rank algorithm.
- *
- * @param vertex Vertex to compute on.
- * @param messages Iterator of messages from previous superstep.
- */
- public static void computePageRank(
- MutableVertex<LongWritable, DoubleWritable, DoubleWritable,
- DoubleWritable> vertex, Iterable<DoubleWritable> messages) {
- if (vertex.getSuperstep() >= 1) {
- double sum = 0;
- for (DoubleWritable message : messages) {
- sum += message.get();
- }
- DoubleWritable vertexValue = new DoubleWritable(
- (0.15f / vertex.getTotalNumVertices()) + 0.85f * sum);
- vertex.setValue(vertexValue);
- }
-
- if (vertex.getSuperstep() < vertex.getConf().getInt(SUPERSTEP_COUNT, -1)) {
- long edges = vertex.getNumEdges();
- vertex.sendMessageToAllEdges(
- new DoubleWritable(vertex.getValue().get() / edges));
- } else {
- vertex.voteToHalt();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankVertex.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankVertex.java
new file mode 100644
index 0000000..9c3b9f5
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankVertex.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.benchmark;
+
+import org.apache.giraph.graph.Vertex;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.LongWritable;
+
+import java.io.IOException;
+
+/**
+ * PageRank algorithm.
+ */
+public class PageRankVertex extends Vertex<LongWritable, DoubleWritable,
+ DoubleWritable, DoubleWritable> {
+ /** Number of supersteps */
+ public static final String SUPERSTEP_COUNT =
+ "PageRankBenchmark.superstepCount";
+
+ @Override
+ public void compute(Iterable<DoubleWritable> messages) throws IOException {
+ if (getSuperstep() >= 1) {
+ double sum = 0;
+ for (DoubleWritable message : messages) {
+ sum += message.get();
+ }
+ getValue().set((0.15f / getTotalNumVertices()) + 0.85f * sum);
+ }
+
+ if (getSuperstep() < getConf().getInt(SUPERSTEP_COUNT, 0)) {
+ long edges = getNumEdges();
+ sendMessageToAllEdges(new DoubleWritable(getValue().get() / edges));
+ } else {
+ voteToHalt();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/RandomMessageBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/RandomMessageBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/RandomMessageBenchmark.java
index c8e33dd..aa20cca 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/RandomMessageBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/RandomMessageBenchmark.java
@@ -26,11 +26,11 @@ import org.apache.commons.cli.PosixParser;
import org.apache.giraph.aggregators.LongSumAggregator;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.io.formats.PseudoRandomInputFormatConstants;
-import org.apache.giraph.master.DefaultMasterCompute;
-import org.apache.giraph.vertex.EdgeListVertex;
+import org.apache.giraph.io.formats.PseudoRandomVertexInputFormat;
import org.apache.giraph.job.GiraphJob;
+import org.apache.giraph.master.DefaultMasterCompute;
+import org.apache.giraph.graph.Vertex;
import org.apache.giraph.worker.WorkerContext;
-import org.apache.giraph.io.formats.PseudoRandomVertexInputFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DoubleWritable;
@@ -249,8 +249,8 @@ public class RandomMessageBenchmark implements Tool {
/**
* Actual message computation (messaging in this case)
*/
- public static class RandomMessageVertex extends EdgeListVertex<
- LongWritable, DoubleWritable, DoubleWritable, BytesWritable> {
+ public static class RandomMessageVertex extends Vertex<LongWritable,
+ DoubleWritable, DoubleWritable, BytesWritable> {
@Override
public void compute(Iterable<BytesWritable> messages) {
RandomMessageBenchmarkWorkerContext workerContext =
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/RepresentativeVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/RepresentativeVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/RepresentativeVertexPageRankBenchmark.java
deleted file mode 100644
index 331ae41..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/RepresentativeVertexPageRankBenchmark.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.benchmark;
-
-import java.io.IOException;
-import org.apache.giraph.vertex.RepresentativeVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * Same benchmark code as {@link PageRankBenchmark}, but uses
- * {@link org.apache.giraph.vertex.RepresentativeVertex}
- * implementation.
- */
-public class RepresentativeVertexPageRankBenchmark extends
- RepresentativeVertex<LongWritable, DoubleWritable,
- DoubleWritable, DoubleWritable> {
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws
- IOException {
- PageRankComputation.computePageRank(this, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsBenchmark.java
index 1843da9..1753f4f 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsBenchmark.java
@@ -23,21 +23,18 @@ import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
-import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.combiner.MinimumDoubleCombiner;
+import org.apache.giraph.conf.GiraphConstants;
+import org.apache.giraph.edge.ArrayListEdges;
+import org.apache.giraph.edge.HashMapEdges;
import org.apache.giraph.io.formats.PseudoRandomInputFormatConstants;
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.giraph.job.GiraphJob;
import org.apache.giraph.io.formats.PseudoRandomVertexInputFormat;
+import org.apache.giraph.job.GiraphJob;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.Logger;
-import java.io.IOException;
-
/**
* Single-source shortest paths benchmark.
*/
@@ -48,19 +45,6 @@ public class ShortestPathsBenchmark implements Tool {
/** Configuration */
private Configuration conf;
- /**
- * Vertex implementation
- */
- public static class ShortestPathsBenchmarkVertex extends
- EdgeListVertex<LongWritable, DoubleWritable, DoubleWritable,
- DoubleWritable> {
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws IOException {
- ShortestPathsComputation.computeShortestPaths(this, messages);
- }
- }
-
-
@Override
public Configuration getConf() {
return conf;
@@ -89,9 +73,9 @@ public class ShortestPathsBenchmark implements Tool {
true,
"Edges per vertex");
options.addOption("c",
- "vertexClass",
+ "edgesClass",
true,
- "Vertex class (0 for HashMapVertex, 1 for EdgeListVertex)");
+ "Vertex edges class (0 for HashMapEdges, 1 for ArrayListEdges)");
options.addOption("nc",
"noCombiner",
false,
@@ -123,12 +107,12 @@ public class ShortestPathsBenchmark implements Tool {
int workers = Integer.parseInt(cmd.getOptionValue('w'));
GiraphJob job = new GiraphJob(getConf(), getClass().getName());
+ job.getConfiguration().setVertexClass(ShortestPathsVertex.class);
if (!cmd.hasOption('c') ||
(Integer.parseInt(cmd.getOptionValue('c')) == 1)) {
- job.getConfiguration().setVertexClass(ShortestPathsBenchmarkVertex.class);
+ job.getConfiguration().setVertexEdgesClass(ArrayListEdges.class);
} else {
- job.getConfiguration().setVertexClass(
- HashMapVertexShortestPathsBenchmark.class);
+ job.getConfiguration().setVertexEdgesClass(HashMapEdges.class);
}
LOG.info("Using class " +
job.getConfiguration().get(GiraphConstants.VERTEX_CLASS));
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsComputation.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsComputation.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsComputation.java
deleted file mode 100644
index 19f5575..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsComputation.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.benchmark;
-
-import org.apache.giraph.graph.Edge;
-import org.apache.giraph.vertex.Vertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * Default single-source shortest paths computation.
- */
-public class ShortestPathsComputation {
- /** Source id. */
- public static final String SOURCE_ID = "ShortestPathsBenchmark.sourceId";
- /** Default source id. */
- public static final long SOURCE_ID_DEFAULT = 1;
-
- /** Do not construct. */
- private ShortestPathsComputation() { };
-
- /**
- * Is this vertex the source?
- *
- * @param vertex Candidate vertex
- * @return Whether the vertex is the source.
- */
- private static boolean isSource(Vertex<LongWritable, DoubleWritable,
- DoubleWritable, DoubleWritable> vertex) {
- return vertex.getId().get() ==
- vertex.getContext().getConfiguration().getLong(SOURCE_ID,
- SOURCE_ID_DEFAULT);
- }
-
- /**
- * Generic single-source shortest paths algorithm.
- *
- * @param vertex Vertex to run
- * @param messages Incoming messages for vertex
- */
- public static void computeShortestPaths(
- Vertex<LongWritable, DoubleWritable, DoubleWritable,
- DoubleWritable> vertex,
- Iterable<DoubleWritable> messages) {
- if (vertex.getSuperstep() == 0) {
- vertex.setValue(new DoubleWritable(Double.MAX_VALUE));
- }
-
- double minDist = isSource(vertex) ? 0d : Double.MAX_VALUE;
- for (DoubleWritable message : messages) {
- minDist = Math.min(minDist, message.get());
- }
-
- if (minDist < vertex.getValue().get()) {
- vertex.setValue(new DoubleWritable(minDist));
- for (Edge<LongWritable, DoubleWritable> edge : vertex.getEdges()) {
- double distance = minDist + edge.getValue().get();
- vertex.sendMessage(edge.getTargetVertexId(),
- new DoubleWritable(distance));
- }
- }
-
- vertex.voteToHalt();
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsVertex.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsVertex.java
new file mode 100644
index 0000000..4c8bd31
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsVertex.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.benchmark;
+
+import org.apache.giraph.edge.Edge;
+import org.apache.giraph.graph.Vertex;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.LongWritable;
+
+import java.io.IOException;
+
+/**
+ * Shortest paths algorithm.
+ */
+public class ShortestPathsVertex extends Vertex<LongWritable, DoubleWritable,
+ DoubleWritable, DoubleWritable> {
+ /** Source id. */
+ public static final String SOURCE_ID = "ShortestPathsBenchmark.sourceId";
+ /** Default source id. */
+ public static final long SOURCE_ID_DEFAULT = 1;
+
+ private boolean isSource() {
+ return getId().get() == getConf().getLong(SOURCE_ID, SOURCE_ID_DEFAULT);
+ }
+
+ @Override
+ public void compute(Iterable<DoubleWritable> messages) throws IOException {
+ if (getSuperstep() == 0) {
+ setValue(new DoubleWritable(Double.MAX_VALUE));
+ }
+
+ double minDist = isSource() ? 0d : Double.MAX_VALUE;
+ for (DoubleWritable message : messages) {
+ minDist = Math.min(minDist, message.get());
+ }
+
+ if (minDist < getValue().get()) {
+ setValue(new DoubleWritable(minDist));
+ for (Edge<LongWritable, DoubleWritable> edge : getEdges()) {
+ double distance = minDist + edge.getValue().get();
+ sendMessage(edge.getTargetVertexId(),
+ new DoubleWritable(distance));
+ }
+ }
+
+ voteToHalt();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java b/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
index 65c3ebc..574895c 100644
--- a/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
@@ -18,8 +18,7 @@
package org.apache.giraph.bsp;
-import java.io.IOException;
-
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
@@ -28,6 +27,8 @@ import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.log4j.Logger;
+import java.io.IOException;
+
/**
* This is for internal use only. Allows the vertex output format routines
* to be called as if a normal Hadoop job.
@@ -39,27 +40,27 @@ public class BspOutputFormat extends OutputFormat<Text, Text> {
@Override
public void checkOutputSpecs(JobContext context)
throws IOException, InterruptedException {
- if (BspUtils.getVertexOutputFormatClass(context.getConfiguration()) ==
- null) {
+ ImmutableClassesGiraphConfiguration conf =
+ new ImmutableClassesGiraphConfiguration(context.getConfiguration());
+ if (!conf.hasVertexOutputFormat()) {
LOG.warn("checkOutputSpecs: ImmutableOutputCommiter" +
" will not check anything");
return;
}
- BspUtils.createVertexOutputFormat(context.getConfiguration()).
- checkOutputSpecs(context);
+ conf.createVertexOutputFormat().checkOutputSpecs(context);
}
@Override
public OutputCommitter getOutputCommitter(TaskAttemptContext context)
throws IOException, InterruptedException {
- if (BspUtils.getVertexOutputFormatClass(context.getConfiguration()) ==
- null) {
+ ImmutableClassesGiraphConfiguration conf =
+ new ImmutableClassesGiraphConfiguration(context.getConfiguration());
+ if (!conf.hasVertexOutputFormat()) {
LOG.warn("getOutputCommitter: Returning " +
"ImmutableOutputCommiter (does nothing).");
return new ImmutableOutputCommitter();
}
- return BspUtils.createVertexOutputFormat(context.getConfiguration()).
- getOutputCommitter(context);
+ return conf.createVertexOutputFormat().getOutputCommitter(context);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/bsp/BspUtils.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/bsp/BspUtils.java b/giraph-core/src/main/java/org/apache/giraph/bsp/BspUtils.java
deleted file mode 100644
index 335047d..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/bsp/BspUtils.java
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.bsp;
-
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.aggregators.AggregatorWriter;
-import org.apache.giraph.combiner.Combiner;
-import org.apache.giraph.master.DefaultMasterCompute;
-import org.apache.giraph.graph.DefaultVertexResolver;
-import org.apache.giraph.worker.DefaultWorkerContext;
-import org.apache.giraph.io.EdgeInputFormat;
-import org.apache.giraph.aggregators.TextAggregatorWriter;
-import org.apache.giraph.vertex.Vertex;
-import org.apache.giraph.io.VertexInputFormat;
-import org.apache.giraph.io.VertexOutputFormat;
-import org.apache.giraph.graph.VertexResolver;
-import org.apache.giraph.partition.GraphPartitionerFactory;
-import org.apache.giraph.partition.HashPartitionerFactory;
-import org.apache.giraph.master.MasterCompute;
-import org.apache.giraph.worker.WorkerContext;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * Help to use the configuration to get the appropriate classes or
- * instantiate them.
- */
-public class BspUtils {
- /**
- * Do not construct.
- */
- private BspUtils() { }
-
- /**
- * Get the user's subclassed {@link GraphPartitionerFactory}.
- *
- * @param <I> Vertex id
- * @param <V> Vertex data
- * @param <E> Edge data
- * @param <M> Message data
- * @param conf Configuration to check
- * @return User's graph partitioner
- */
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public static <I extends WritableComparable, V extends Writable,
- E extends Writable, M extends Writable>
- Class<? extends GraphPartitionerFactory<I, V, E, M>>
- getGraphPartitionerClass(Configuration conf) {
- return (Class<? extends GraphPartitionerFactory<I, V, E, M>>)
- conf.getClass(GiraphConstants.GRAPH_PARTITIONER_FACTORY_CLASS,
- HashPartitionerFactory.class,
- GraphPartitionerFactory.class);
- }
-
- /**
- * Create a user graph partitioner class
- *
- * @param <I> Vertex id
- * @param <V> Vertex data
- * @param <E> Edge data
- * @param <M> Message data
- * @param conf Configuration to check
- * @return Instantiated user graph partitioner class
- */
- @SuppressWarnings("rawtypes")
- public static <I extends WritableComparable, V extends Writable,
- E extends Writable, M extends Writable>
- GraphPartitionerFactory<I, V, E, M>
- createGraphPartitioner(Configuration conf) {
- Class<? extends GraphPartitionerFactory<I, V, E, M>>
- graphPartitionerFactoryClass = getGraphPartitionerClass(conf);
- return ReflectionUtils.newInstance(graphPartitionerFactoryClass, conf);
- }
-
- /**
- * Get the user's subclassed {@link org.apache.giraph.io.VertexInputFormat}.
- *
- * @param <I> Vertex id
- * @param <V> Vertex data
- * @param <E> Edge data
- * @param <M> Message data
- * @param conf Configuration to check
- * @return User's vertex input format class
- */
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public static <I extends WritableComparable,
- V extends Writable,
- E extends Writable,
- M extends Writable>
- Class<? extends VertexInputFormat<I, V, E, M>>
- getVertexInputFormatClass(Configuration conf) {
- return (Class<? extends VertexInputFormat<I, V, E, M>>)
- conf.getClass(GiraphConstants.VERTEX_INPUT_FORMAT_CLASS,
- null,
- VertexInputFormat.class);
- }
-
- /**
- * Get the user's subclassed {@link org.apache.giraph.io.VertexOutputFormat}.
- *
- * @param <I> Vertex id
- * @param <V> Vertex data
- * @param <E> Edge data
- * @param conf Configuration to check
- * @return User's vertex output format class
- */
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public static <I extends WritableComparable,
- V extends Writable,
- E extends Writable>
- Class<? extends VertexOutputFormat<I, V, E>>
- getVertexOutputFormatClass(Configuration conf) {
- return (Class<? extends VertexOutputFormat<I, V, E>>)
- conf.getClass(GiraphConstants.VERTEX_OUTPUT_FORMAT_CLASS,
- null,
- VertexOutputFormat.class);
- }
-
- /**
- * Create a user vertex output format class
- *
- * @param <I> Vertex id
- * @param <V> Vertex data
- * @param <E> Edge data
- * @param conf Configuration to check
- * @return Instantiated user vertex output format class
- */
- @SuppressWarnings("rawtypes")
- public static <I extends WritableComparable, V extends Writable,
- E extends Writable> VertexOutputFormat<I, V, E>
- createVertexOutputFormat(Configuration conf) {
- Class<? extends VertexOutputFormat<I, V, E>> vertexOutputFormatClass =
- getVertexOutputFormatClass(conf);
- return ReflectionUtils.newInstance(vertexOutputFormatClass, conf);
- }
-
- /**
- * Get the user's subclassed {@link org.apache.giraph.io.EdgeInputFormat}.
- *
- * @param <I> Vertex id
- * @param <E> Edge data
- * @param conf Configuration to check
- * @return User's edge input format class
- */
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public static <I extends WritableComparable, E extends Writable>
- Class<? extends EdgeInputFormat<I, E>>
- getEdgeInputFormatClass(Configuration conf) {
- return (Class<? extends EdgeInputFormat<I, E>>)
- conf.getClass(GiraphConstants.EDGE_INPUT_FORMAT_CLASS,
- null,
- EdgeInputFormat.class);
- }
-
- /**
- * Get the user's subclassed {@link AggregatorWriter}.
- *
- * @param conf Configuration to check
- * @return User's aggregator writer class
- */
- public static Class<? extends AggregatorWriter>
- getAggregatorWriterClass(Configuration conf) {
- return conf.getClass(GiraphConstants.AGGREGATOR_WRITER_CLASS,
- TextAggregatorWriter.class,
- AggregatorWriter.class);
- }
-
- /**
- * Get the user's subclassed {@link org.apache.giraph.combiner.Combiner}.
- *
- * @param <I> Vertex id
- * @param <M> Message data
- * @param conf Configuration to check
- * @return User's vertex combiner class
- */
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public static <I extends WritableComparable, M extends Writable>
- Class<? extends Combiner<I, M>> getCombinerClass(Configuration conf) {
- return (Class<? extends Combiner<I, M>>)
- conf.getClass(GiraphConstants.VERTEX_COMBINER_CLASS,
- null,
- Combiner.class);
- }
-
- /**
- * Get the user's subclassed VertexResolver.
- *
- *
- * @param <I> Vertex id
- * @param <V> Vertex data
- * @param <E> Edge data
- * @param <M> Message data
- * @param conf Configuration to check
- * @return User's vertex resolver class
- */
- @SuppressWarnings({ "unchecked", "rawtypes" })
- public static <I extends WritableComparable, V extends Writable,
- E extends Writable, M extends Writable>
- Class<? extends VertexResolver<I, V, E, M>>
- getVertexResolverClass(Configuration conf) {
- return (Class<? extends VertexResolver<I, V, E, M>>)
- conf.getClass(GiraphConstants.VERTEX_RESOLVER_CLASS,
- DefaultVertexResolver.class,
- VertexResolver.class);
- }
-
- /**
- * Get the user's subclassed WorkerContext.
- *
- * @param conf Configuration to check
- * @return User's worker context class
- */
- public static Class<? extends WorkerContext>
- getWorkerContextClass(Configuration conf) {
- return (Class<? extends WorkerContext>)
- conf.getClass(GiraphConstants.WORKER_CONTEXT_CLASS,
- DefaultWorkerContext.class,
- WorkerContext.class);
- }
-
- /**
- * Get the user's subclassed {@link org.apache.giraph.master.MasterCompute}
- *
- * @param conf Configuration to check
- * @return User's master class
- */
- public static Class<? extends MasterCompute>
- getMasterComputeClass(Configuration conf) {
- return (Class<? extends MasterCompute>)
- conf.getClass(GiraphConstants.MASTER_COMPUTE_CLASS,
- DefaultMasterCompute.class,
- MasterCompute.class);
- }
-
- /**
- * Get the user's subclassed {@link org.apache.giraph.vertex.Vertex}
- *
- * @param <I> Vertex id
- * @param <V> Vertex data
- * @param <E> Edge data
- * @param <M> Message data
- * @param conf Configuration to check
- * @return User's vertex class
- */
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public static <I extends WritableComparable, V extends Writable,
- E extends Writable, M extends Writable>
- Class<? extends Vertex<I, V, E, M>> getVertexClass(Configuration conf) {
- return (Class<? extends Vertex<I, V, E, M>>)
- conf.getClass(GiraphConstants.VERTEX_CLASS,
- null,
- Vertex.class);
- }
-
- /**
- * Create a user vertex
- *
- * @param <I> Vertex id
- * @param <V> Vertex data
- * @param <E> Edge data
- * @param <M> Message data
- * @param conf Configuration to check
- * @return Instantiated user vertex
- */
- @SuppressWarnings("rawtypes")
- public static <I extends WritableComparable, V extends Writable,
- E extends Writable, M extends Writable> Vertex<I, V, E, M>
- createVertex(Configuration conf) {
- Class<? extends Vertex<I, V, E, M>> vertexClass = getVertexClass(conf);
- Vertex<I, V, E, M> vertex =
- ReflectionUtils.newInstance(vertexClass, conf);
- return vertex;
- }
-
- /**
- * Get the user's subclassed vertex index class.
- *
- * @param <I> Vertex id
- * @param conf Configuration to check
- * @return User's vertex index class
- */
- @SuppressWarnings("unchecked")
- public static <I extends Writable> Class<I>
- getVertexIdClass(Configuration conf) {
- return (Class<I>) conf.getClass(GiraphConstants.VERTEX_ID_CLASS,
- WritableComparable.class);
- }
-
- /**
- * Create a user vertex index
- *
- * @param <I> Vertex id
- * @param conf Configuration to check
- * @return Instantiated user vertex index
- */
- @SuppressWarnings("rawtypes")
- public static <I extends WritableComparable>
- I createVertexId(Configuration conf) {
- Class<I> vertexIdClass = getVertexIdClass(conf);
- try {
- return vertexIdClass.newInstance();
- } catch (InstantiationException e) {
- throw new IllegalArgumentException(
- "createVertexId: Failed to instantiate", e);
- } catch (IllegalAccessException e) {
- throw new IllegalArgumentException(
- "createVertexId: Illegally accessed", e);
- }
- }
-
- /**
- * Get the user's subclassed vertex value class.
- *
- * @param <V> Vertex data
- * @param conf Configuration to check
- * @return User's vertex value class
- */
- @SuppressWarnings("unchecked")
- public static <V extends Writable> Class<V>
- getVertexValueClass(Configuration conf) {
- return (Class<V>) conf.getClass(GiraphConstants.VERTEX_VALUE_CLASS,
- Writable.class);
- }
-
- /**
- * Create a user vertex value
- *
- * @param <V> Vertex data
- * @param conf Configuration to check
- * @return Instantiated user vertex value
- */
- @SuppressWarnings("unchecked")
- public static <V extends Writable> V
- createVertexValue(Configuration conf) {
- Class<V> vertexValueClass = getVertexValueClass(conf);
- if (vertexValueClass == NullWritable.class) {
- return (V) NullWritable.get();
- } else {
- try {
- return vertexValueClass.newInstance();
- } catch (InstantiationException e) {
- throw new IllegalArgumentException(
- "createVertexValue: Failed to instantiate", e);
- } catch (IllegalAccessException e) {
- throw new IllegalArgumentException(
- "createVertexValue: Illegally accessed", e);
- }
- }
- }
-
- /**
- * Get the user's subclassed edge value class.
- *
- * @param <E> Edge data
- * @param conf Configuration to check
- * @return User's vertex edge value class
- */
- @SuppressWarnings("unchecked")
- public static <E extends Writable> Class<E>
- getEdgeValueClass(Configuration conf) {
- return (Class<E>) conf.getClass(GiraphConstants.EDGE_VALUE_CLASS,
- Writable.class);
- }
-
- /**
- * Get the user's subclassed vertex message value class.
- *
- * @param <M> Message data
- * @param conf Configuration to check
- * @return User's vertex message value class
- */
- @SuppressWarnings("unchecked")
- public static <M extends Writable> Class<M>
- getMessageValueClass(Configuration conf) {
- return (Class<M>) conf.getClass(GiraphConstants.MESSAGE_VALUE_CLASS,
- Writable.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/comm/MsgList.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/MsgList.java b/giraph-core/src/main/java/org/apache/giraph/comm/MsgList.java
deleted file mode 100644
index 140b6e8..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/comm/MsgList.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.comm;
-
-import org.apache.giraph.bsp.BspUtils;
-import org.apache.giraph.utils.ArrayListWritable;
-import org.apache.hadoop.io.Writable;
-
-/**
- * Wrapper around {@link ArrayListWritable} that allows the message class to
- * be set prior to calling readFields().
- *
- * @param <M> message type
- */
-public class MsgList<M extends Writable> extends ArrayListWritable<M> {
- /** Defining a layout version for a serializable class. */
- private static final long serialVersionUID = 100L;
-
- /**
- * Default constructor.
- */
- public MsgList() {
- super();
- }
-
- /**
- * Copy constructor.
- *
- * @param msgList List of messages for writing.
- */
- public MsgList(MsgList<M> msgList) {
- super(msgList);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void setClass() {
- setClass((Class<M>) BspUtils.getMessageValueClass(getConf()));
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/comm/SendEdgeCache.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/SendEdgeCache.java b/giraph-core/src/main/java/org/apache/giraph/comm/SendEdgeCache.java
index f239c1a..679cf6f 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/SendEdgeCache.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/SendEdgeCache.java
@@ -21,7 +21,7 @@ package org.apache.giraph.comm;
import org.apache.giraph.bsp.CentralizedServiceWorker;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.graph.Edge;
+import org.apache.giraph.edge.Edge;
import org.apache.giraph.utils.ByteArrayVertexIdEdges;
import org.apache.giraph.utils.PairList;
import org.apache.giraph.worker.WorkerInfo;
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/comm/SendMutationsCache.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/SendMutationsCache.java b/giraph-core/src/main/java/org/apache/giraph/comm/SendMutationsCache.java
index 9ea968a..67f74f1 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/SendMutationsCache.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/SendMutationsCache.java
@@ -17,9 +17,9 @@
*/
package org.apache.giraph.comm;
-import org.apache.giraph.graph.Edge;
+import org.apache.giraph.edge.Edge;
import org.apache.giraph.graph.VertexMutations;
-import org.apache.giraph.vertex.Vertex;
+import org.apache.giraph.graph.Vertex;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/comm/SendPartitionCache.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/SendPartitionCache.java b/giraph-core/src/main/java/org/apache/giraph/comm/SendPartitionCache.java
index 911b6ba..31cf052 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/SendPartitionCache.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/SendPartitionCache.java
@@ -21,7 +21,7 @@ import com.google.common.collect.Maps;
import java.util.Map;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.graph.GiraphTransferRegulator;
-import org.apache.giraph.vertex.Vertex;
+import org.apache.giraph.graph.Vertex;
import org.apache.giraph.partition.Partition;
import org.apache.giraph.partition.PartitionOwner;
import org.apache.hadoop.io.Writable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/comm/ServerData.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/ServerData.java b/giraph-core/src/main/java/org/apache/giraph/comm/ServerData.java
index 7b4baa1..e6dff8c 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/ServerData.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/ServerData.java
@@ -25,7 +25,7 @@ import org.apache.giraph.comm.messages.MessageStoreByPartition;
import org.apache.giraph.comm.messages.MessageStoreFactory;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.graph.EdgeStore;
+import org.apache.giraph.edge.EdgeStore;
import org.apache.giraph.graph.VertexMutations;
import org.apache.giraph.partition.DiskBackedPartitionStore;
import org.apache.giraph.partition.PartitionStore;
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/comm/WorkerClientRequestProcessor.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/WorkerClientRequestProcessor.java b/giraph-core/src/main/java/org/apache/giraph/comm/WorkerClientRequestProcessor.java
index 43311f4..5b82d82 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/WorkerClientRequestProcessor.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/WorkerClientRequestProcessor.java
@@ -17,10 +17,10 @@
*/
package org.apache.giraph.comm;
-import org.apache.giraph.graph.Edge;
+import org.apache.giraph.edge.Edge;
import org.apache.giraph.partition.Partition;
import org.apache.giraph.partition.PartitionOwner;
-import org.apache.giraph.vertex.Vertex;
+import org.apache.giraph.graph.Vertex;
import org.apache.giraph.worker.WorkerInfo;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/comm/messages/DiskBackedMessageStoreByPartition.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/DiskBackedMessageStoreByPartition.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/DiskBackedMessageStoreByPartition.java
index 6e6cb9b..02bfb1f 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/DiskBackedMessageStoreByPartition.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/messages/DiskBackedMessageStoreByPartition.java
@@ -21,6 +21,12 @@ package org.apache.giraph.comm.messages;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.giraph.bsp.CentralizedServiceWorker;
+import org.apache.giraph.utils.ByteArrayVertexIdMessages;
+import org.apache.giraph.utils.EmptyIterable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -28,11 +34,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentMap;
-import org.apache.giraph.bsp.CentralizedServiceWorker;
-import org.apache.giraph.utils.ByteArrayVertexIdMessages;
-import org.apache.giraph.utils.EmptyIterable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
/**
* Message store which separates data by partitions,
@@ -128,7 +129,7 @@ public class DiskBackedMessageStoreByPartition<I extends WritableComparable,
if (hasMessagesForVertex(vertexId)) {
return getMessageStore(vertexId).getVertexMessages(vertexId);
} else {
- return EmptyIterable.<M>emptyIterable();
+ return EmptyIterable.get();
}
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/3f5009ae/giraph-core/src/main/java/org/apache/giraph/comm/messages/SequentialFileMessageStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/SequentialFileMessageStore.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/SequentialFileMessageStore.java
index c2c6141..bdc5435 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/SequentialFileMessageStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/messages/SequentialFileMessageStore.java
@@ -179,7 +179,7 @@ public class SequentialFileMessageStore<I extends WritableComparable,
}
if (nextVertexId == null || vertexId.compareTo(nextVertexId) < 0) {
- return EmptyIterable.<M>emptyIterable();
+ return EmptyIterable.get();
}
return readMessagesForCurrentVertex();