You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2013/01/04 21:52:39 UTC
[23/23] git commit: GIRAPH-409: Refactor / cleanups (nitay)
Updated Branches:
refs/heads/trunk e4e6af68f -> 1684891ec
GIRAPH-409: Refactor / cleanups (nitay)
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/1684891e
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/1684891e
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/1684891e
Branch: refs/heads/trunk
Commit: 1684891ecf471cc7d52c95b187fcf290f7e1ce14
Parents: e4e6af6
Author: Nitay Joffe <ni...@apache.org>
Authored: Thu Jan 3 19:10:42 2013 -0500
Committer: Nitay Joffe <ni...@apache.org>
Committed: Fri Jan 4 15:52:17 2013 -0500
----------------------------------------------------------------------
CHANGELOG | 2 +
.../io/accumulo/AccumuloVertexInputFormat.java | 4 +-
.../io/accumulo/AccumuloVertexOutputFormat.java | 4 +-
.../io/accumulo/TestAccumuloVertexFormat.java | 2 +-
.../edgemarker/AccumuloEdgeInputFormat.java | 4 +-
.../edgemarker/AccumuloEdgeOutputFormat.java | 4 +-
.../main/java/org/apache/giraph/GiraphRunner.java | 14 +-
.../org/apache/giraph/aggregators/Aggregator.java | 67 +
.../giraph/aggregators/AggregatorWrapper.java | 146 ++
.../giraph/aggregators/AggregatorWriter.java | 73 +
.../apache/giraph/aggregators/BasicAggregator.java | 1 -
.../giraph/aggregators/TextAggregatorWriter.java | 126 +
.../giraph/benchmark/AggregatorsBenchmark.java | 8 +-
.../benchmark/EdgeListVertexPageRankBenchmark.java | 6 +-
.../benchmark/HashMapVertexPageRankBenchmark.java | 4 +-
.../HashMapVertexShortestPathsBenchmark.java | 4 +-
.../MultiGraphEdgeListVertexPageRankBenchmark.java | 2 +-
...GraphRepresentativeVertexPageRankBenchmark.java | 2 +-
.../apache/giraph/benchmark/PageRankBenchmark.java | 6 +-
.../giraph/benchmark/PageRankComputation.java | 2 +-
.../giraph/benchmark/RandomMessageBenchmark.java | 8 +-
.../RepresentativeVertexPageRankBenchmark.java | 4 +-
.../giraph/benchmark/ShortestPathsBenchmark.java | 6 +-
.../giraph/benchmark/ShortestPathsComputation.java | 2 +-
.../org/apache/giraph/bsp/BspOutputFormat.java | 1 -
.../java/org/apache/giraph/bsp/BspService.java | 1090 +++++++++
.../main/java/org/apache/giraph/bsp/BspUtils.java | 397 +++
.../org/apache/giraph/bsp/CentralizedService.java | 2 +-
.../giraph/bsp/CentralizedServiceMaster.java | 4 +-
.../giraph/bsp/CentralizedServiceWorker.java | 18 +-
.../giraph/bsp/ImmutableOutputCommitter.java | 2 +-
.../java/org/apache/giraph/combiner/Combiner.java | 53 +
.../apache/giraph/combiner/DoubleSumCombiner.java | 39 +
.../giraph/combiner/MinimumDoubleCombiner.java | 41 +
.../apache/giraph/combiner/MinimumIntCombiner.java | 40 +
.../apache/giraph/combiner/SimpleSumCombiner.java | 40 +
.../org/apache/giraph/combiner/package-info.java | 21 +
.../java/org/apache/giraph/comm/MasterClient.java | 2 +-
.../main/java/org/apache/giraph/comm/MsgList.java | 2 +-
.../org/apache/giraph/comm/SendMessageCache.java | 4 +-
.../org/apache/giraph/comm/SendMutationsCache.java | 2 +-
.../org/apache/giraph/comm/SendPartitionCache.java | 6 +-
.../java/org/apache/giraph/comm/ServerData.java | 6 +-
.../java/org/apache/giraph/comm/WorkerClient.java | 2 +-
.../giraph/comm/WorkerClientRequestProcessor.java | 8 +-
.../org/apache/giraph/comm/WorkerClientServer.java | 36 -
.../comm/aggregators/AggregatorOutputStream.java | 2 +-
.../giraph/comm/aggregators/AggregatorUtils.java | 4 +-
.../comm/aggregators/AllAggregatorServerData.java | 2 +-
.../aggregators/OwnerAggregatorServerData.java | 2 +-
.../comm/aggregators/SendAggregatorCache.java | 2 +-
.../comm/messages/OneMessagePerVertexStore.java | 2 +-
.../org/apache/giraph/comm/netty/ByteCounter.java | 4 +-
.../giraph/comm/netty/NettyMasterClient.java | 4 +-
.../NettyWorkerAggregatorRequestProcessor.java | 2 +-
.../giraph/comm/netty/NettyWorkerClient.java | 4 +-
.../netty/NettyWorkerClientRequestProcessor.java | 10 +-
.../giraph/comm/netty/NettyWorkerServer.java | 4 +-
.../netty/handler/MasterRequestServerHandler.java | 2 +-
.../giraph/comm/netty/handler/RequestDecoder.java | 6 +-
.../giraph/comm/netty/handler/RequestEncoder.java | 6 +-
.../giraph/comm/netty/handler/RequestInfo.java | 4 +-
.../comm/netty/handler/RequestServerHandler.java | 6 +-
.../apache/giraph/comm/requests/MasterRequest.java | 2 +-
.../requests/SendAggregatorsToMasterRequest.java | 2 +-
.../requests/SendAggregatorsToOwnerRequest.java | 2 +-
.../requests/SendAggregatorsToWorkerRequest.java | 2 +-
.../giraph/comm/requests/SendVertexRequest.java | 2 +-
.../java/org/apache/giraph/conf/GiraphClasses.java | 30 +-
.../apache/giraph/conf/GiraphConfiguration.java | 20 +-
.../conf/ImmutableClassesGiraphConfiguration.java | 38 +-
.../giraph/examples/AggregatorsTestVertex.java | 4 +-
.../giraph/examples/ConnectedComponentsVertex.java | 2 +-
.../apache/giraph/examples/DoubleSumCombiner.java | 40 -
.../giraph/examples/GeneratedVertexReader.java | 2 +-
.../org/apache/giraph/examples/IdentityVertex.java | 2 +-
.../LongDoubleFloatDoubleTextInputFormat.java | 8 +-
.../giraph/examples/MinimumDoubleCombiner.java | 42 -
.../apache/giraph/examples/MinimumIntCombiner.java | 41 -
...lizingLongDoubleFloatDoubleTextInputFormat.java | 8 +-
.../apache/giraph/examples/RandomWalkVertex.java | 4 +-
.../giraph/examples/RandomWalkWorkerContext.java | 2 +-
.../giraph/examples/SimpleAggregatorWriter.java | 2 +-
.../giraph/examples/SimpleCheckpointVertex.java | 10 +-
.../giraph/examples/SimpleCombinerVertex.java | 2 +-
.../apache/giraph/examples/SimpleFailVertex.java | 2 +-
.../giraph/examples/SimpleInDegreeCountVertex.java | 2 +-
.../giraph/examples/SimpleMasterComputeVertex.java | 6 +-
.../apache/giraph/examples/SimpleMsgVertex.java | 2 +-
.../giraph/examples/SimpleMutateGraphVertex.java | 4 +-
.../examples/SimpleOutDegreeCountVertex.java | 2 +-
.../giraph/examples/SimplePageRankVertex.java | 14 +-
.../giraph/examples/SimpleShortestPathsVertex.java | 2 +-
.../apache/giraph/examples/SimpleSumCombiner.java | 41 -
.../giraph/examples/SimpleSuperstepVertex.java | 10 +-
.../examples/SimpleTextVertexOutputFormat.java | 4 +-
.../examples/SimpleTriangleClosingVertex.java | 2 +-
.../examples/SimpleVertexWithWorkerContext.java | 4 +-
.../org/apache/giraph/examples/VerifyMessage.java | 6 +-
...exWithDoubleValueFloatEdgeTextOutputFormat.java | 4 +-
.../graph/AddressesAndPartitionsWritable.java | 4 +-
.../java/org/apache/giraph/graph/Aggregator.java | 67 -
.../org/apache/giraph/graph/AggregatorWrapper.java | 146 --
.../org/apache/giraph/graph/AggregatorWriter.java | 73 -
.../giraph/graph/BasicVertexValueReader.java | 56 -
.../java/org/apache/giraph/graph/BspService.java | 1069 --------
.../org/apache/giraph/graph/BspServiceMaster.java | 1884 --------------
.../org/apache/giraph/graph/BspServiceWorker.java | 1392 -----------
.../java/org/apache/giraph/graph/BspUtils.java | 523 ----
.../java/org/apache/giraph/graph/Combiner.java | 53 -
.../org/apache/giraph/graph/ComputeCallable.java | 12 +-
.../apache/giraph/graph/DefaultMasterCompute.java | 50 -
.../apache/giraph/graph/DefaultVertexResolver.java | 2 +
.../apache/giraph/graph/DefaultWorkerContext.java | 41 -
.../org/apache/giraph/graph/EdgeInputFormat.java | 75 -
.../giraph/graph/EdgeInputSplitsCallable.java | 165 --
.../graph/EdgeInputSplitsCallableFactory.java | 94 -
.../org/apache/giraph/graph/EdgeListVertex.java | 72 -
.../apache/giraph/graph/EdgeListVertexBase.java | 108 -
.../java/org/apache/giraph/graph/EdgeReader.java | 84 -
.../org/apache/giraph/graph/GiraphInputFormat.java | 42 -
.../java/org/apache/giraph/graph/GiraphJob.java | 1 +
.../giraph/graph/GiraphTransferRegulator.java | 3 +-
.../apache/giraph/graph/GiraphTypeValidator.java | 5 +
.../java/org/apache/giraph/graph/GlobalStats.java | 2 +-
.../java/org/apache/giraph/graph/GraphMapper.java | 17 +-
.../java/org/apache/giraph/graph/GraphState.java | 1 +
.../org/apache/giraph/graph/HashMapVertex.java | 160 --
.../giraph/graph/InputSplitPathOrganizer.java | 195 --
.../apache/giraph/graph/InputSplitsCallable.java | 389 ---
.../giraph/graph/InputSplitsCallableFactory.java | 40 -
.../apache/giraph/graph/IntIntNullIntVertex.java | 100 -
.../apache/giraph/graph/IntNullNullNullVertex.java | 59 -
.../graph/LongDoubleFloatDoubleEdgeListVertex.java | 181 --
.../giraph/graph/LongDoubleFloatDoubleVertex.java | 217 --
.../giraph/graph/LongDoubleNullDoubleVertex.java | 137 --
.../giraph/graph/MasterAggregatorHandler.java | 344 ---
.../apache/giraph/graph/MasterAggregatorUsage.java | 71 -
.../org/apache/giraph/graph/MasterCompute.java | 175 --
.../java/org/apache/giraph/graph/MasterInfo.java | 35 -
.../java/org/apache/giraph/graph/MasterThread.java | 184 --
.../giraph/graph/MultiGraphEdgeListVertex.java | 61 -
.../graph/MultiGraphRepresentativeVertex.java | 48 -
.../org/apache/giraph/graph/MutableVertex.java | 118 -
.../apache/giraph/graph/RepresentativeVertex.java | 68 -
.../giraph/graph/RepresentativeVertexBase.java | 285 ---
.../apache/giraph/graph/SimpleMutableVertex.java | 130 -
.../java/org/apache/giraph/graph/SimpleVertex.java | 104 -
.../apache/giraph/graph/TextAggregatorWriter.java | 126 -
.../main/java/org/apache/giraph/graph/Vertex.java | 405 ---
.../org/apache/giraph/graph/VertexChanges.java | 1 +
.../org/apache/giraph/graph/VertexInputFormat.java | 82 -
.../giraph/graph/VertexInputSplitsCallable.java | 180 --
.../graph/VertexInputSplitsCallableFactory.java | 94 -
.../org/apache/giraph/graph/VertexMutations.java | 1 +
.../apache/giraph/graph/VertexOutputFormat.java | 81 -
.../java/org/apache/giraph/graph/VertexReader.java | 87 -
.../org/apache/giraph/graph/VertexResolver.java | 4 +-
.../giraph/graph/VertexValueInputFormat.java | 58 -
.../org/apache/giraph/graph/VertexValueReader.java | 61 -
.../java/org/apache/giraph/graph/VertexWriter.java | 67 -
.../giraph/graph/WorkerAggregatorHandler.java | 294 ---
.../apache/giraph/graph/WorkerAggregatorUsage.java | 44 -
.../org/apache/giraph/graph/WorkerContext.java | 121 -
.../java/org/apache/giraph/graph/WorkerInfo.java | 35 -
.../giraph/graph/WorkerThreadAggregatorUsage.java | 31 -
.../graph/partition/BasicPartitionOwner.java | 163 --
.../giraph/graph/partition/ByteArrayPartition.java | 277 ---
.../graph/partition/DiskBackedPartitionStore.java | 364 ---
.../graph/partition/GraphPartitionerFactory.java | 52 -
.../graph/partition/HashMasterPartitioner.java | 157 --
.../graph/partition/HashPartitionerFactory.java | 60 -
.../partition/HashRangePartitionerFactory.java | 60 -
.../partition/HashRangeWorkerPartitioner.java | 50 -
.../graph/partition/HashWorkerPartitioner.java | 119 -
.../graph/partition/MasterGraphPartitioner.java | 83 -
.../apache/giraph/graph/partition/Partition.java | 122 -
.../giraph/graph/partition/PartitionBalancer.java | 288 ---
.../giraph/graph/partition/PartitionExchange.java | 77 -
.../giraph/graph/partition/PartitionOwner.java | 81 -
.../giraph/graph/partition/PartitionStats.java | 178 --
.../giraph/graph/partition/PartitionStore.java | 116 -
.../giraph/graph/partition/PartitionUtils.java | 151 --
.../graph/partition/RangeMasterPartitioner.java | 43 -
.../graph/partition/RangePartitionOwner.java | 74 -
.../graph/partition/RangePartitionStats.java | 68 -
.../graph/partition/RangePartitionerFactory.java | 43 -
.../giraph/graph/partition/RangeSplitHint.java | 72 -
.../graph/partition/RangeWorkerPartitioner.java | 78 -
.../giraph/graph/partition/SimplePartition.java | 188 --
.../graph/partition/SimplePartitionStore.java | 105 -
.../graph/partition/WorkerGraphPartitioner.java | 94 -
.../giraph/graph/partition/package-info.java | 21 -
.../SuperstepHashPartitionerFactory.java | 14 +-
.../io/AdjacencyListTextVertexInputFormat.java | 181 --
.../io/AdjacencyListTextVertexOutputFormat.java | 88 -
.../apache/giraph/io/BasicVertexValueReader.java | 56 +
.../java/org/apache/giraph/io/EdgeInputFormat.java | 75 +
.../main/java/org/apache/giraph/io/EdgeReader.java | 85 +
.../giraph/io/GeneratedVertexInputFormat.java | 56 -
.../apache/giraph/io/GiraphFileInputFormat.java | 339 ---
.../org/apache/giraph/io/GiraphInputFormat.java | 42 +
.../apache/giraph/io/GiraphTextInputFormat.java | 51 -
.../giraph/io/IdWithValueTextOutputFormat.java | 98 -
.../giraph/io/IntIntNullIntTextInputFormat.java | 93 -
.../io/IntIntTextVertexValueInputFormat.java | 76 -
.../giraph/io/IntNullNullNullTextInputFormat.java | 75 -
.../giraph/io/IntNullTextEdgeInputFormat.java | 78 -
.../apache/giraph/io/JsonBase64VertexFormat.java | 36 -
.../giraph/io/JsonBase64VertexInputFormat.java | 151 --
.../giraph/io/JsonBase64VertexOutputFormat.java | 109 -
...JsonLongDoubleFloatDoubleVertexInputFormat.java | 108 -
...sonLongDoubleFloatDoubleVertexOutputFormat.java | 77 -
...DoubleDoubleAdjacencyListVertexInputFormat.java | 84 -
.../giraph/io/PseudoRandomEdgeInputFormat.java | 197 --
.../giraph/io/PseudoRandomVertexInputFormat.java | 198 --
.../giraph/io/SequenceFileVertexInputFormat.java | 115 -
...DoubleDoubleAdjacencyListVertexInputFormat.java | 78 -
.../org/apache/giraph/io/TextEdgeInputFormat.java | 283 ---
.../apache/giraph/io/TextVertexInputFormat.java | 432 ----
.../apache/giraph/io/TextVertexOutputFormat.java | 165 --
.../giraph/io/TextVertexValueInputFormat.java | 260 --
.../org/apache/giraph/io/VertexInputFormat.java | 82 +
.../org/apache/giraph/io/VertexOutputFormat.java | 81 +
.../java/org/apache/giraph/io/VertexReader.java | 88 +
.../apache/giraph/io/VertexValueInputFormat.java | 58 +
.../org/apache/giraph/io/VertexValueReader.java | 62 +
.../java/org/apache/giraph/io/VertexWriter.java | 68 +
.../AdjacencyListTextVertexInputFormat.java | 181 ++
.../AdjacencyListTextVertexOutputFormat.java | 88 +
.../io/formats/GeneratedVertexInputFormat.java | 56 +
.../giraph/io/formats/GiraphFileInputFormat.java | 339 +++
.../giraph/io/formats/GiraphTextInputFormat.java | 51 +
.../io/formats/IdWithValueTextOutputFormat.java | 98 +
.../io/formats/IntIntNullIntTextInputFormat.java | 93 +
.../formats/IntIntTextVertexValueInputFormat.java | 76 +
.../io/formats/IntNullNullNullTextInputFormat.java | 75 +
.../io/formats/IntNullTextEdgeInputFormat.java | 78 +
.../giraph/io/formats/JsonBase64VertexFormat.java | 36 +
.../io/formats/JsonBase64VertexInputFormat.java | 151 ++
.../io/formats/JsonBase64VertexOutputFormat.java | 109 +
...JsonLongDoubleFloatDoubleVertexInputFormat.java | 108 +
...sonLongDoubleFloatDoubleVertexOutputFormat.java | 77 +
...DoubleDoubleAdjacencyListVertexInputFormat.java | 84 +
.../io/formats/PseudoRandomEdgeInputFormat.java | 197 ++
.../io/formats/PseudoRandomVertexInputFormat.java | 198 ++
.../io/formats/SequenceFileVertexInputFormat.java | 115 +
...DoubleDoubleAdjacencyListVertexInputFormat.java | 78 +
.../giraph/io/formats/TextEdgeInputFormat.java | 283 +++
.../giraph/io/formats/TextVertexInputFormat.java | 432 ++++
.../giraph/io/formats/TextVertexOutputFormat.java | 165 ++
.../io/formats/TextVertexValueInputFormat.java | 260 ++
.../org/apache/giraph/io/formats/package-info.java | 21 +
.../java/org/apache/giraph/io/package-info.java | 2 +-
.../org/apache/giraph/master/BspServiceMaster.java | 1895 +++++++++++++++
.../apache/giraph/master/DefaultMasterCompute.java | 50 +
.../giraph/master/MasterAggregatorHandler.java | 348 +++
.../giraph/master/MasterAggregatorUsage.java | 72 +
.../org/apache/giraph/master/MasterCompute.java | 177 ++
.../java/org/apache/giraph/master/MasterInfo.java | 37 +
.../org/apache/giraph/master/MasterThread.java | 185 ++
.../org/apache/giraph/master/package-info.java | 2 +-
.../apache/giraph/metrics/AggregatedMetrics.java | 2 +-
.../org/apache/giraph/metrics/GiraphMetrics.java | 2 +-
.../apache/giraph/metrics/GiraphTimerContext.java | 6 +-
.../giraph/metrics/SuperstepMetricsRegistry.java | 2 +-
.../giraph/metrics/WorkerSuperstepMetrics.java | 2 +-
.../giraph/partition/BasicPartitionOwner.java | 163 ++
.../giraph/partition/ByteArrayPartition.java | 277 +++
.../giraph/partition/DiskBackedPartitionStore.java | 364 +++
.../giraph/partition/GraphPartitionerFactory.java | 52 +
.../giraph/partition/HashMasterPartitioner.java | 157 ++
.../giraph/partition/HashPartitionerFactory.java | 60 +
.../partition/HashRangePartitionerFactory.java | 60 +
.../partition/HashRangeWorkerPartitioner.java | 50 +
.../giraph/partition/HashWorkerPartitioner.java | 119 +
.../giraph/partition/MasterGraphPartitioner.java | 83 +
.../org/apache/giraph/partition/Partition.java | 122 +
.../apache/giraph/partition/PartitionBalancer.java | 288 +++
.../apache/giraph/partition/PartitionExchange.java | 77 +
.../apache/giraph/partition/PartitionOwner.java | 81 +
.../apache/giraph/partition/PartitionStats.java | 178 ++
.../apache/giraph/partition/PartitionStore.java | 116 +
.../apache/giraph/partition/PartitionUtils.java | 151 ++
.../giraph/partition/RangeMasterPartitioner.java | 43 +
.../giraph/partition/RangePartitionOwner.java | 74 +
.../giraph/partition/RangePartitionStats.java | 68 +
.../giraph/partition/RangePartitionerFactory.java | 43 +
.../apache/giraph/partition/RangeSplitHint.java | 72 +
.../giraph/partition/RangeWorkerPartitioner.java | 78 +
.../apache/giraph/partition/SimplePartition.java | 188 ++
.../giraph/partition/SimplePartitionStore.java | 105 +
.../giraph/partition/WorkerGraphPartitioner.java | 94 +
.../org/apache/giraph/partition/package-info.java | 21 +
.../main/java/org/apache/giraph/time/FakeTime.java | 61 +
.../java/org/apache/giraph/time/SystemTime.java | 71 +
.../src/main/java/org/apache/giraph/time/Time.java | 101 +
.../main/java/org/apache/giraph/time/Times.java | 137 ++
.../java/org/apache/giraph/time/package-info.java | 21 +
.../java/org/apache/giraph/utils/FakeTime.java | 61 -
.../apache/giraph/utils/InternalVertexRunner.java | 2 +-
.../java/org/apache/giraph/utils/SystemTime.java | 71 -
.../main/java/org/apache/giraph/utils/Time.java | 101 -
.../main/java/org/apache/giraph/utils/Times.java | 137 --
.../org/apache/giraph/vertex/EdgeListVertex.java | 73 +
.../apache/giraph/vertex/EdgeListVertexBase.java | 110 +
.../org/apache/giraph/vertex/HashMapVertex.java | 162 ++
.../apache/giraph/vertex/IntIntNullIntVertex.java | 100 +
.../giraph/vertex/IntNullNullNullVertex.java | 61 +
.../LongDoubleFloatDoubleEdgeListVertex.java | 184 ++
.../giraph/vertex/LongDoubleFloatDoubleVertex.java | 219 ++
.../giraph/vertex/LongDoubleNullDoubleVertex.java | 139 ++
.../giraph/vertex/MultiGraphEdgeListVertex.java | 62 +
.../vertex/MultiGraphRepresentativeVertex.java | 49 +
.../org/apache/giraph/vertex/MutableVertex.java | 119 +
.../apache/giraph/vertex/RepresentativeVertex.java | 69 +
.../giraph/vertex/RepresentativeVertexBase.java | 286 +++
.../apache/giraph/vertex/SimpleMutableVertex.java | 132 +
.../org/apache/giraph/vertex/SimpleVertex.java | 106 +
.../main/java/org/apache/giraph/vertex/Vertex.java | 410 ++++
.../org/apache/giraph/vertex/package-info.java | 21 +
.../org/apache/giraph/worker/BspServiceWorker.java | 1405 +++++++++++
.../apache/giraph/worker/DefaultWorkerContext.java | 41 +
.../giraph/worker/EdgeInputSplitsCallable.java | 170 ++
.../worker/EdgeInputSplitsCallableFactory.java | 95 +
.../giraph/worker/InputSplitPathOrganizer.java | 195 ++
.../apache/giraph/worker/InputSplitsCallable.java | 392 +++
.../giraph/worker/InputSplitsCallableFactory.java | 40 +
.../giraph/worker/VertexInputSplitsCallable.java | 185 ++
.../worker/VertexInputSplitsCallableFactory.java | 95 +
.../giraph/worker/WorkerAggregatorHandler.java | 295 +++
.../giraph/worker/WorkerAggregatorUsage.java | 44 +
.../org/apache/giraph/worker/WorkerContext.java | 122 +
.../java/org/apache/giraph/worker/WorkerInfo.java | 37 +
.../giraph/worker/WorkerThreadAggregatorUsage.java | 31 +
.../org/apache/giraph/worker/package-info.java | 21 +
.../java/org/apache/giraph/zk/PredicateLock.java | 4 +-
.../src/test/java/org/apache/giraph/BspCase.java | 2 +-
.../test/java/org/apache/giraph/TestBspBasic.java | 14 +-
.../test/java/org/apache/giraph/TestEdgeInput.java | 165 --
.../org/apache/giraph/TestGiraphConfiguration.java | 72 -
.../org/apache/giraph/TestGraphPartitioner.java | 4 +-
.../org/apache/giraph/TestJsonBase64Format.java | 104 -
.../java/org/apache/giraph/TestMasterObserver.java | 103 -
.../test/java/org/apache/giraph/TestPageRank.java | 104 -
.../java/org/apache/giraph/TestPredicateLock.java | 148 --
.../java/org/apache/giraph/TestVertexTypes.java | 211 --
.../java/org/apache/giraph/TestZooKeeperExt.java | 173 --
.../aggregators/TestAggregatorsHandling.java | 197 ++
.../java/org/apache/giraph/bsp/BspUtilsTest.java | 195 ++
.../org/apache/giraph/comm/ConnectionTest.java | 4 +-
.../org/apache/giraph/comm/RequestFailureTest.java | 4 +-
.../java/org/apache/giraph/comm/RequestTest.java | 10 +-
.../org/apache/giraph/comm/SaslConnectionTest.java | 4 +-
.../org/apache/giraph/comm/TestMessageStores.java | 2 +-
.../giraph/conf/TestGiraphConfiguration.java | 71 +
.../examples/ConnectedComponentsVertexTest.java | 5 +-
.../giraph/examples/MinimumIntCombinerTest.java | 3 +-
.../examples/SimpleShortestPathsVertexTest.java | 4 +-
.../org/apache/giraph/examples/TestPageRank.java | 104 +
.../examples/TryMultiIpcBindingPortsTest.java | 5 +-
.../giraph/graph/TestAggregatorsHandling.java | 197 --
.../giraph/graph/TestIntIntNullIntVertex.java | 78 -
.../apache/giraph/graph/TestMultiGraphVertex.java | 129 -
.../org/apache/giraph/graph/TestMutableVertex.java | 467 ----
.../partition/TestGiraphTransferRegulator.java | 93 -
.../graph/partition/TestPartitionStores.java | 211 --
.../TestAdjacencyListTextVertexOutputFormat.java | 6 +-
.../java/org/apache/giraph/io/TestEdgeInput.java | 166 ++
.../giraph/io/TestIdWithValueTextOutputFormat.java | 6 +-
.../org/apache/giraph/io/TestJsonBase64Format.java | 105 +
...DoubleDoubleAdjacencyListVertexInputFormat.java | 9 +-
...DoubleDoubleAdjacencyListVertexInputFormat.java | 11 +-
.../apache/giraph/master/TestMasterObserver.java | 102 +
.../partition/TestGiraphTransferRegulator.java | 94 +
.../giraph/partition/TestPartitionStores.java | 211 ++
.../java/org/apache/giraph/utils/BspUtilsTest.java | 193 --
.../java/org/apache/giraph/utils/MockUtils.java | 6 +-
.../giraph/vertex/TestIntIntNullIntVertex.java | 79 +
.../apache/giraph/vertex/TestMultiGraphVertex.java | 130 +
.../apache/giraph/vertex/TestMutableVertex.java | 467 ++++
.../org/apache/giraph/vertex/TestVertexTypes.java | 209 ++
.../org/apache/giraph/zk/TestPredicateLock.java | 148 ++
.../org/apache/giraph/zk/TestZooKeeperExt.java | 173 ++
.../org/apache/giraph/zk/TestZooKeeperManager.java | 1 -
.../giraph/io/hbase/HBaseVertexInputFormat.java | 4 +-
.../giraph/io/hbase/HBaseVertexOutputFormat.java | 4 +-
.../io/hbase/TestHBaseRootMarkerVertextFormat.java | 2 +-
.../io/hbase/edgemarker/TableEdgeInputFormat.java | 4 +-
.../io/hbase/edgemarker/TableEdgeOutputFormat.java | 4 +-
.../io/hcatalog/HCatalogEdgeInputFormat.java | 4 +-
.../io/hcatalog/HCatalogVertexInputFormat.java | 6 +-
.../io/hcatalog/HCatalogVertexOutputFormat.java | 6 +-
.../hcatalog/HCatalogVertexValueInputFormat.java | 4 +-
.../giraph/io/hcatalog/HiveGiraphRunner.java | 8 +-
395 files changed, 20885 insertions(+), 20804 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 0e14a80..bfb9a55 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
Giraph Change Log
Release 0.2.0 - unreleased
+ GIRAPH-409: Refactor / cleanups (nitay)
+
GIRAPH-465: MapFunctions cleanup (nitay)
GIRAPH-464: MasterObserver#applicationFailed callback (nitay)
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexInputFormat.java b/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexInputFormat.java
index cdd805a..c1b632e 100644
--- a/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexInputFormat.java
+++ b/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexInputFormat.java
@@ -21,8 +21,8 @@ import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.graph.VertexInputFormat;
-import org.apache.giraph.graph.VertexReader;
+import org.apache.giraph.io.VertexInputFormat;
+import org.apache.giraph.io.VertexReader;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexOutputFormat.java b/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexOutputFormat.java
index 32fa70f..182afad 100644
--- a/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexOutputFormat.java
+++ b/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexOutputFormat.java
@@ -19,8 +19,8 @@ package org.apache.giraph.io.accumulo;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
import org.apache.accumulo.core.data.Mutation;
-import org.apache.giraph.graph.VertexOutputFormat;
-import org.apache.giraph.graph.VertexWriter;
+import org.apache.giraph.io.VertexOutputFormat;
+import org.apache.giraph.io.VertexWriter;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/TestAccumuloVertexFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/TestAccumuloVertexFormat.java b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/TestAccumuloVertexFormat.java
index 5885b64..e68e18d 100644
--- a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/TestAccumuloVertexFormat.java
+++ b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/TestAccumuloVertexFormat.java
@@ -35,7 +35,7 @@ import org.apache.giraph.BspCase;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.io.accumulo.edgemarker.AccumuloEdgeInputFormat;
import org.apache.giraph.io.accumulo.edgemarker.AccumuloEdgeOutputFormat;
-import org.apache.giraph.graph.EdgeListVertex;
+import org.apache.giraph.vertex.EdgeListVertex;
import org.apache.giraph.graph.GiraphJob;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
index b670144..4f0d253 100644
--- a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
+++ b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
@@ -20,8 +20,8 @@ package org.apache.giraph.io.accumulo.edgemarker;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.giraph.graph.Edge;
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.graph.VertexReader;
+import org.apache.giraph.vertex.Vertex;
+import org.apache.giraph.io.VertexReader;
import org.apache.giraph.io.accumulo.AccumuloVertexInputFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
index ff00fd6..70288f6 100644
--- a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
+++ b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
@@ -20,8 +20,8 @@ package org.apache.giraph.io.accumulo.edgemarker;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.giraph.io.accumulo.AccumuloVertexOutputFormat;
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.graph.VertexWriter;
+import org.apache.giraph.vertex.Vertex;
+import org.apache.giraph.io.VertexWriter;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/GiraphRunner.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/GiraphRunner.java b/giraph-core/src/main/java/org/apache/giraph/GiraphRunner.java
index a550b0c..1edb262 100644
--- a/giraph-core/src/main/java/org/apache/giraph/GiraphRunner.java
+++ b/giraph-core/src/main/java/org/apache/giraph/GiraphRunner.java
@@ -24,15 +24,15 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.examples.Algorithm;
-import org.apache.giraph.graph.AggregatorWriter;
-import org.apache.giraph.graph.Combiner;
+import org.apache.giraph.aggregators.AggregatorWriter;
+import org.apache.giraph.combiner.Combiner;
import org.apache.giraph.graph.GiraphJob;
import org.apache.giraph.graph.GiraphTypeValidator;
-import org.apache.giraph.graph.MasterCompute;
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.graph.VertexInputFormat;
-import org.apache.giraph.graph.VertexOutputFormat;
-import org.apache.giraph.graph.WorkerContext;
+import org.apache.giraph.master.MasterCompute;
+import org.apache.giraph.vertex.Vertex;
+import org.apache.giraph.io.VertexInputFormat;
+import org.apache.giraph.io.VertexOutputFormat;
+import org.apache.giraph.worker.WorkerContext;
import org.apache.giraph.utils.AnnotationUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/aggregators/Aggregator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/Aggregator.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/Aggregator.java
new file mode 100644
index 0000000..514e470
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/Aggregator.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.aggregators;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Interface for Aggregator. Allows aggregate operations for all vertices
+ * in a given superstep.
+ *
+ * @param <A> Aggregated value
+ */
+public interface Aggregator<A extends Writable> {
+ /**
+ * Add a new value.
+ * Needs to be commutative and associative
+ *
+ * @param value Value to be aggregated.
+ */
+ void aggregate(A value);
+
+ /**
+ * Return new aggregated value which is neutral to aggregate operation.
+ * Must be changeable without affecting internals of Aggregator
+ *
+ * @return Neutral value
+ */
+ A createInitialValue();
+
+ /**
+ * Return current aggregated value.
+ * Needs to be initialized if aggregate or setAggregatedValue
+ * have not been called before.
+ *
+ * @return Aggregated
+ */
+ A getAggregatedValue();
+
+ /**
+ * Set aggregated value.
+ * Can be used for initialization or reset.
+ *
+ * @param value Value to be set.
+ */
+ void setAggregatedValue(A value);
+
+ /**
+ * Reset the value of aggregator to neutral value
+ */
+ void reset();
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWrapper.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWrapper.java
new file mode 100644
index 0000000..6b98085
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWrapper.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.aggregators;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Wrapper for aggregators. Keeps two instances of an aggregator - one for
+ * the value from previous super step, and one for the value which is being
+ * generated in current super step.
+ *
+ * @param <A> Aggregated value
+ */
+public class AggregatorWrapper<A extends Writable> {
+ /** False iff aggregator should be reset at the end of each super step */
+ private final boolean persistent;
+ /** Value aggregated in previous super step */
+ private A previousAggregatedValue;
+ /** Aggregator for next super step */
+ private final Aggregator<A> currentAggregator;
+ /** Whether anyone changed current value since the moment it was reset */
+ private boolean changed;
+
+ /**
+ * @param aggregatorClass Class type of the aggregator
+ * @param persistent False iff aggregator should be reset at the end of
+ * each super step
+ * @throws IllegalAccessException
+ * @throws InstantiationException
+ */
+ public AggregatorWrapper(Class<? extends Aggregator<A>> aggregatorClass,
+ boolean persistent) throws IllegalAccessException,
+ InstantiationException {
+ this.persistent = persistent;
+ currentAggregator = aggregatorClass.newInstance();
+ changed = false;
+ previousAggregatedValue = currentAggregator.createInitialValue();
+ }
+
+ /**
+ * Get aggregated value from previous super step
+ *
+ * @return Aggregated value from previous super step
+ */
+ public A getPreviousAggregatedValue() {
+ return previousAggregatedValue;
+ }
+
+ /**
+ * Set aggregated value for previous super step
+ *
+ * @param value Aggregated value to set
+ */
+ public void setPreviousAggregatedValue(A value) {
+ previousAggregatedValue = value;
+ }
+
+ /**
+ * Check if aggregator is persistent
+ *
+ * @return False iff aggregator should be reset at the end of each super step
+ */
+ public boolean isPersistent() {
+ return persistent;
+ }
+
+ /**
+ * Check if current aggregator was changed
+ *
+ * @return Whether anyone changed current value since the moment it was reset
+ */
+ public boolean isChanged() {
+ return changed;
+ }
+
+ /**
+ * Add a new value to current aggregator
+ *
+ * @param value Value to be aggregated
+ */
+ public synchronized void aggregateCurrent(A value) {
+ changed = true;
+ currentAggregator.aggregate(value);
+ }
+
+ /**
+ * Get current aggregated value
+ *
+ * @return Current aggregated value
+ */
+ public A getCurrentAggregatedValue() {
+ return currentAggregator.getAggregatedValue();
+ }
+
+ /**
+ * Set aggregated value of current aggregator
+ *
+ * @param value Value to set it to
+ */
+ public void setCurrentAggregatedValue(A value) {
+ changed = true;
+ currentAggregator.setAggregatedValue(value);
+ }
+
+ /**
+ * Reset the value of current aggregator to neutral value
+ */
+ public void resetCurrentAggregator() {
+ changed = false;
+ currentAggregator.reset();
+ }
+
+ /**
+ * Return new aggregated value which is neutral to aggregate operation
+ *
+ * @return Neutral value
+ */
+ public A createInitialValue() {
+ return currentAggregator.createInitialValue();
+ }
+
+ /**
+ * Get class of wrapped aggregator
+ *
+ * @return Aggregator class
+ */
+ public Class<? extends Aggregator> getAggregatorClass() {
+ return currentAggregator.getClass();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWriter.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWriter.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWriter.java
new file mode 100644
index 0000000..c1c6678
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWriter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.aggregators;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Mapper.Context;
+
+/**
+ * An AggregatorWriter is used to export Aggregators during or at the end of
+ * each computation. It runs on the master and it's called at the end of each
+ * superstep. The special signal {@link AggregatorWriter#LAST_SUPERSTEP} is
+ * passed to {@link AggregatorWriter#writeAggregator(Iterable, long)} as the
+ * superstep value to signal the end of computation.
+ */
+public interface AggregatorWriter {
+ /** Signal for last superstep */
+ int LAST_SUPERSTEP = -1;
+
+ /**
+ * The method is called at the initialization of the AggregatorWriter.
+ * More precisely, the aggregatorWriter is initialized each time a new
+ * master is elected.
+ *
+ * @param context Mapper Context where the master is running on
+ * @param applicationAttempt ID of the applicationAttempt, used to
+ * disambiguate aggregator writes for different attempts
+ * @throws IOException
+ */
+ @SuppressWarnings("rawtypes")
+ void initialize(Context context, long applicationAttempt) throws IOException;
+
+ /**
+ * The method is called at the end of each superstep. The user might decide
+ * whether to write the aggregators values for the current superstep. For
+ * the last superstep, {@link AggregatorWriter#LAST_SUPERSTEP} is passed.
+ *
+ * @param aggregatorMap Map from aggregator name to aggregator value
+ * @param superstep Current superstep
+ * @throws IOException
+ */
+ void writeAggregator(
+ Iterable<Entry<String, Writable>> aggregatorMap,
+ long superstep) throws IOException;
+
+ /**
+ * The method is called at the end of a successful computation. The method
+ * is not called when the job fails and a new master is elected. For this
+ * reason it's advised to flush data at the end of
+ * {@link AggregatorWriter#writeAggregator(Iterable, long)}.
+ *
+ * @throws IOException
+ */
+ void close() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/aggregators/BasicAggregator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/BasicAggregator.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/BasicAggregator.java
index 6210001..07a4100 100644
--- a/giraph-core/src/main/java/org/apache/giraph/aggregators/BasicAggregator.java
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/BasicAggregator.java
@@ -18,7 +18,6 @@
package org.apache.giraph.aggregators;
-import org.apache.giraph.graph.Aggregator;
import org.apache.hadoop.io.Writable;
/**
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/aggregators/TextAggregatorWriter.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/TextAggregatorWriter.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/TextAggregatorWriter.java
new file mode 100644
index 0000000..ef43714
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/TextAggregatorWriter.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.aggregators;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+
+import com.google.common.base.Charsets;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Mapper.Context;
+
+/**
+ * Default implementation of {@link AggregatorWriter}. Each line consists of
+ * text and contains the aggregator name, the aggregator value and the
+ * aggregator class.
+ */
+public class TextAggregatorWriter implements AggregatorWriter {
+ /** The filename of the outputfile */
+ public static final String FILENAME =
+ "giraph.textAggregatorWriter.filename";
+ /** Signal for "never write" frequency */
+ public static final int NEVER = 0;
+ /** Signal for "write only the final values" frequency */
+ public static final int AT_THE_END = -1;
+ /** Signal for "write values in every superstep" frequency */
+ public static final int ALWAYS = -1;
+ /** The frequency of writing:
+ * - NEVER: never write, files aren't created at all
+ * - AT_THE_END: aggregators are written only when the computation is over
+ * - int: i.e. 1 is every superstep, 2 every two supersteps and so on
+ */
+ public static final String FREQUENCY =
+ "giraph.textAggregatorWriter.frequency";
+ /** Default filename for dumping aggregator values */
+ private static final String DEFAULT_FILENAME = "aggregatorValues";
+ /** Handle to the outputfile */
+ protected FSDataOutputStream output;
+ /** Write every "frequency" supersteps */
+ private int frequency;
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ public void initialize(Context context, long attempt) throws IOException {
+ Configuration conf = context.getConfiguration();
+ frequency = conf.getInt(FREQUENCY, NEVER);
+ String filename = conf.get(FILENAME, DEFAULT_FILENAME);
+ if (frequency != NEVER) {
+ Path p = new Path(filename + "_" + attempt);
+ FileSystem fs = FileSystem.get(conf);
+ if (fs.exists(p)) {
+ throw new RuntimeException("aggregatorWriter file already" +
+ " exists: " + p.getName());
+ }
+ output = fs.create(p);
+ }
+ }
+
+ @Override
+ public void writeAggregator(
+ Iterable<Entry<String, Writable>> aggregatorMap,
+ long superstep) throws IOException {
+ if (shouldWrite(superstep)) {
+ for (Entry<String, Writable> entry : aggregatorMap) {
+ byte[] bytes = aggregatorToString(entry.getKey(), entry.getValue(),
+ superstep).getBytes(Charsets.UTF_8);
+ output.write(bytes, 0, bytes.length);
+ }
+ output.flush();
+ }
+ }
+
+ /**
+ * Implements the way an aggregator is converted into a String.
+ * Override this if you want to implement your own text format.
+ *
+ * @param aggregatorName Name of the aggregator
+ * @param value Value of aggregator
+ * @param superstep Current superstep
+ * @return The String representation for the aggregator
+ */
+ protected String aggregatorToString(String aggregatorName,
+ Writable value,
+ long superstep) {
+ return new StringBuilder("superstep=").append(superstep).append("\t")
+ .append(aggregatorName).append("=").append(value).append("\n")
+ .toString();
+ }
+
+ /**
+ * Should write this superstep?
+ *
+ * @param superstep Superstep to check
+ * @return True if should write, false otherwise
+ */
+ private boolean shouldWrite(long superstep) {
+ return (frequency == AT_THE_END && superstep == LAST_SUPERSTEP) ||
+ (frequency != NEVER && superstep % frequency == 0);
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (output != null) {
+ output.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/AggregatorsBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/AggregatorsBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/AggregatorsBenchmark.java
index df30f2f..f9a0730 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/AggregatorsBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/AggregatorsBenchmark.java
@@ -24,11 +24,11 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.giraph.aggregators.LongSumAggregator;
-import org.apache.giraph.graph.DefaultMasterCompute;
-import org.apache.giraph.graph.DefaultWorkerContext;
-import org.apache.giraph.graph.EdgeListVertex;
+import org.apache.giraph.master.DefaultMasterCompute;
+import org.apache.giraph.worker.DefaultWorkerContext;
+import org.apache.giraph.vertex.EdgeListVertex;
import org.apache.giraph.graph.GiraphJob;
-import org.apache.giraph.io.PseudoRandomVertexInputFormat;
+import org.apache.giraph.io.formats.PseudoRandomVertexInputFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/EdgeListVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/EdgeListVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/EdgeListVertexPageRankBenchmark.java
index 68f9e02..fe1e346 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/EdgeListVertexPageRankBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/EdgeListVertexPageRankBenchmark.java
@@ -19,14 +19,14 @@
package org.apache.giraph.benchmark;
import java.io.IOException;
-import org.apache.giraph.graph.EdgeListVertex;
+import org.apache.giraph.vertex.EdgeListVertex;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
/**
* Same benchmark code as {@link PageRankBenchmark}, but uses
- * {@link org.apache.giraph.graph.EdgeListVertex} implementation rather than
- * {@link org.apache.giraph.graph.HashMapVertex}
+ * {@link org.apache.giraph.vertex.EdgeListVertex} implementation rather than
+ * {@link org.apache.giraph.vertex.HashMapVertex}
*/
public class EdgeListVertexPageRankBenchmark extends EdgeListVertex<
LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> {
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java
index b1c07cd..ef37540 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexPageRankBenchmark.java
@@ -17,7 +17,7 @@
*/
package org.apache.giraph.benchmark;
-import org.apache.giraph.graph.HashMapVertex;
+import org.apache.giraph.vertex.HashMapVertex;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
@@ -26,7 +26,7 @@ import java.io.IOException;
/**
* Same benchmark code as {@link PageRankBenchmark}, but uses
* {@link HashMapVertex} implementation rather than
- * {@link org.apache.giraph.graph.EdgeListVertex}
+ * {@link org.apache.giraph.vertex.EdgeListVertex}
*/
public class HashMapVertexPageRankBenchmark extends HashMapVertex<
LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> {
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexShortestPathsBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexShortestPathsBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexShortestPathsBenchmark.java
index 68dff09..a9d6deb 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexShortestPathsBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/HashMapVertexShortestPathsBenchmark.java
@@ -18,7 +18,7 @@
package org.apache.giraph.benchmark;
-import org.apache.giraph.graph.HashMapVertex;
+import org.apache.giraph.vertex.HashMapVertex;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
@@ -27,7 +27,7 @@ import java.io.IOException;
/**
* Same benchmark code as {@link ShortestPathsBenchmark}, but uses
* {@link HashMapVertex} implementation rather than
- * {@link org.apache.giraph.graph.EdgeListVertex}
+ * {@link org.apache.giraph.vertex.EdgeListVertex}
*/
public class HashMapVertexShortestPathsBenchmark extends HashMapVertex<
LongWritable, DoubleWritable, DoubleWritable, DoubleWritable> {
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphEdgeListVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphEdgeListVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphEdgeListVertexPageRankBenchmark.java
index c4df68a..712ca99 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphEdgeListVertexPageRankBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphEdgeListVertexPageRankBenchmark.java
@@ -18,7 +18,7 @@
package org.apache.giraph.benchmark;
-import org.apache.giraph.graph.MultiGraphEdgeListVertex;
+import org.apache.giraph.vertex.MultiGraphEdgeListVertex;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphRepresentativeVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphRepresentativeVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphRepresentativeVertexPageRankBenchmark.java
index aa8ee6b..9628832 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphRepresentativeVertexPageRankBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/MultiGraphRepresentativeVertexPageRankBenchmark.java
@@ -18,7 +18,7 @@
package org.apache.giraph.benchmark;
-import org.apache.giraph.graph.MultiGraphRepresentativeVertex;
+import org.apache.giraph.vertex.MultiGraphRepresentativeVertex;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
index 75e4d8f..3ef471a 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
@@ -24,10 +24,10 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.examples.DoubleSumCombiner;
+import org.apache.giraph.combiner.DoubleSumCombiner;
import org.apache.giraph.graph.GiraphJob;
-import org.apache.giraph.io.PseudoRandomEdgeInputFormat;
-import org.apache.giraph.io.PseudoRandomVertexInputFormat;
+import org.apache.giraph.io.formats.PseudoRandomEdgeInputFormat;
+import org.apache.giraph.io.formats.PseudoRandomVertexInputFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java
index d6a8cb7..57cc201 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankComputation.java
@@ -17,7 +17,7 @@
*/
package org.apache.giraph.benchmark;
-import org.apache.giraph.graph.MutableVertex;
+import org.apache.giraph.vertex.MutableVertex;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/RandomMessageBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/RandomMessageBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/RandomMessageBenchmark.java
index 604c4a9..d48aa6d 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/RandomMessageBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/RandomMessageBenchmark.java
@@ -25,11 +25,11 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.giraph.aggregators.LongSumAggregator;
import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.graph.DefaultMasterCompute;
-import org.apache.giraph.graph.EdgeListVertex;
+import org.apache.giraph.master.DefaultMasterCompute;
+import org.apache.giraph.vertex.EdgeListVertex;
import org.apache.giraph.graph.GiraphJob;
-import org.apache.giraph.graph.WorkerContext;
-import org.apache.giraph.io.PseudoRandomVertexInputFormat;
+import org.apache.giraph.worker.WorkerContext;
+import org.apache.giraph.io.formats.PseudoRandomVertexInputFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DoubleWritable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/RepresentativeVertexPageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/RepresentativeVertexPageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/RepresentativeVertexPageRankBenchmark.java
index f12f8ac..331ae41 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/RepresentativeVertexPageRankBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/RepresentativeVertexPageRankBenchmark.java
@@ -19,13 +19,13 @@
package org.apache.giraph.benchmark;
import java.io.IOException;
-import org.apache.giraph.graph.RepresentativeVertex;
+import org.apache.giraph.vertex.RepresentativeVertex;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
/**
* Same benchmark code as {@link PageRankBenchmark}, but uses
- * {@link org.apache.giraph.graph.RepresentativeVertex}
+ * {@link org.apache.giraph.vertex.RepresentativeVertex}
* implementation.
*/
public class RepresentativeVertexPageRankBenchmark extends
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsBenchmark.java
index 21fc0ac..888532d 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsBenchmark.java
@@ -24,10 +24,10 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.examples.MinimumDoubleCombiner;
-import org.apache.giraph.graph.EdgeListVertex;
+import org.apache.giraph.combiner.MinimumDoubleCombiner;
+import org.apache.giraph.vertex.EdgeListVertex;
import org.apache.giraph.graph.GiraphJob;
-import org.apache.giraph.io.PseudoRandomVertexInputFormat;
+import org.apache.giraph.io.formats.PseudoRandomVertexInputFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsComputation.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsComputation.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsComputation.java
index 1a5128d..19f5575 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsComputation.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/ShortestPathsComputation.java
@@ -19,7 +19,7 @@
package org.apache.giraph.benchmark;
import org.apache.giraph.graph.Edge;
-import org.apache.giraph.graph.Vertex;
+import org.apache.giraph.vertex.Vertex;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
http://git-wip-us.apache.org/repos/asf/giraph/blob/1684891e/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java b/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
index 9e43ca6..65c3ebc 100644
--- a/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
@@ -20,7 +20,6 @@ package org.apache.giraph.bsp;
import java.io.IOException;
-import org.apache.giraph.graph.BspUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;