You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ed...@apache.org on 2016/03/15 18:40:33 UTC
[1/5] git commit: updated refs/heads/trunk to fafecee
Repository: giraph
Updated Branches:
refs/heads/trunk c6af3ed8a -> fafecee71
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java b/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java
index 1062479..37876d4 100644
--- a/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java
+++ b/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java
@@ -575,7 +575,7 @@ else[HADOOP_NON_SECURE]*/
if (getConfiguration().hasEdgeInputFormat()) {
// Move edges from temporary storage to their source vertices.
- getServerData().getPartitionStore().moveEdgesToVertices();
+ getServerData().getEdgeStore().moveEdgesToVertices();
}
// Generate the partition stats for the input superstep and process
@@ -783,7 +783,7 @@ else[HADOOP_NON_SECURE]*/
globalCommHandler.finishSuperstep(workerAggregatorRequestProcessor);
MessageStore<I, Writable> incomingMessageStore =
- getServerData().getPartitionStore().getIncomingMessageStore();
+ getServerData().getIncomingMessageStore();
if (incomingMessageStore instanceof AsyncMessageStoreWrapper) {
((AsyncMessageStoreWrapper) incomingMessageStore).waitToComplete();
}
@@ -1010,13 +1010,16 @@ else[HADOOP_NON_SECURE]*/
long nextPrintMsecs = System.currentTimeMillis() + 15000;
int partitionIndex = 0;
int numPartitions = getPartitionStore().getNumPartitions();
+ LOG.info("Write thread started!");
while (true) {
Partition<I, V, E> partition =
getPartitionStore().getNextPartition();
+ LOG.info("partition is : " + partition);
if (partition == null) {
break;
}
+ LOG.info("start to write a partition");
long verticesWritten = 0;
for (Vertex<I, V, E> vertex : partition) {
vertexWriter.writeVertex(vertex);
@@ -1033,6 +1036,7 @@ else[HADOOP_NON_SECURE]*/
nextPrintMsecs = System.currentTimeMillis() + 15000;
nextPrintVertices = verticesWritten + 250000;
}
+ LOG.info("done writing vertices");
if (verticesWritten >= nextUpdateProgressVertices) {
WorkerProgress.get().addVerticesStored(
@@ -1288,10 +1292,12 @@ else[HADOOP_NON_SECURE]*/
workerContext.write(checkpointOutputStream);
getContext().progress();
+ // TODO: checkpointing messages along with vertices to avoid multiple loads
+ // of a partition when out-of-core is enabled.
for (Integer partitionId : getPartitionStore().getPartitionIds()) {
// write messages
checkpointOutputStream.writeInt(partitionId);
- getServerData().getPartitionStore().getCurrentMessageStore()
+ getServerData().getCurrentMessageStore()
.writePartition(checkpointOutputStream, partitionId);
getContext().progress();
@@ -1539,9 +1545,11 @@ else[HADOOP_NON_SECURE]*/
getConfiguration().updateSuperstepClasses(superstepClasses);
getServerData().resetMessageStores();
+ // TODO: checkpointing messages along with vertices to avoid multiple
+ // loads of a partition when out-of-core is enabled.
for (int i = 0; i < partitions; i++) {
int partitionId = checkpointStream.readInt();
- getServerData().getPartitionStore().getCurrentMessageStore()
+ getServerData().getCurrentMessageStore()
.readFieldsForPartition(checkpointStream, partitionId);
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/test/java/org/apache/giraph/comm/RequestFailureTest.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/comm/RequestFailureTest.java b/giraph-core/src/test/java/org/apache/giraph/comm/RequestFailureTest.java
index 2785217..c88aac7 100644
--- a/giraph-core/src/test/java/org/apache/giraph/comm/RequestFailureTest.java
+++ b/giraph-core/src/test/java/org/apache/giraph/comm/RequestFailureTest.java
@@ -106,15 +106,14 @@ public class RequestFailureTest {
private void checkResult(int numRequests) throws IOException {
// Check the output
Iterable<IntWritable> vertices =
- serverData.getPartitionStore().getIncomingMessageStore()
- .getPartitionDestinationVertices(0);
+ serverData.getIncomingMessageStore().getPartitionDestinationVertices(0);
int keySum = 0;
int messageSum = 0;
for (IntWritable vertexId : vertices) {
keySum += vertexId.get();
Iterable<IntWritable> messages =
- serverData.getPartitionStore().<IntWritable>getIncomingMessageStore()
- .getVertexMessages(vertexId);
+ serverData.<IntWritable>getIncomingMessageStore().getVertexMessages(
+ vertexId);
synchronized (messages) {
for (IntWritable message : messages) {
messageSum += message.get();
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/test/java/org/apache/giraph/comm/RequestTest.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/comm/RequestTest.java b/giraph-core/src/test/java/org/apache/giraph/comm/RequestTest.java
index 2688da1..0462770 100644
--- a/giraph-core/src/test/java/org/apache/giraph/comm/RequestTest.java
+++ b/giraph-core/src/test/java/org/apache/giraph/comm/RequestTest.java
@@ -178,15 +178,14 @@ public class RequestTest {
// Check the output
Iterable<IntWritable> vertices =
- serverData.getPartitionStore().getIncomingMessageStore()
- .getPartitionDestinationVertices(0);
+ serverData.getIncomingMessageStore().getPartitionDestinationVertices(0);
int keySum = 0;
int messageSum = 0;
for (IntWritable vertexId : vertices) {
keySum += vertexId.get();
Iterable<IntWritable> messages =
- serverData.getPartitionStore().<IntWritable>getIncomingMessageStore()
- .getVertexMessages(vertexId);
+ serverData.<IntWritable>getIncomingMessageStore().getVertexMessages(
+ vertexId);
synchronized (messages) {
for (IntWritable message : messages) {
messageSum += message.get();
@@ -224,15 +223,14 @@ public class RequestTest {
// Check the output
Iterable<IntWritable> vertices =
- serverData.getPartitionStore().getIncomingMessageStore()
- .getPartitionDestinationVertices(0);
+ serverData.getIncomingMessageStore().getPartitionDestinationVertices(0);
int keySum = 0;
int messageSum = 0;
for (IntWritable vertexId : vertices) {
keySum += vertexId.get();
Iterable<IntWritable> messages =
- serverData.getPartitionStore().<IntWritable>getIncomingMessageStore()
- .getVertexMessages(vertexId);
+ serverData.<IntWritable>getIncomingMessageStore().getVertexMessages(
+ vertexId);
synchronized (messages) {
for (IntWritable message : messages) {
messageSum += message.get();
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/test/java/org/apache/giraph/partition/TestPartitionStores.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/partition/TestPartitionStores.java b/giraph-core/src/test/java/org/apache/giraph/partition/TestPartitionStores.java
index 249a337..7893940 100644
--- a/giraph-core/src/test/java/org/apache/giraph/partition/TestPartitionStores.java
+++ b/giraph-core/src/test/java/org/apache/giraph/partition/TestPartitionStores.java
@@ -23,6 +23,7 @@ import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
import org.apache.giraph.bsp.BspService;
import org.apache.giraph.bsp.CentralizedServiceWorker;
+import org.apache.giraph.comm.ServerData;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
@@ -32,7 +33,7 @@ import org.apache.giraph.graph.Vertex;
import org.apache.giraph.io.formats.IdWithValueTextOutputFormat;
import org.apache.giraph.io.formats.IntIntNullTextVertexInputFormat;
import org.apache.giraph.io.formats.JsonLongDoubleFloatDoubleVertexInputFormat;
-import org.apache.giraph.ooc.DiskBackedPartitionStore;
+import org.apache.giraph.ooc.data.DiskBackedPartitionStore;
import org.apache.giraph.utils.InternalVertexRunner;
import org.apache.giraph.utils.NoOpComputation;
import org.apache.giraph.utils.UnsafeByteArrayInputStream;
@@ -50,7 +51,6 @@ import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ExecutorCompletionService;
@@ -98,18 +98,14 @@ public class TestPartitionStores {
public void setUp() {
GiraphConfiguration configuration = new GiraphConfiguration();
configuration.setComputationClass(MyComputation.class);
- conf = new ImmutableClassesGiraphConfiguration<IntWritable, IntWritable,
- NullWritable>(configuration);
+ conf = new ImmutableClassesGiraphConfiguration<>(configuration);
context = Mockito.mock(Mapper.Context.class);
}
@Test
public void testSimplePartitionStore() {
- CentralizedServiceWorker<IntWritable, IntWritable, NullWritable>
- serviceWorker = Mockito.mock(CentralizedServiceWorker.class);
PartitionStore<IntWritable, IntWritable, NullWritable>
- partitionStore = new SimplePartitionStore<IntWritable, IntWritable,
- NullWritable>(conf, context, serviceWorker);
+ partitionStore = new SimplePartitionStore<>(conf, context);
testReadWrite(partitionStore, conf);
partitionStore.shutdown();
}
@@ -166,10 +162,15 @@ public class TestPartitionStores {
serviceWorker = Mockito.mock(CentralizedServiceWorker.class);
Mockito.when(serviceWorker.getSuperstep()).thenReturn(
BspService.INPUT_SUPERSTEP);
-
- PartitionStore<IntWritable, IntWritable, NullWritable> partitionStore =
- new DiskBackedPartitionStore<IntWritable, IntWritable, NullWritable>(
- conf, context, serviceWorker);
+ ServerData<IntWritable, IntWritable, NullWritable>
+ serverData = new ServerData<>(serviceWorker, conf, context);
+ Mockito.when(serviceWorker.getServerData()).thenReturn(serverData);
+
+ DiskBackedPartitionStore<IntWritable, IntWritable, NullWritable>
+ partitionStore =
+ (DiskBackedPartitionStore<IntWritable, IntWritable, NullWritable>)
+ serverData.getPartitionStore();
+ partitionStore.initialize();
testReadWrite(partitionStore, conf);
partitionStore.shutdown();
FileUtils.deleteDirectory(directory);
@@ -185,16 +186,19 @@ public class TestPartitionStores {
CentralizedServiceWorker<IntWritable, IntWritable, NullWritable>
serviceWorker = Mockito.mock(CentralizedServiceWorker.class);
-
Mockito.when(serviceWorker.getSuperstep()).thenReturn(
BspService.INPUT_SUPERSTEP);
-
- PartitionStore<IntWritable, IntWritable, NullWritable> partitionStore =
- new DiskBackedPartitionStore<IntWritable, IntWritable, NullWritable>(
- conf, context, serviceWorker);
+ ServerData<IntWritable, IntWritable, NullWritable>
+ serverData = new ServerData<>(serviceWorker, conf, context);
+ Mockito.when(serviceWorker.getServerData()).thenReturn(serverData);
+
+ DiskBackedPartitionStore<IntWritable, IntWritable, NullWritable>
+ partitionStore =
+ (DiskBackedPartitionStore<IntWritable, IntWritable, NullWritable>)
+ serverData.getPartitionStore();
+ partitionStore.initialize();
testReadWrite(partitionStore, conf);
partitionStore.shutdown();
-
FileUtils.deleteDirectory(directory);
}
@@ -275,18 +279,18 @@ public class TestPartitionStores {
GiraphConstants.STATIC_GRAPH.set(conf, true);
testMultiThreaded();
}
-
+/*
@Test
public void testDiskBackedPartitionStoreAdaptiveOOC() throws Exception {
GiraphConstants.STATIC_GRAPH.set(conf, true);
testMultiThreaded();
}
-
+*/
private void testMultiThreaded() throws Exception {
final AtomicInteger vertexCounter = new AtomicInteger(0);
ExecutorService pool = Executors.newFixedThreadPool(NUM_OF_THREADS);
ExecutorCompletionService<Boolean> executor =
- new ExecutorCompletionService<Boolean>(pool);
+ new ExecutorCompletionService<>(pool);
File directory = Files.createTempDir();
GiraphConstants.PARTITIONS_DIRECTORY.set(
@@ -298,21 +302,25 @@ public class TestPartitionStores {
Mockito.when(serviceWorker.getSuperstep()).thenReturn(
BspService.INPUT_SUPERSTEP);
-
- PartitionStore<IntWritable, IntWritable, NullWritable> store =
- new DiskBackedPartitionStore<IntWritable, IntWritable, NullWritable>(
- conf, context, serviceWorker);
+ ServerData<IntWritable, IntWritable, NullWritable>
+ serverData = new ServerData<>(serviceWorker, conf, context);
+ Mockito.when(serviceWorker.getServerData()).thenReturn(serverData);
+
+ DiskBackedPartitionStore<IntWritable, IntWritable, NullWritable>
+ store =
+ (DiskBackedPartitionStore<IntWritable, IntWritable, NullWritable>)
+ serverData.getPartitionStore();
store.initialize();
// Create a new Graph in memory using multiple threads
for (int i = 0; i < NUM_OF_THREADS; ++i) {
- List<Integer> partitionIds = new ArrayList<Integer>();
+ List<Integer> partitionIds = new ArrayList<>();
for (int id = i; id < NUM_OF_PARTITIONS; id += NUM_OF_THREADS) {
partitionIds.add(id);
}
Worker worker =
new Worker(vertexCounter, store, partitionIds, conf);
- executor.submit(worker, new Boolean(true));
+ executor.submit(worker, true);
}
for (int i = 0; i < NUM_OF_THREADS; ++i)
executor.take();
@@ -341,11 +349,8 @@ public class TestPartitionStores {
for (int i = 0; i < NUM_OF_PARTITIONS; ++i) {
partition = store.getNextPartition();
assert partition != null;
- Iterator<Vertex<IntWritable, IntWritable, NullWritable>> vertexes =
- partition.iterator();
- while (vertexes.hasNext()) {
- Vertex<IntWritable, IntWritable, NullWritable> v = vertexes.next();
+ for (Vertex<IntWritable, IntWritable, NullWritable> v : partition) {
totalValues += v.getId().get();
}
store.putPartition(partition);
@@ -358,7 +363,6 @@ public class TestPartitionStores {
private Partition<IntWritable, IntWritable, NullWritable>
getPartition(PartitionStore<IntWritable, IntWritable,
NullWritable> partitionStore, int partitionId) {
- partitionStore.startIteration();
Partition p;
Partition result = null;
while ((p = partitionStore.getNextPartition()) != null) {
@@ -403,8 +407,11 @@ public class TestPartitionStores {
partitionStore.addPartition(createPartition(conf, 3, v5));
partitionStore.addPartition(createPartition(conf, 4, v7));
+ partitionStore.startIteration();
getPartition(partitionStore, 1);
+ partitionStore.startIteration();
getPartition(partitionStore, 2);
+ partitionStore.startIteration();
partitionStore.removePartition(3);
getPartition(partitionStore, 4);
@@ -435,16 +442,12 @@ public class TestPartitionStores {
* @param expected expected results
*/
private void checkResults(Iterable<String> results, String[] expected) {
- Iterator<String> result = results.iterator();
-
- assert results != null;
- while(result.hasNext()) {
- String resultStr = result.next();
+ for (String str : results) {
boolean found = false;
- for (int j = 0; j < expected.length; ++j) {
- if (expected[j].equals(resultStr)) {
+ for (String expectedStr : expected) {
+ if (expectedStr.equals(str)) {
found = true;
}
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-examples/src/test/java/org/apache/giraph/TestOutOfCore.java
----------------------------------------------------------------------
diff --git a/giraph-examples/src/test/java/org/apache/giraph/TestOutOfCore.java b/giraph-examples/src/test/java/org/apache/giraph/TestOutOfCore.java
index ad9ba6f..6fdfc75 100644
--- a/giraph-examples/src/test/java/org/apache/giraph/TestOutOfCore.java
+++ b/giraph-examples/src/test/java/org/apache/giraph/TestOutOfCore.java
@@ -18,23 +18,14 @@
package org.apache.giraph;
-import org.apache.giraph.bsp.CentralizedServiceWorker;
-import org.apache.giraph.combiner.DoubleSumMessageCombiner;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.examples.GeneratedVertexReader;
import org.apache.giraph.examples.SimplePageRankComputation;
import org.apache.giraph.examples.SimplePageRankComputation.SimplePageRankVertexInputFormat;
import org.apache.giraph.examples.SimplePageRankComputation.SimplePageRankVertexOutputFormat;
-import org.apache.giraph.graph.BasicComputation;
-import org.apache.giraph.graph.Vertex;
+
import org.apache.giraph.job.GiraphJob;
-import org.apache.giraph.ooc.CheckMemoryCallable;
-import org.apache.giraph.ooc.DiskBackedPartitionStore;
-import org.apache.giraph.ooc.MemoryEstimator;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.LongWritable;
import org.junit.Test;
import java.io.IOException;
@@ -47,40 +38,14 @@ import static org.junit.Assert.assertTrue;
*/
public class TestOutOfCore extends BspCase {
final static int NUM_PARTITIONS = 32;
- final static int MAX_NUM_PARTITIONS_IN_MEMORY = 16;
- final static int FAIR_NUM_PARTITIONS_IN_MEMORY = 12;
+ final static int NUM_PARTITIONS_IN_MEMORY = 16;
public TestOutOfCore() {
super(TestOutOfCore.class.getName());
}
- public static class TestMemoryEstimator implements MemoryEstimator {
- private DiskBackedPartitionStore partitionStore;
- @Override
- public void initialize(CentralizedServiceWorker serviceWorker) {
- partitionStore =
- (DiskBackedPartitionStore) serviceWorker.getPartitionStore();
- }
-
- @Override
- public double freeMemoryMB() {
- int numPartitions = partitionStore.getNumPartitionInMemory();
- if (numPartitions > MAX_NUM_PARTITIONS_IN_MEMORY) {
- return 1;
- } else if (numPartitions > FAIR_NUM_PARTITIONS_IN_MEMORY) {
- return 10;
- } else {
- return 40;
- }
- }
-
- @Override
- public double maxMemoryMB() {
- return 100;
- }
- }
/**
- * Run a job that tests the adaptive out-of-core mechanism
+ * Run a job that tests the fixed out-of-core mechanism
*
* @throws IOException
* @throws ClassNotFoundException
@@ -99,13 +64,12 @@ public class TestOutOfCore extends BspCase {
SimplePageRankComputation.SimplePageRankMasterCompute.class);
GiraphConstants.USER_PARTITION_COUNT.set(conf, NUM_PARTITIONS);
GiraphConstants.USE_OUT_OF_CORE_GRAPH.set(conf, true);
- GiraphConstants.OUT_OF_CORE_MEM_ESTIMATOR
- .set(conf, TestMemoryEstimator.class);
- CheckMemoryCallable.CHECK_MEMORY_INTERVAL.set(conf, 5);
+ GiraphConstants.MAX_PARTITIONS_IN_MEMORY.set(conf, NUM_PARTITIONS_IN_MEMORY);
GiraphConstants.NUM_COMPUTE_THREADS.set(conf, 8);
GiraphConstants.NUM_INPUT_THREADS.set(conf, 8);
GiraphConstants.NUM_OOC_THREADS.set(conf, 4);
GiraphConstants.NUM_OUTPUT_THREADS.set(conf, 8);
+ GiraphConstants.PARTITIONS_DIRECTORY.set(conf, "disk0,disk1,disk2");
GiraphJob job = prepareJob(getCallingMethodName(), conf,
getTempPath(getCallingMethodName()));
// Overwrite the number of vertices set in BspCase
[5/5] git commit: updated refs/heads/trunk to fafecee
Posted by ed...@apache.org.
New out-of-core infrastructure (first patch including fixed out-of-core mechanism)
Summary: This is a re-design of out-of-core mechanism. The new implementation allows for much more intelligent partition scheduling and IO.
Test Plan:
mvn clean verify
Reviewers: maja.kabiljo, sergey.edunov, avery.ching, dionysis.logothetis
Reviewed By: dionysis.logothetis
Differential Revision: https://reviews.facebook.net/D54549
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/fafecee7
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/fafecee7
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/fafecee7
Branch: refs/heads/trunk
Commit: fafecee712bc9b2ce8ef081d8170cdf99c48288b
Parents: c6af3ed
Author: Sergey Edunov <ed...@fb.com>
Authored: Tue Mar 15 10:40:20 2016 -0700
Committer: Sergey Edunov <ed...@fb.com>
Committed: Tue Mar 15 10:40:20 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/giraph/comm/ServerData.java | 213 +-
.../giraph/comm/messages/MessageData.java | 82 -
.../comm/messages/SimpleMessageStore.java | 3 +-
.../out_of_core/DiskBackedMessageStore.java | 6 +-
.../primitives/IdByteArrayMessageStore.java | 4 +-
.../primitives/IdOneMessagePerVertexStore.java | 3 +-
.../primitives/IntByteArrayMessageStore.java | 4 +-
.../primitives/IntFloatMessageStore.java | 3 +-
.../primitives/LongDoubleMessageStore.java | 3 +-
.../long_id/LongAbstractMessageStore.java | 3 +-
.../NettyWorkerClientRequestProcessor.java | 2 +-
.../SendPartitionCurrentMessagesRequest.java | 4 +-
.../comm/requests/SendWorkerEdgesRequest.java | 2 +-
.../requests/SendWorkerMessagesRequest.java | 4 +-
.../SendWorkerOneMessageToManyRequest.java | 13 +-
.../org/apache/giraph/conf/GiraphConstants.java | 11 -
.../apache/giraph/edge/AbstractEdgeStore.java | 22 +-
.../java/org/apache/giraph/edge/EdgeStore.java | 21 +-
.../apache/giraph/graph/GraphTaskManager.java | 3 +-
.../giraph/ooc/AdaptiveOutOfCoreEngine.java | 284 ---
.../apache/giraph/ooc/CheckMemoryCallable.java | 478 ----
.../giraph/ooc/DiskBackedPartitionStore.java | 2149 ------------------
.../apache/giraph/ooc/FixedOutOfCoreEngine.java | 147 ++
.../giraph/ooc/FixedOutOfCoreIOScheduler.java | 211 ++
.../apache/giraph/ooc/JVMMemoryEstimator.java | 45 -
.../org/apache/giraph/ooc/MemoryEstimator.java | 44 -
.../org/apache/giraph/ooc/OutOfCoreEngine.java | 187 +-
.../apache/giraph/ooc/OutOfCoreIOCallable.java | 90 +
.../giraph/ooc/OutOfCoreIOCallableFactory.java | 184 ++
.../apache/giraph/ooc/OutOfCoreIOScheduler.java | 105 +
.../giraph/ooc/OutOfCoreProcessorCallable.java | 170 --
.../giraph/ooc/data/DiskBackedEdgeStore.java | 207 ++
.../giraph/ooc/data/DiskBackedMessageStore.java | 297 +++
.../ooc/data/DiskBackedPartitionStore.java | 469 ++++
.../giraph/ooc/data/MetaPartitionManager.java | 947 ++++++++
.../giraph/ooc/data/OutOfCoreDataManager.java | 385 ++++
.../apache/giraph/ooc/data/package-info.java | 22 +
.../org/apache/giraph/ooc/io/IOCommand.java | 65 +
.../giraph/ooc/io/LoadPartitionIOCommand.java | 92 +
.../giraph/ooc/io/StoreDataBufferIOCommand.java | 89 +
.../ooc/io/StoreIncomingMessageIOCommand.java | 60 +
.../giraph/ooc/io/StorePartitionIOCommand.java | 76 +
.../org/apache/giraph/ooc/io/WaitIOCommand.java | 58 +
.../org/apache/giraph/ooc/io/package-info.java | 21 +
.../apache/giraph/partition/PartitionData.java | 116 -
.../apache/giraph/partition/PartitionStore.java | 206 +-
.../giraph/partition/SimplePartitionStore.java | 46 +-
.../apache/giraph/worker/BspServiceWorker.java | 16 +-
.../apache/giraph/comm/RequestFailureTest.java | 7 +-
.../org/apache/giraph/comm/RequestTest.java | 14 +-
.../giraph/partition/TestPartitionStores.java | 81 +-
.../java/org/apache/giraph/TestOutOfCore.java | 46 +-
52 files changed, 4103 insertions(+), 3717 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/ServerData.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/ServerData.java b/giraph-core/src/main/java/org/apache/giraph/comm/ServerData.java
index b177446..be34820 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/ServerData.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/ServerData.java
@@ -31,20 +31,32 @@ import com.google.common.collect.Maps;
import org.apache.giraph.bsp.CentralizedServiceWorker;
import org.apache.giraph.comm.aggregators.AllAggregatorServerData;
import org.apache.giraph.comm.aggregators.OwnerAggregatorServerData;
+import org.apache.giraph.comm.messages.MessageStore;
+import org.apache.giraph.comm.messages.MessageStoreFactory;
+import org.apache.giraph.comm.messages.queue.AsyncMessageStoreWrapper;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.edge.EdgeStore;
+import org.apache.giraph.edge.EdgeStoreFactory;
import org.apache.giraph.graph.Vertex;
import org.apache.giraph.graph.VertexMutations;
import org.apache.giraph.graph.VertexResolver;
-import org.apache.giraph.ooc.DiskBackedPartitionStore;
+import org.apache.giraph.ooc.data.DiskBackedEdgeStore;
+import org.apache.giraph.ooc.data.DiskBackedMessageStore;
+import org.apache.giraph.ooc.data.DiskBackedPartitionStore;
+import org.apache.giraph.ooc.FixedOutOfCoreEngine;
+import org.apache.giraph.ooc.OutOfCoreEngine;
import org.apache.giraph.partition.Partition;
import org.apache.giraph.partition.PartitionStore;
import org.apache.giraph.partition.SimplePartitionStore;
+import org.apache.giraph.utils.ReflectionUtils;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.log4j.Logger;
+import static org.apache.giraph.conf.GiraphConstants.MESSAGE_STORE_FACTORY_CLASS;
+
/**
* Anything that the server stores
*
@@ -56,12 +68,26 @@ import org.apache.log4j.Logger;
public class ServerData<I extends WritableComparable,
V extends Writable, E extends Writable> {
/** Class logger */
- private static final Logger LOG =
- Logger.getLogger(ServerData.class);
+ private static final Logger LOG = Logger.getLogger(ServerData.class);
/** Configuration */
private final ImmutableClassesGiraphConfiguration<I, V, E> conf;
/** Partition store for this worker. */
private volatile PartitionStore<I, V, E> partitionStore;
+ /** Edge store for this worker. */
+ private final EdgeStore<I, V, E> edgeStore;
+ /** Message store factory */
+ private final MessageStoreFactory<I, Writable, MessageStore<I, Writable>>
+ messageStoreFactory;
+ /**
+ * Message store for incoming messages (messages which will be consumed
+ * in the next super step)
+ */
+ private volatile MessageStore<I, Writable> incomingMessageStore;
+ /**
+ * Message store for current messages (messages which we received in
+ * previous super step and which will be consumed in current super step)
+ */
+ private volatile MessageStore<I, Writable> currentMessageStore;
/**
* Map of partition ids to vertex mutations from other workers. These are
* mutations that should be applied before execution of *current* super step.
@@ -81,7 +107,7 @@ public class ServerData<I extends WritableComparable,
ConcurrentMap<I, VertexMutations<I, V, E>>>
partitionMutations = Maps.newConcurrentMap();
/**
- * Holds aggregtors which current worker owns from current superstep
+ * Holds aggregators which current worker owns from current superstep
*/
private final OwnerAggregatorServerData ownerAggregatorData;
/**
@@ -100,6 +126,8 @@ public class ServerData<I extends WritableComparable,
/** Job context (for progress) */
private final Mapper<?, ?, ?, ?>.Context context;
+ /** Out-of-core engine */
+ private final OutOfCoreEngine oocEngine;
/**
* Constructor.
@@ -114,13 +142,32 @@ public class ServerData<I extends WritableComparable,
Mapper<?, ?, ?, ?>.Context context) {
this.serviceWorker = service;
this.conf = conf;
+ this.messageStoreFactory = createMessageStoreFactory();
+ EdgeStoreFactory<I, V, E> edgeStoreFactory = conf.createEdgeStoreFactory();
+ edgeStoreFactory.initialize(service, conf, context);
+ EdgeStore<I, V, E> inMemoryEdgeStore = edgeStoreFactory.newStore();
+ PartitionStore<I, V, E> inMemoryPartitionStore =
+ new SimplePartitionStore<I, V, E>(conf, context);
if (GiraphConstants.USE_OUT_OF_CORE_GRAPH.get(conf)) {
+ int maxPartitionsInMemory =
+ GiraphConstants.MAX_PARTITIONS_IN_MEMORY.get(conf);
+ if (maxPartitionsInMemory == 0) {
+ throw new IllegalStateException("ServerData: Adaptive " +
+ "out-of-core engine is not supported yet! Number of partitions in" +
+ " memory should be greater than 0.");
+ } else {
+ oocEngine = new FixedOutOfCoreEngine(conf, service,
+ maxPartitionsInMemory);
+ }
partitionStore =
- new DiskBackedPartitionStore<I, V, E>(conf, context,
- getServiceWorker());
+ new DiskBackedPartitionStore<I, V, E>(inMemoryPartitionStore,
+ conf, context, service, oocEngine);
+ edgeStore =
+ new DiskBackedEdgeStore<I, V, E>(inMemoryEdgeStore, conf, oocEngine);
} else {
- partitionStore =
- new SimplePartitionStore<I, V, E>(conf, context, getServiceWorker());
+ partitionStore = inMemoryPartitionStore;
+ edgeStore = inMemoryEdgeStore;
+ oocEngine = null;
}
ownerAggregatorData = new OwnerAggregatorServerData(context);
allAggregatorData = new AllAggregatorServerData(context, conf);
@@ -128,6 +175,42 @@ public class ServerData<I extends WritableComparable,
}
/**
+ * Decide which message store should be used for current application,
+ * and create the factory for that store
+ *
+ * @return Message store factory
+ */
+ private MessageStoreFactory<I, Writable, MessageStore<I, Writable>>
+ createMessageStoreFactory() {
+ Class<? extends MessageStoreFactory> messageStoreFactoryClass =
+ MESSAGE_STORE_FACTORY_CLASS.get(conf);
+
+ MessageStoreFactory messageStoreFactoryInstance =
+ ReflectionUtils.newInstance(messageStoreFactoryClass);
+ messageStoreFactoryInstance.initialize(serviceWorker, conf);
+
+ return messageStoreFactoryInstance;
+ }
+
+ /**
+ * Return the out-of-core engine for this worker.
+ *
+ * @return The out-of-core engine
+ */
+ public OutOfCoreEngine getOocEngine() {
+ return oocEngine;
+ }
+
+ /**
+ * Return the edge store for this worker.
+ *
+ * @return The edge store
+ */
+ public EdgeStore<I, V, E> getEdgeStore() {
+ return edgeStore;
+ }
+
+ /**
* Return the partition store for this worker.
*
* @return The partition store
@@ -137,21 +220,103 @@ public class ServerData<I extends WritableComparable,
}
/**
+ * Get message store for incoming messages (messages which will be consumed
+ * in the next super step)
+ *
+ * @param <M> Message data
+ * @return Incoming message store
+ */
+ public <M extends Writable> MessageStore<I, M> getIncomingMessageStore() {
+ return (MessageStore<I, M>) incomingMessageStore;
+ }
+
+ /**
+ * Get message store for current messages (messages which we received in
+ * previous super step and which will be consumed in current super step)
+ *
+ * @param <M> Message data
+ * @return Current message store
+ */
+ public <M extends Writable> MessageStore<I, M> getCurrentMessageStore() {
+ return (MessageStore<I, M>) currentMessageStore;
+ }
+
+ /**
* Re-initialize message stores.
* Discards old values if any.
+ *
* @throws IOException
*/
public void resetMessageStores() throws IOException {
- getPartitionStore().resetMessageStores();
- currentWorkerToWorkerMessages =
- Collections.synchronizedList(new ArrayList<Writable>());
- incomingWorkerToWorkerMessages =
- Collections.synchronizedList(new ArrayList<Writable>());
+ if (currentMessageStore != null) {
+ currentMessageStore.clearAll();
+ currentMessageStore = null;
+ }
+ if (incomingMessageStore != null) {
+ incomingMessageStore.clearAll();
+ incomingMessageStore = null;
+ }
+ prepareSuperstep();
}
- /** Prepare for next super step */
+ /** Prepare for next superstep */
public void prepareSuperstep() {
- partitionStore.prepareSuperstep();
+ if (currentMessageStore != null) {
+ try {
+ currentMessageStore.clearAll();
+ } catch (IOException e) {
+ throw new IllegalStateException(
+ "Failed to clear previous message store");
+ }
+ }
+
+ MessageStore<I, Writable> nextCurrentMessageStore;
+ MessageStore<I, Writable> nextIncomingMessageStore;
+ MessageStore<I, Writable> messageStore;
+
+ // First create the necessary in-memory message stores. If out-of-core
+ // mechanism is enabled, we wrap the in-memory message stores within
+ // disk-backed messages stores.
+ if (incomingMessageStore != null) {
+ nextCurrentMessageStore = incomingMessageStore;
+ } else {
+ messageStore = messageStoreFactory.newStore(
+ conf.getIncomingMessageClasses());
+ if (oocEngine == null) {
+ nextCurrentMessageStore = messageStore;
+ } else {
+ nextCurrentMessageStore = new DiskBackedMessageStore<>(
+ conf, messageStore,
+ conf.getIncomingMessageClasses().useMessageCombiner(),
+ serviceWorker.getSuperstep());
+ }
+ }
+
+ messageStore = messageStoreFactory.newStore(
+ conf.getOutgoingMessageClasses());
+ if (oocEngine == null) {
+ nextIncomingMessageStore = messageStore;
+ } else {
+ nextIncomingMessageStore = new DiskBackedMessageStore<>(
+ conf, messageStore,
+ conf.getOutgoingMessageClasses().useMessageCombiner(),
+ serviceWorker.getSuperstep() + 1);
+ }
+
+ // If out-of-core engine is enabled, we avoid overlapping of out-of-core
+ // decisions with change of superstep. This avoidance is done to simplify
+ // the design and reduce excessive use of synchronization primitives.
+ if (oocEngine != null) {
+ oocEngine.getSuperstepLock().writeLock().lock();
+ }
+ currentMessageStore = nextCurrentMessageStore;
+ incomingMessageStore = nextIncomingMessageStore;
+ if (oocEngine != null) {
+ oocEngine.getMetaPartitionManager().resetMessages();
+ oocEngine.getSuperstepLock().writeLock().unlock();
+ }
+ currentMessageStore.finalizeStore();
+
currentWorkerToWorkerMessages = incomingWorkerToWorkerMessages;
incomingWorkerToWorkerMessages =
Collections.synchronizedList(new ArrayList<Writable>());
@@ -252,8 +417,7 @@ public class ServerData<I extends WritableComparable,
VertexMutations<I, V, E> vertexMutations = entry.getValue();
Vertex<I, V, E> vertex = vertexResolver.resolve(vertexId,
originalVertex, vertexMutations,
- getPartitionStore().getCurrentMessageStore()
- .hasMessagesForVertex(entry.getKey()));
+ getCurrentMessageStore().hasMessagesForVertex(entry.getKey()));
if (LOG.isDebugEnabled()) {
LOG.debug("resolvePartitionMutations: Resolved vertex index " +
@@ -269,8 +433,7 @@ public class ServerData<I extends WritableComparable,
} else if (originalVertex != null) {
partition.removeVertex(vertexId);
try {
- getPartitionStore().getCurrentMessageStore()
- .clearVertexMessages(vertexId);
+ getCurrentMessageStore().clearVertexMessages(vertexId);
} catch (IOException e) {
throw new IllegalStateException("resolvePartitionMutations: " +
"Caught IOException while clearing messages for a deleted " +
@@ -283,7 +446,7 @@ public class ServerData<I extends WritableComparable,
// Keep track of vertices which are not here in the partition, but have
// received messages
- Iterable<I> destinations = getPartitionStore().getCurrentMessageStore().
+ Iterable<I> destinations = getCurrentMessageStore().
getPartitionDestinationVertices(partitionId);
if (!Iterables.isEmpty(destinations)) {
for (I vertexId : destinations) {
@@ -307,4 +470,14 @@ public class ServerData<I extends WritableComparable,
}
}
}
+
+ /**
+ * In case of async message store we have to wait for all messages
+ * to be processed before going into next superstep.
+ */
+ public void waitForComplete() {
+ if (incomingMessageStore instanceof AsyncMessageStoreWrapper) {
+ ((AsyncMessageStoreWrapper) incomingMessageStore).waitToComplete();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/messages/MessageData.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/MessageData.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/MessageData.java
deleted file mode 100644
index f974823..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/MessageData.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.comm.messages;
-
-import org.apache.giraph.utils.VertexIdMessages;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import java.io.IOException;
-
-/**
- * Structure that keeps message information.
- *
- * @param <I> Vertex id
- */
-public interface MessageData<I extends WritableComparable> {
- /**
- * Get message store for incoming messages (messages which will be consumed
- * in the next super step)
- *
- * @param <M> Message data type
- * @return Incoming message store
- */
- <M extends Writable> MessageStore<I, M> getIncomingMessageStore();
-
- /**
- * Get message store for current messages (messages which we received in
- * previous super step and which will be consumed in current super step)
- *
- * @param <M> Message data type
- * @return Current message store
- */
- <M extends Writable> MessageStore<I, M> getCurrentMessageStore();
-
- /**
- * Re-initialize message stores.
- * Discards old values if any.
-
- * @throws IOException
- */
- void resetMessageStores() throws IOException;
-
- /**
- * Adds messages for partition to current message store
- *
- * @param <M> Message data type
- * @param partitionId Id of partition
- * @param messages Collection of vertex ids and messages we want to add
- * @throws IOException
- */
- <M extends Writable> void addPartitionCurrentMessages(
- int partitionId, VertexIdMessages<I, M> messages)
- throws IOException;
-
- /**
- * Adds messages for partition to incoming message store
- *
- * @param <M> Message data type
- * @param partitionId Id of partition
- * @param messages Collection of vertex ids and messages we want to add
- * @throws IOException
- */
- <M extends Writable> void addPartitionIncomingMessages(
- int partitionId, VertexIdMessages<I, M> messages)
- throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/messages/SimpleMessageStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/SimpleMessageStore.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/SimpleMessageStore.java
index a1d3625..054302d 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/SimpleMessageStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/messages/SimpleMessageStore.java
@@ -212,7 +212,8 @@ public abstract class SimpleMessageStore<I extends WritableComparable,
@Override
public boolean hasMessagesForPartition(int partitionId) {
- return !map.get(partitionId).isEmpty();
+ ConcurrentMap<I, T> partitionMessages = map.get(partitionId);
+ return partitionMessages != null && !partitionMessages.isEmpty();
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/messages/out_of_core/DiskBackedMessageStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/out_of_core/DiskBackedMessageStore.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/out_of_core/DiskBackedMessageStore.java
index b28d15b..0d7009b 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/out_of_core/DiskBackedMessageStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/messages/out_of_core/DiskBackedMessageStore.java
@@ -132,8 +132,10 @@ public class DiskBackedMessageStore<I extends WritableComparable,
@Override
public boolean hasMessagesForPartition(int partitionId) {
- return !Iterables
- .isEmpty(getMessageStore(partitionId).getDestinationVertices());
+ PartitionDiskBackedMessageStore<I, M> partitionMessages =
+ getMessageStore(partitionId);
+ return partitionMessages != null && !Iterables
+ .isEmpty(partitionMessages.getDestinationVertices());
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IdByteArrayMessageStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IdByteArrayMessageStore.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IdByteArrayMessageStore.java
index e1e7a3f..2e39857 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IdByteArrayMessageStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IdByteArrayMessageStore.java
@@ -187,7 +187,9 @@ public class IdByteArrayMessageStore<I extends WritableComparable,
@Override
public boolean hasMessagesForPartition(int partitionId) {
- return map.get(partitionId).size() != 0;
+ Basic2ObjectMap<I, DataInputOutput> partitionMessages =
+ map.get(partitionId);
+ return partitionMessages != null && partitionMessages.size() != 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IdOneMessagePerVertexStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IdOneMessagePerVertexStore.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IdOneMessagePerVertexStore.java
index b172d24..42fe992 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IdOneMessagePerVertexStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IdOneMessagePerVertexStore.java
@@ -163,7 +163,8 @@ public class IdOneMessagePerVertexStore<I extends WritableComparable,
@Override
public boolean hasMessagesForPartition(int partitionId) {
- return map.get(partitionId).size() != 0;
+ Basic2ObjectMap<I, M> partitionMessages = map.get(partitionId);
+ return partitionMessages != null && partitionMessages.size() != 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IntByteArrayMessageStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IntByteArrayMessageStore.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IntByteArrayMessageStore.java
index 2fbc35c..4c363f3 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IntByteArrayMessageStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IntByteArrayMessageStore.java
@@ -178,7 +178,9 @@ public class IntByteArrayMessageStore<M extends Writable>
@Override
public boolean hasMessagesForPartition(int partitionId) {
- return !map.get(partitionId).isEmpty();
+ Int2ObjectOpenHashMap<DataInputOutput> partitionMessages =
+ map.get(partitionId);
+ return partitionMessages != null && !partitionMessages.isEmpty();
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IntFloatMessageStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IntFloatMessageStore.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IntFloatMessageStore.java
index 3186224..280f5b9 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IntFloatMessageStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/IntFloatMessageStore.java
@@ -139,7 +139,8 @@ public class IntFloatMessageStore
@Override
public boolean hasMessagesForPartition(int partitionId) {
- return !map.get(partitionId).isEmpty();
+ Int2FloatOpenHashMap partitionMessages = map.get(partitionId);
+ return partitionMessages != null && !partitionMessages.isEmpty();
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/LongDoubleMessageStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/LongDoubleMessageStore.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/LongDoubleMessageStore.java
index 6278c16..d8a3fde 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/LongDoubleMessageStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/LongDoubleMessageStore.java
@@ -140,7 +140,8 @@ public class LongDoubleMessageStore
@Override
public boolean hasMessagesForPartition(int partitionId) {
- return !map.get(partitionId).isEmpty();
+ Long2DoubleOpenHashMap partitionMessages = map.get(partitionId);
+ return partitionMessages != null && !partitionMessages.isEmpty();
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/long_id/LongAbstractMessageStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/long_id/LongAbstractMessageStore.java b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/long_id/LongAbstractMessageStore.java
index d8e5246..a0c977e 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/long_id/LongAbstractMessageStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/messages/primitives/long_id/LongAbstractMessageStore.java
@@ -102,7 +102,8 @@ public abstract class LongAbstractMessageStore<M extends Writable, T>
@Override
public boolean hasMessagesForPartition(int partitionId) {
- return !map.get(partitionId).isEmpty();
+ Long2ObjectOpenHashMap<T> partitionMessages = map.get(partitionId);
+ return partitionMessages != null && !partitionMessages.isEmpty();
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyWorkerClientRequestProcessor.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyWorkerClientRequestProcessor.java b/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyWorkerClientRequestProcessor.java
index e9b072a..1cd1bd6 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyWorkerClientRequestProcessor.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyWorkerClientRequestProcessor.java
@@ -210,7 +210,7 @@ public class NettyWorkerClientRequestProcessor<I extends WritableComparable,
Partition<I, V, E> partition) {
final int partitionId = partition.getId();
MessageStore<I, Writable> messageStore =
- serverData.getPartitionStore().getCurrentMessageStore();
+ serverData.getCurrentMessageStore();
ByteArrayVertexIdMessages<I, Writable> vertexIdMessages =
new ByteArrayVertexIdMessages<I, Writable>(
configuration.createOutgoingMessageValueFactory());
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendPartitionCurrentMessagesRequest.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendPartitionCurrentMessagesRequest.java b/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendPartitionCurrentMessagesRequest.java
index ab66aa3..b59d0cf 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendPartitionCurrentMessagesRequest.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendPartitionCurrentMessagesRequest.java
@@ -86,8 +86,8 @@ public class SendPartitionCurrentMessagesRequest<I extends WritableComparable,
@Override
public void doRequest(ServerData<I, V, E> serverData) {
try {
- serverData.getPartitionStore()
- .addPartitionCurrentMessages(partitionId, vertexIdMessageMap);
+ serverData.<M>getCurrentMessageStore().addPartitionMessages(partitionId,
+ vertexIdMessageMap);
} catch (IOException e) {
throw new RuntimeException("doRequest: Got IOException ", e);
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerEdgesRequest.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerEdgesRequest.java b/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerEdgesRequest.java
index aeda197..00cf6ef 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerEdgesRequest.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerEdgesRequest.java
@@ -69,7 +69,7 @@ public class SendWorkerEdgesRequest<I extends WritableComparable,
iterator = partitionVertexData.getIterator();
while (iterator.hasNext()) {
iterator.next();
- serverData.getPartitionStore()
+ serverData.getEdgeStore()
.addPartitionEdges(iterator.getCurrentFirst(),
iterator.getCurrentSecond());
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerMessagesRequest.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerMessagesRequest.java b/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerMessagesRequest.java
index e9be327..6953998 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerMessagesRequest.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerMessagesRequest.java
@@ -71,8 +71,8 @@ public class SendWorkerMessagesRequest<I extends WritableComparable,
while (iterator.hasNext()) {
iterator.next();
try {
- serverData.getPartitionStore().
- addPartitionIncomingMessages(iterator.getCurrentFirst(),
+ serverData.getIncomingMessageStore().
+ addPartitionMessages(iterator.getCurrentFirst(),
iterator.getCurrentSecond());
} catch (IOException e) {
throw new RuntimeException("doRequest: Got IOException ", e);
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerOneMessageToManyRequest.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerOneMessageToManyRequest.java b/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerOneMessageToManyRequest.java
index aeb1b1d..f8d0473 100644
--- a/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerOneMessageToManyRequest.java
+++ b/giraph-core/src/main/java/org/apache/giraph/comm/requests/SendWorkerOneMessageToManyRequest.java
@@ -27,6 +27,7 @@ import java.util.Map.Entry;
import org.apache.giraph.bsp.CentralizedServiceWorker;
import org.apache.giraph.comm.ServerData;
+import org.apache.giraph.comm.messages.MessageStore;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.partition.PartitionOwner;
import org.apache.giraph.utils.ByteArrayOneMessageToManyIds;
@@ -93,11 +94,10 @@ public class SendWorkerOneMessageToManyRequest<I extends WritableComparable,
@Override
public void doRequest(ServerData serverData) {
try {
- if (serverData.getPartitionStore().getIncomingMessageStore()
- .isPointerListEncoding()) {
+ MessageStore<I, M> messageStore = serverData.getIncomingMessageStore();
+ if (messageStore.isPointerListEncoding()) {
// if message store is pointer list based then send data as is
- serverData.getPartitionStore()
- .addPartitionIncomingMessages(-1, oneMessageToManyIds);
+ messageStore.addPartitionMessages(-1, oneMessageToManyIds);
} else { // else split the data per partition and send individually
CentralizedServiceWorker<I, ?, ?> serviceWorker =
serverData.getServiceWorker();
@@ -144,9 +144,8 @@ public class SendWorkerOneMessageToManyRequest<I extends WritableComparable,
for (Entry<Integer, ByteArrayVertexIdMessages> idMsgs :
partitionIdMsgs.entrySet()) {
if (!idMsgs.getValue().isEmpty()) {
- serverData.getPartitionStore()
- .addPartitionIncomingMessages(idMsgs.getKey(),
- idMsgs.getValue());
+ serverData.getIncomingMessageStore().addPartitionMessages(
+ idMsgs.getKey(), idMsgs.getValue());
}
}
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
index 8ad3767..4787d37 100644
--- a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
+++ b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
@@ -71,8 +71,6 @@ import org.apache.giraph.mapping.translate.TranslateEdge;
import org.apache.giraph.master.DefaultMasterCompute;
import org.apache.giraph.master.MasterCompute;
import org.apache.giraph.master.MasterObserver;
-import org.apache.giraph.ooc.JVMMemoryEstimator;
-import org.apache.giraph.ooc.MemoryEstimator;
import org.apache.giraph.partition.GraphPartitionerFactory;
import org.apache.giraph.partition.HashPartitionerFactory;
import org.apache.giraph.partition.Partition;
@@ -976,15 +974,6 @@ public interface GiraphConstants {
new BooleanConfOption("giraph.useOutOfCoreGraph", false,
"Enable out-of-core graph.");
- /**
- * Memory estimator class used in adaptive out-of-core mechanism for deciding
- * when data should go to disk.
- */
- ClassConfOption<MemoryEstimator> OUT_OF_CORE_MEM_ESTIMATOR =
- ClassConfOption.create("giraph.outOfCoreMemoryEstimator",
- JVMMemoryEstimator.class, MemoryEstimator.class,
- "Memory estimator class used for out-of-core decisions");
-
/** Number of threads participating in swapping graph/messages to disk. */
IntConfOption NUM_OOC_THREADS =
new IntConfOption("giraph.numOutOfCoreThreads", 1,
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/edge/AbstractEdgeStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/AbstractEdgeStore.java b/giraph-core/src/main/java/org/apache/giraph/edge/AbstractEdgeStore.java
index 9609047..0f3d668 100644
--- a/giraph-core/src/main/java/org/apache/giraph/edge/AbstractEdgeStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/edge/AbstractEdgeStore.java
@@ -42,6 +42,8 @@ import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
+import static com.google.common.base.Preconditions.checkState;
+
/**
* Basic implementation of edges store, extended this to easily define simple
* and primitive edge stores
@@ -157,7 +159,7 @@ public abstract class AbstractEdgeStore<I extends WritableComparable,
getPartitionEdgesIterator(Map<K, OutEdges<I, E>> partitionEdges);
@Override
- public boolean hasPartitionEdges(int partitionId) {
+ public boolean hasEdgesForPartition(int partitionId) {
return transientEdges.containsKey(partitionId);
}
@@ -165,21 +167,21 @@ public abstract class AbstractEdgeStore<I extends WritableComparable,
public void writePartitionEdgeStore(int partitionId, DataOutput output)
throws IOException {
Map<K, OutEdges<I, E>> edges = transientEdges.remove(partitionId);
- output.writeInt(edges.size());
- for (Map.Entry<K, OutEdges<I, E>> edge : edges.entrySet()) {
- writeVertexKey(edge.getKey(), output);
- edge.getValue().write(output);
+ if (edges != null) {
+ output.writeInt(edges.size());
+ for (Map.Entry<K, OutEdges<I, E>> edge : edges.entrySet()) {
+ writeVertexKey(edge.getKey(), output);
+ edge.getValue().write(output);
+ }
}
}
@Override
public void readPartitionEdgeStore(int partitionId, DataInput input)
throws IOException {
- if (transientEdges.containsKey(partitionId)) {
- throw new IllegalStateException("readPartitionEdgeStore: reading a " +
- "partition that is already there in the partition store " +
- "(impossible)");
- }
+ checkState(!transientEdges.containsKey(partitionId),
+ "readPartitionEdgeStore: reading a partition that is already there in" +
+ " the partition store (impossible)");
Map<K, OutEdges<I, E>> partitionEdges = getPartitionEdges(partitionId);
int numEntries = input.readInt();
for (int i = 0; i < numEntries; ++i) {
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java b/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java
index 1c9d85f..f485042 100644
--- a/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java
@@ -46,21 +46,12 @@ public interface EdgeStore<I extends WritableComparable,
/**
* Move all edges from temporary storage to their source vertices.
- * Note: this method is not thread-safe.
+ * Note: this method is not thread-safe and is called once all vertices and
+ * edges are read in INPUT_SUPERSTEP.
*/
void moveEdgesToVertices();
/**
- * Whether the store contains edges for the given partition.
- * Note: This method is thread-safe
- *
- * @param partitionId Partition id under query
- * @return true if the store has any edge for the given partition, false
- * otherwise
- */
- boolean hasPartitionEdges(int partitionId);
-
- /**
* Deserialize the edges of a given partition, and removes the associated data
* from the store.
* Note: This method is not thread-safe (i.e. should not be called for the
@@ -84,4 +75,12 @@ public interface EdgeStore<I extends WritableComparable,
*/
void readPartitionEdgeStore(int partitionId, DataInput input)
throws IOException;
+
+ /**
+ * Check if edge store has edge for a given partition
+ *
+ * @param partitionId Id of partition
+ * @return True iff edge store have messages for the given partition
+ */
+ boolean hasEdgesForPartition(int partitionId);
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java b/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java
index 3c09957..62a87de 100644
--- a/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java
@@ -332,8 +332,7 @@ end[PURE_YARN]*/
prepareForSuperstep(graphState);
context.progress();
MessageStore<I, Writable> messageStore =
- serviceWorker.getServerData().getPartitionStore()
- .getCurrentMessageStore();
+ serviceWorker.getServerData().getCurrentMessageStore();
int numPartitions = serviceWorker.getPartitionStore().getNumPartitions();
int numThreads = Math.min(numComputeThreads, numPartitions);
if (LOG.isInfoEnabled()) {
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/AdaptiveOutOfCoreEngine.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/AdaptiveOutOfCoreEngine.java b/giraph-core/src/main/java/org/apache/giraph/ooc/AdaptiveOutOfCoreEngine.java
deleted file mode 100644
index d5b0e20..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/AdaptiveOutOfCoreEngine.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.ooc;
-
-import com.google.common.collect.Lists;
-import org.apache.giraph.bsp.CentralizedServiceWorker;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.utils.CallableFactory;
-import org.apache.giraph.utils.LogStacktraceCallable;
-import org.apache.giraph.utils.ThreadUtils;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.log4j.Logger;
-
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Adaptive out-of-core mechanism. This mechanism spawns two types of threads:
- * 1) check-memory thread, which periodically monitors the amount of available
- * memory and decides whether data should go on disk. This threads is
- * basically the brain behind the out-of-core mechanism, commands
- * "out-of-core processor threads" (type 2 thread below) to move
- * appropriate data to disk,
- * 2) out-of-core processor threads. This is a team of threads responsible for
- * offloading appropriate data to disk. "check-memory thread" decides on
- * which data should go to disk, and "out-of-core processor threads" do the
- * offloading.
- *
- * @param <I> Vertex id
- * @param <V> Vertex value
- * @param <E> Edge data
- */
-public class AdaptiveOutOfCoreEngine<I extends WritableComparable,
- V extends Writable, E extends Writable> implements
- OutOfCoreEngine<I, V, E> {
- /** Class logger. */
- private static final Logger LOG =
- Logger.getLogger(AdaptiveOutOfCoreEngine.class);
-
- // ---- Synchronization Variables ----
- /** Barrier to coordinate check-memory and OOC-processing threads */
- private final CyclicBarrier gate;
- /**
- * Signal to determine whether OOC processing threads are done processing OOC
- * requests
- */
- private final CyclicBarrier doneOocSignal;
- /** Signal to determine whether the computation is terminated */
- private final CountDownLatch doneCompute;
- /** Finisher signal to OOC processing threads */
- private volatile boolean done;
-
- // ---- OOC Commands ----
- /**
- * List of partitions that are on disk, and their loaded *vertices* during
- * INPUT_SUPERSTEP are ready to be flushed to disk
- */
- private final BlockingQueue<Integer> partitionsWithInputVertices;
- /**
- * List of partitions that are on disk, and their loaded *edges* during
- * INPUT_SUPERSTEP are ready to be flushed to disk
- */
- private final BlockingQueue<Integer> partitionsWithInputEdges;
- /**
- * List of partitions that are on disk, and their message buffers (either
- * messages for current superstep, or incoming messages for next superstep)
- * are ready to be flushed to disk
- */
- private final BlockingQueue<Integer> partitionsWithPendingMessages;
- /** Number of partitions to be written to disk */
- private final AtomicInteger numPartitionsToSpill;
-
- /** Executor service for check memory thread */
- private ExecutorService checkMemoryExecutor;
- /** Executor service for out-of-core processor threads */
- private ExecutorService outOfCoreProcessorExecutor;
-
- /** Configuration */
- private ImmutableClassesGiraphConfiguration<I, V, E> conf;
- /** Worker */
- private final CentralizedServiceWorker<I, V, E> serviceWorker;
-
- /** Cached value for number of out-of-core threads specified by user */
- private int numOocThreads;
-
- /** Result of check-memory thread (to be checked for graceful termination) */
- private Future<Void> checkMemoryResult;
- /**
- * Results of out-of-core processor threads (to be checked for graceful
- * termination)
- */
- private List<Future<Void>> oocProcessorResults;
-
- /**
- * Creates an instance of adaptive mechanism
- * @param conf Configuration
- * @param serviceWorker Worker service
- */
- public AdaptiveOutOfCoreEngine(ImmutableClassesGiraphConfiguration conf,
- CentralizedServiceWorker<I, V, E> serviceWorker) {
- this.conf = conf;
- this.serviceWorker = serviceWorker;
-
- this.numOocThreads = conf.getNumOocThreads();
- this.gate = new CyclicBarrier(numOocThreads + 1);
- this.doneOocSignal = new CyclicBarrier(numOocThreads + 1);
- this.doneCompute = new CountDownLatch(1);
- this.done = false;
- this.partitionsWithInputVertices = new ArrayBlockingQueue<Integer>(100);
- this.partitionsWithInputEdges = new ArrayBlockingQueue<Integer>(100);
- this.partitionsWithPendingMessages = new ArrayBlockingQueue<Integer>(100);
- this.numPartitionsToSpill = new AtomicInteger(0);
- }
-
- @Override
- public void initialize() {
- if (LOG.isInfoEnabled()) {
- LOG.info("initialize: initializing out-of-core engine");
- }
- CallableFactory<Void> checkMemoryCallableFactory =
- new CallableFactory<Void>() {
- @Override
- public Callable<Void> newCallable(int callableId) {
- return new CheckMemoryCallable<I, V, E>(
- AdaptiveOutOfCoreEngine.this, conf, serviceWorker);
- }
- };
- checkMemoryExecutor = Executors.newSingleThreadExecutor(
- ThreadUtils.createThreadFactory("check-memory"));
- checkMemoryResult = checkMemoryExecutor.submit(new LogStacktraceCallable<>(
- checkMemoryCallableFactory.newCallable(0)));
-
- CallableFactory<Void> outOfCoreProcessorCallableFactory =
- new CallableFactory<Void>() {
- @Override
- public Callable<Void> newCallable(int callableId) {
- return new OutOfCoreProcessorCallable<I, V, E>(
- AdaptiveOutOfCoreEngine.this, serviceWorker);
- }
- };
- outOfCoreProcessorExecutor =
- Executors.newFixedThreadPool(numOocThreads,
- ThreadUtils.createThreadFactory("ooc-%d"));
- oocProcessorResults = Lists.newArrayListWithCapacity(numOocThreads);
- for (int i = 0; i < numOocThreads; ++i) {
- Future<Void> future = outOfCoreProcessorExecutor.submit(
- new LogStacktraceCallable<>(
- outOfCoreProcessorCallableFactory.newCallable(i)));
- oocProcessorResults.add(future);
- }
- }
-
- @Override
- public void shutdown() {
- doneCompute.countDown();
- checkMemoryExecutor.shutdown();
- if (checkMemoryResult.isCancelled()) {
- throw new IllegalStateException(
- "shutdown: memory check thread did not " + "terminate gracefully!");
- }
- outOfCoreProcessorExecutor.shutdown();
- for (int i = 0; i < numOocThreads; ++i) {
- if (oocProcessorResults.get(i).isCancelled()) {
- throw new IllegalStateException("shutdown: out-of-core processor " +
- "thread " + i + " did not terminate gracefully.");
- }
- }
- }
-
- /**
- * @return the latch that signals whether the whole computation is done
- */
- public CountDownLatch getDoneCompute() {
- return doneCompute;
- }
-
- /**
- * @return whether the computation is done
- */
- public boolean isDone() {
- return done;
- }
-
- /**
- * @return list of partitions that have large enough buffers of vertices read
- * in INPUT_SUPERSTEP.
- */
- public BlockingQueue<Integer> getPartitionsWithInputVertices() {
- return partitionsWithInputVertices;
- }
-
- /**
- * @return list of partitions that have large enough buffers of edges read
- * in INPUT_SUPERSTEP.
- */
- public BlockingQueue<Integer> getPartitionsWithInputEdges() {
- return partitionsWithInputEdges;
- }
-
- /**
- * @return list of partitions that have large enough message buffers.
- */
- public BlockingQueue<Integer> getPartitionsWithPendingMessages() {
- return partitionsWithPendingMessages;
- }
-
- /**
- * @return number of partitions to spill to disk
- */
- public AtomicInteger getNumPartitionsToSpill() {
- return numPartitionsToSpill;
- }
-
- /**
- * Wait on gate with which OOC processor threads are notified to execute
- * commands provided by brain (memory-check thread).
- *
- * @throws BrokenBarrierException
- * @throws InterruptedException
- */
- public void waitOnGate() throws BrokenBarrierException, InterruptedException {
- gate.await();
- }
-
- /**
- * Reset the gate for reuse.
- */
- public void resetGate() {
- gate.reset();
- }
-
- /**
- * Wait on signal from all OOC processor threads that the offloading of data
- * is complete.
- *
- * @throws BrokenBarrierException
- * @throws InterruptedException
- */
- public void waitOnOocSignal()
- throws BrokenBarrierException, InterruptedException {
- doneOocSignal.await();
- }
-
- /**
- * Reset the completion signal of OOC processor threads for reuse.
- */
- public void resetOocSignal() {
- doneOocSignal.reset();
- }
-
- /**
- * Set the computation as done (i.e. setting the state that determines the
- * whole computation is done).
- */
- public void setDone() {
- done = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/CheckMemoryCallable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/CheckMemoryCallable.java b/giraph-core/src/main/java/org/apache/giraph/ooc/CheckMemoryCallable.java
deleted file mode 100644
index 2b2c990..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/CheckMemoryCallable.java
+++ /dev/null
@@ -1,478 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.ooc;
-
-import org.apache.giraph.bsp.BspService;
-import org.apache.giraph.bsp.CentralizedServiceWorker;
-import org.apache.giraph.conf.FloatConfOption;
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.conf.IntConfOption;
-import org.apache.giraph.utils.PairList;
-import org.apache.giraph.utils.ReflectionUtils;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.log4j.Logger;
-
-import java.util.Stack;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Adaptive out-of-core mechanism brain. This class provides one thread per
- * worker that periodically checks the free memory on the worker and compares it
- * with total amount of memory given to that worker to run the job. The period
- * at which the thread checks for the memory is specified by the user. Also,
- * user can specify the fraction of memory where anytime free memory is less
- * than that fraction of total memory, actions would be taken to free up space
- * in memory (this fraction is called LOW_FREE_MEMORY_FRACTION). Also, user can
- * specify another fraction of available memory where memory pressure is fair
- * and some of the data on disk (if there is any) can be brought back to memory
- * again (this fraction is called FAIR_FREE_MEMORY_FRACTION).
- *
- * In the adaptive out-of-core mechanism, if amount of free memory becomes less
- * than LOW_FREE_MEMORY_FRACTION, some data are being considered as potentials
- * to transfer to disk. These data can be in the following categories:
- * 1) Vertex buffers read in INPUT_SUPERSTEP. These are vertex input splits
- * read for a partition that is out-of-core and PartitionStore holds these
- * vertex buffers in in-memory buffers (and postpone their merge with the
- * actual partition until the partition is loaded back in memory).
- * 2) Edge buffers read in INPUT_SUPERSTEP. These are similar buffers to
- * vertex buffers, but they keep edge data in INPUT_SUPERSTEP.
- * 3) Partitions.
- *
- * This brain prefers the first two categories in INPUT_SUPERSTEP as long as
- * size of buffers are large enough that it is worth writing them to disk. In
- * case where brain decides on spilling partitions to disk, the brain decides
- * only on the "number of partitions" to spill to disk. It is "out-of-core
- * processor threads" responsibility to find that many partitions to spill to
- * disk. The number of partitions to spill is a fraction of number of partitions
- * currently in memory. It is recommended that this fraction be equal to
- * subtraction of LOW_FREE_MEMORY_FRACTION from FAIR_FREE_MEMORY_FRACTION. Here
- * is an example to clarify on this recommendation. Assume
- * LOW_FREE_MEMORY_FRACTION is 5% and FAIR_FREE_MEMORY_FRACTION is 15%. Also
- * assume that the partitions are similar in their memory footprint (which is a
- * valid assumption for most of the partitioning techniques). If free memory is
- * a bit less than 5% of total available memory, if we offload 10%
- * (15% - 5% = 10%), then the amount of free memory will increase to a bit less
- * than 15% of total available memory.
- *
- * @param <I> Vertex id
- * @param <V> Vertex value
- * @param <E> Edge value
- */
-public class CheckMemoryCallable<I extends WritableComparable,
- V extends Writable, E extends Writable> implements Callable<Void> {
- /**
- * Lowest free memory fraction to start doing necessary actions to go
- * out-of-core.
- */
- public static final FloatConfOption LOW_FREE_MEMORY_FRACTION =
- new FloatConfOption("giraph.lowFreeMemoryFraction", 0.1f,
- "If free memory fraction goes below this value, GC is called " +
- "manually and necessary actions are taken if we have to go " +
- "out-of-core");
- /**
- * Expected memory fraction to achieve after detecting that the job is running
- * low in memory. Basically, this memory fraction is the target to achieve
- * once we decide to offload data on disk.
- */
- public static final FloatConfOption MID_FREE_MEMORY_FRACTION =
- new FloatConfOption("giraph.midFreeMemoryFraction", 0.15f,
- "Once out-of-core mechanism decides to offload data on disk, it " +
- "offloads data on disk until free memory fraction reaches this " +
- "fraction.");
- /**
- * Memory fraction at which the job gets the best performance considering the
- * choice of GC strategy. It means, if the amount of free memory is more than
- * this fraction we will not see severe amount of GC calls.
- */
- public static final FloatConfOption FAIR_FREE_MEMORY_FRACTION =
- new FloatConfOption("giraph.fairFreeMemoryFraction", 0.3f,
- "The fraction of free memory at which the job shows the best GC " +
- "performance. This fraction might be dependent on GC strategy " +
- "used in running the job, but generally 0.3 is a reasonable " +
- "fraction for most strategies.");
- /**
- * Memory fraction at which the job has enough space so we can back off from
- * the last out-of-core decision, i.e. lazily bringing the last bunch of data
- * spilled to disk.
- */
- public static final FloatConfOption HIGH_FREE_MEMORY_FRACTION =
- new FloatConfOption("giraph.highFreeMemoryFraction", 0.4f,
- "Once free memory reaches at this fraction, last out-of-core " +
- "decision is lazily rolled back, i.e. we back off from " +
- "out-of-core.");
- /** Time interval at which checking memory is done periodically. */
- public static final IntConfOption CHECK_MEMORY_INTERVAL =
- new IntConfOption("giraph.checkMemoryInterval", 5000,
- "Time interval (in milliseconds) at which checking memory is done" +
- " to decide if there should be any out-of-core action.");
- /** Coefficient by which the number of partitions in memory changes. */
- public static final FloatConfOption OOC_GRAPH_MODIFICATION_COEFFICIENT =
- new FloatConfOption("giraph.graphPartitionModificationCoefficient", 0.3f,
- "If we decide to go out-of-core or back-off from out-of-core, this " +
- "is the multiplier by which the number of in-memory partitions" +
- "will change.");
-
- /** Class logger */
- private static final Logger LOG = Logger.getLogger(CheckMemoryCallable.class);
-
- /** Worker */
- private final CentralizedServiceWorker<I, V, E> serviceWorker;
- /** Partition store */
- private final DiskBackedPartitionStore<I, V, E> partitionStore;
-
- // ---- Cached Config Values ----
- /** Cached value of LOW_FREE_MEMORY_FRACTION */
- private float lowFreeMemoryFraction;
- /** Cached value for MID_FREE_MEMORY_FRACTION */
- private float midFreeMemoryFraction;
- /** Cached value of FAIR_FREE_MEMORY_FRACTION */
- private float fairFreeMemoryFraction;
- /** Cached value for HIGH_FREE_MEMORY_FRACTION */
- private float highFreeMemoryFraction;
- /** Cached value of CHECK_MEMORY_INTERVAL */
- private int checkInterval;
- /** Cached value for OOC_GRAPH_MODIFICATION_COEFFICIENT */
- private float modificationCoefficient;
-
- /** List of counts of number of partitions every time we shrink the store */
- private Stack<Integer> oocPartitionCounts;
- /** Memory estimator instance */
- private final MemoryEstimator memoryEstimator;
- /** Adaptive out-of-core engine */
- private final AdaptiveOutOfCoreEngine<I, V, E> oocEngine;
-
- /**
- * Constructor for check-memory thread.
- *
- * @param oocEngine out-of-core engine
- * @param conf job configuration
- * @param serviceWorker worker service
- */
- public CheckMemoryCallable(AdaptiveOutOfCoreEngine<I, V, E> oocEngine,
- ImmutableClassesGiraphConfiguration<I, V, E> conf,
- CentralizedServiceWorker<I, V, E> serviceWorker) {
- this.oocEngine = oocEngine;
- this.serviceWorker = serviceWorker;
- this.partitionStore =
- (DiskBackedPartitionStore<I, V, E>) serviceWorker.getPartitionStore();
-
- this.oocPartitionCounts = new Stack<>();
-
- this.lowFreeMemoryFraction = LOW_FREE_MEMORY_FRACTION.get(conf);
- this.midFreeMemoryFraction = MID_FREE_MEMORY_FRACTION.get(conf);
- this.fairFreeMemoryFraction = FAIR_FREE_MEMORY_FRACTION.get(conf);
- this.highFreeMemoryFraction = HIGH_FREE_MEMORY_FRACTION.get(conf);
- this.checkInterval = CHECK_MEMORY_INTERVAL.get(conf);
- this.modificationCoefficient = OOC_GRAPH_MODIFICATION_COEFFICIENT.get(conf);
-
- memoryEstimator = ReflectionUtils
- .newInstance(GiraphConstants.OUT_OF_CORE_MEM_ESTIMATOR.get(conf));
- }
-
- /**
- * Checks whether the available free memory is enough for an efficient
- * execution. If memory is limited, offload partitions to disk.
- * Also, if available memory is more than a threshold, loads partitions from
- * disk (if there is any) to memory.
- */
- @Override
- @edu.umd.cs.findbugs.annotations.SuppressWarnings("DM_GC")
- public Void call() {
- if (LOG.isInfoEnabled()) {
- LOG.info("call: check-memory thread started.");
- }
- memoryEstimator.initialize(serviceWorker);
- CountDownLatch doneCompute = oocEngine.getDoneCompute();
- while (doneCompute.getCount() != 0) {
- double maxMemory = memoryEstimator.maxMemoryMB();
- double freeMemory = memoryEstimator.freeMemoryMB();
- boolean gcDone = false;
- if (freeMemory < lowFreeMemoryFraction * maxMemory) {
- // This is typically a bad scenario where previous GCs were not
- // successful to free up enough memory. If we keep staying in this
- // situation, usually, either the computation slows down dramatically,
- // or the computation throws OOM error. So, we do GC manually, and
- // make sure that out-of-core is the solution to get out of this
- // situation.
- if (LOG.isInfoEnabled()) {
- LOG.info("call: Memory is very limited now. Calling GC manually. " +
- String.format("freeMemory = %.2fMB", freeMemory));
- }
- long gcStartTime = System.currentTimeMillis();
- System.gc();
- gcDone = true;
- freeMemory = memoryEstimator.freeMemoryMB();
- if (LOG.isInfoEnabled()) {
- LOG.info("call: GC is done. " + String
- .format("GC time = %.2f sec, and freeMemory = %.2fMB",
- (System.currentTimeMillis() - gcStartTime) / 1000.0,
- freeMemory));
- }
- }
- // If we have enough memory, we roll back the latest shrink in number of
- // partition slots.
- // If we do not have enough memory, but we are not in a bad scenario
- // either, we gradually increase the number of partition slots in memory.
- // If we are low in free memory, we first push unnecessary data to disk
- // and then push some partitions to disk if necessary.
- int numInMemory = partitionStore.getNumPartitionInMemory();
- int maxInMemory = partitionStore.getNumPartitionSlots();
- int numInTotal = partitionStore.getNumPartitions();
- if (freeMemory > highFreeMemoryFraction * maxMemory) {
- if (numInMemory >= maxInMemory && !oocPartitionCounts.isEmpty()) {
- partitionStore.increasePartitionSlots(oocPartitionCounts.pop());
- }
- } else if (freeMemory > fairFreeMemoryFraction * maxMemory) {
- // Only gradually increase the number of partition slots if all slots
- // are already used, and we have things out-of-core
- if (!oocPartitionCounts.isEmpty() || maxInMemory < numInTotal) {
- if (numInMemory >= maxInMemory) {
- partitionStore.increasePartitionSlots(1);
- if (!oocPartitionCounts.isEmpty()) {
- int num = oocPartitionCounts.pop();
- if (num > 1) {
- oocPartitionCounts.push(num - 1);
- }
- }
- }
- }
- } else if (gcDone && freeMemory < midFreeMemoryFraction * maxMemory) {
- BlockingQueue<Integer> partitionsWithInputVertices =
- oocEngine.getPartitionsWithInputVertices();
- BlockingQueue<Integer> partitionsWithInputEdges =
- oocEngine.getPartitionsWithInputEdges();
- BlockingQueue<Integer> partitionsWithPendingMessages =
- oocEngine.getPartitionsWithPendingMessages();
- AtomicInteger numPartitionsToSpill =
- oocEngine.getNumPartitionsToSpill();
- while (freeMemory < midFreeMemoryFraction * maxMemory) {
- // Offload input vertex buffer of OOC partitions if we are in
- // INPUT_SUPERSTEP
- if (serviceWorker.getSuperstep() == BspService.INPUT_SUPERSTEP) {
- // List of pairs (partitionId, approximate memory footprint of
- // vertex buffers of that partition).
- PairList<Integer, Integer> pairs =
- partitionStore.getOocPartitionIdsWithPendingInputVertices();
- freeMemory -= createCommands(pairs, partitionsWithInputVertices);
- }
-
- // Offload edge store of OOC partitions if we are in INPUT_SUPERSTEP
- if (freeMemory < midFreeMemoryFraction * maxMemory &&
- serviceWorker.getSuperstep() == BspService.INPUT_SUPERSTEP) {
- PairList<Integer, Integer> pairs =
- partitionStore.getOocPartitionIdsWithPendingInputEdges();
- freeMemory -= createCommands(pairs, partitionsWithInputEdges);
- }
-
- // Offload message buffers of OOC partitions if we are still low in
- // free memory
- if (freeMemory < midFreeMemoryFraction * maxMemory) {
- PairList<Integer, Integer> pairs =
- partitionStore.getOocPartitionIdsWithPendingMessages();
- freeMemory -= createCommands(pairs, partitionsWithPendingMessages);
- }
-
- // Offload partitions if we are still low in free memory
- if (freeMemory < midFreeMemoryFraction * maxMemory) {
- numPartitionsToSpill
- .set(getNextOocPartitionCount(freeMemory, maxMemory));
- }
-
- if (!partitionsWithInputVertices.isEmpty() ||
- !partitionsWithInputEdges.isEmpty() ||
- !partitionsWithPendingMessages.isEmpty() ||
- numPartitionsToSpill.get() != 0) {
- if (LOG.isInfoEnabled()) {
- LOG.info("call: signal out-of-core processor threads to start " +
- "offloading. These threads will spill vertex buffers of " +
- partitionsWithInputVertices.size() + " partitions, edge " +
- "buffers of " + partitionsWithInputEdges.size() +
- " partitions, pending message buffers of " +
- partitionsWithPendingMessages.size() + " partitions, and " +
- numPartitionsToSpill.get() + " whole partitions");
- }
- // Opening the gate for OOC processing threads to start spilling
- // data on disk
- try {
- oocEngine.waitOnGate();
- } catch (InterruptedException e) {
- throw new IllegalStateException("call: Caught " +
- "InterruptedException while opening the gate for OOC " +
- "processing threads");
- } catch (BrokenBarrierException e) {
- throw new IllegalStateException("call: Caught " +
- "BrokenBarrierException while opening the gate for OOC " +
- "processing threads");
- }
- oocEngine.resetGate();
-
- if (LOG.isInfoEnabled()) {
- LOG.info("call: waiting on OOC processors to finish offloading " +
- "data to disk");
- }
- // Wait until all OOC processing threads are done swapping data to
- // disk
- try {
- oocEngine.waitOnOocSignal();
- } catch (InterruptedException e) {
- throw new IllegalStateException("call: Caught " +
- "InterruptedException. Looks like memory check thread is " +
- "interrupted while waiting on OOC processing threads.");
- } catch (BrokenBarrierException e) {
- throw new IllegalStateException("call: Caught " +
- "BrokenBarrierException. Looks like some OOC processing " +
- "threads broke while writing data on disk.");
- }
- oocEngine.resetOocSignal();
- }
-
- gcDone = false;
- long gcStartTime = 0;
- if (freeMemory < midFreeMemoryFraction * maxMemory) {
- // Calling GC manually to actually free up the memory for data that
- // is offloaded to disk
- if (LOG.isInfoEnabled()) {
- LOG.info("call: calling GC manually to free up space for " +
- "recently offloaded data.");
- }
- gcStartTime = System.currentTimeMillis();
- System.gc();
- gcDone = true;
- }
- freeMemory = memoryEstimator.freeMemoryMB();
- if (LOG.isInfoEnabled()) {
- LOG.info("call: " +
- (gcDone ?
- ("GC is done. " + String.format("GC time = %.2f sec. ",
- (System.currentTimeMillis() - gcStartTime) / 1000.0)) :
- "") +
- "Finished offloading data to disk. " +
- String.format("freeMemory = %.2fMB", freeMemory));
- }
- }
- }
-
- // Either wait for the computation to be done, or the time interval passes
- try {
- doneCompute.await(checkInterval, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- throw new IllegalStateException("call: Caught InterruptedException " +
- "while waiting for computation to be done and/or " + checkInterval +
- "milliseconds passes.");
- }
- }
-
- // Setting 'done' before the gate here and checking 'done' in OOC processing
- // threads after the gate, guarantees that OOC processing threads see the
- // new value of done and terminate gracefully.
- oocEngine.setDone();
- try {
- oocEngine.waitOnGate();
- } catch (InterruptedException e) {
- throw new IllegalStateException("call: Caught InterruptedException " +
- "while waiting for the last time on gate in the current superstep");
- } catch (BrokenBarrierException e) {
- throw new IllegalStateException("call: Caught BrokenBarrierException " +
- "while waiting for the last time on gate in the current superstep");
- }
- return null;
- }
-
- /**
- * Returns the number of partitions that should go out-of-core at this point.
- *
- * @return number of partitions that should go out-of-core
- * @param freeMemory amount of free memory (in MB)
- * @param maxMemory amount of max memory (in MB)
- */
- private int getNextOocPartitionCount(double freeMemory, double maxMemory) {
- int numSlots = partitionStore.getNumPartitionSlots();
- if (numSlots == Integer.MAX_VALUE) {
- numSlots = partitionStore.getNumPartitions();
- partitionStore.setNumPartitionSlots(numSlots);
- }
-
- double freeFraction = freeMemory / maxMemory;
- double multiplier = Math.min(
- // User-specified favorable size to spill to disk
- modificationCoefficient,
- // Approximate fraction of current data to spill in order to reach the
- // fair fraction of free memory
- (fairFreeMemoryFraction - freeFraction) / (1 - freeFraction));
- int count = Math.max((int) (numSlots * multiplier), 1);
- if (count >= numSlots) {
- LOG.warn("getNextOocPartitionCount: Memory capacity is " +
- numSlots + " partitions, and OOC mechanism is " +
- "trying to put " + count + " partitions to disk. This is not " +
- "possible");
- // We should have at least one partition in memory
- count = numSlots - 1;
- if (count == 0) {
- LOG.warn("It seems that size of one partition is too large for the " +
- "available memory. Try to run the job with more partitions!");
- }
- }
- if (count != 0) {
- oocPartitionCounts.push(count);
- }
- return count;
- }
-
- /**
- * Generate commands for out-of-core processor threads based on the
- * (partitionId, memory foot-print) pairs we have on a particular type of data
- * (either vertex buffer, edge buffer, or message buffer).
- *
- * @param pairs list of pairs (partitionId, estimated memory foot-print that
- * is going to be reduced by offloading the particular data of a
- * partition)
- * @param commands commands to generate for out-of-core processor threads. a
- * command is a partition id, for which the appropriate data
- * should be flushed to disk.
- * @return approximate amount of memory (in MB) that is going to be freed up
- * after executing the generated commands
- */
- private double createCommands(PairList<Integer, Integer> pairs,
- BlockingQueue<Integer> commands) {
- double usedMemory = 0;
- if (pairs.getSize() != 0) {
- PairList<Integer, Integer>.Iterator iterator = pairs.getIterator();
- // Generating commands for out-of-core processor threads to
- // offload data as long as command queue has space.
- while (iterator.hasNext() &&
- commands.remainingCapacity() > 0) {
- iterator.next();
- commands.add(iterator.getCurrentFirst());
- // Having an approximation on the memory foot-print of data to offload
- // helps us to know how much memory is going to become available by
- // offloading the data without using internal functions to estimate
- // free memory again.
- usedMemory += iterator.getCurrentSecond() / 1024.0 / 1024.0;
- }
- }
- return usedMemory;
- }
-}
[2/5] git commit: updated refs/heads/trunk to fafecee
Posted by ed...@apache.org.
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/data/MetaPartitionManager.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/data/MetaPartitionManager.java b/giraph-core/src/main/java/org/apache/giraph/ooc/data/MetaPartitionManager.java
new file mode 100644
index 0000000..6c08dfd
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/data/MetaPartitionManager.java
@@ -0,0 +1,947 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.data;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.giraph.ooc.OutOfCoreEngine;
+import org.apache.giraph.worker.BspServiceWorker;
+import org.apache.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static com.google.common.base.Preconditions.checkState;
+
+/**
+ * Class to keep meta-information about partition data, edge data, and message
+ * data of each partition on a worker.
+ */
+public class MetaPartitionManager {
+ /**
+ * Flag representing no partitions is left to process in the current iteration
+ * cycle over all partitions.
+ */
+ public static final int NO_PARTITION_TO_PROCESS = -1;
+
+ /** Class logger */
+ private static final Logger LOG =
+ Logger.getLogger(MetaPartitionManager.class);
+ /** Different storage states for data */
+ private enum StorageState { IN_MEM, ON_DISK, IN_TRANSIT };
+ /**
+ * Different processing states for partitions. Processing states are reset
+ * at the beginning of each iteration cycle over partitions.
+ */
+ private enum ProcessingState { PROCESSED, UNPROCESSED, IN_PROCESS };
+
+ /** Number of in-memory partitions */
+ private final AtomicInteger numInMemoryPartitions = new AtomicInteger(0);
+ /** Map of partitions to their meta information */
+ private final ConcurrentMap<Integer, MetaPartition> partitions =
+ Maps.newConcurrentMap();
+ /** List of partitions assigned to each IO threads */
+ private final List<PerThreadPartitionStatus> perThreadPartitions;
+ /** For each IO thread, set of partition ids that are on-disk and have
+ * 'large enough' vertex/edge buffers to be offloaded on disk
+ */
+ private final List<Set<Integer>> perThreadVertexEdgeBuffers;
+ /**
+ * For each IO thread, set of partition ids that are on-disk and have
+ * 'large enough' message buffers to be offloaded on disk
+ */
+ private final List<Set<Integer>> perThreadMessageBuffers;
+ /**
+ * Out-of-core engine
+ */
+ private final OutOfCoreEngine oocEngine;
+ /**
+ * Number of processed partitions in the current iteration cycle over all
+ * partitions
+ */
+ private final AtomicInteger numPartitionsProcessed = new AtomicInteger(0);
+ /**
+ * Random number generator to choose a thread to get one of its partition for
+ * processing
+ */
+ private final Random randomGenerator;
+
+ /**
+ * Constructor
+ *
+ * @param numIOThreads number of IO threads
+ * @param oocEngine out-of-core engine
+ */
+ public MetaPartitionManager(int numIOThreads, OutOfCoreEngine oocEngine) {
+ perThreadPartitions = new ArrayList<>(numIOThreads);
+ perThreadVertexEdgeBuffers = new ArrayList<>(numIOThreads);
+ perThreadMessageBuffers = new ArrayList<>(numIOThreads);
+ for (int i = 0; i < numIOThreads; ++i) {
+ perThreadPartitions.add(new PerThreadPartitionStatus());
+ perThreadMessageBuffers.add(Sets.<Integer>newConcurrentHashSet());
+ perThreadVertexEdgeBuffers.add(Sets.<Integer>newConcurrentHashSet());
+ }
+ this.oocEngine = oocEngine;
+ this.randomGenerator = new Random();
+ }
+
+ /**
+ * Get number of partitions in memory
+ *
+ * @return number of partitions in memory
+ */
+ public int getNumInMemoryPartitions() {
+ return numInMemoryPartitions.get();
+ }
+
+ /**
+ * Get total number of partitions
+ *
+ * @return total number of partition
+ */
+ public int getNumPartitions() {
+ return partitions.size();
+ }
+
+ /**
+ * Whether a given partition is available
+ *
+ * @param partitionId id of the partition to check if this worker owns it
+ * @return true if the worker owns the partition, false otherwise
+ */
+ public boolean hasPartition(Integer partitionId) {
+ return partitions.containsKey(partitionId);
+ }
+
+ /**
+ * Return the list of all available partitions as an iterable
+ *
+ * @return list of all available partitions
+ */
+ public Iterable<Integer> getPartitionIds() {
+ return partitions.keySet();
+ }
+
+ /**
+ * Add a partition
+ *
+ * @param partitionId id of a partition to add
+ */
+ public void addPartition(int partitionId) {
+ MetaPartition meta = new MetaPartition(partitionId);
+ MetaPartition temp = partitions.putIfAbsent(partitionId, meta);
+ // Check if the given partition is new
+ if (temp == null) {
+ int ownerThread = oocEngine.getIOScheduler()
+ .getOwnerThreadId(partitionId);
+ Set<MetaPartition> partitionSet =
+ perThreadPartitions.get(ownerThread).getInMemoryProcessed();
+ synchronized (partitionSet) {
+ partitionSet.add(meta);
+ }
+ numInMemoryPartitions.getAndIncrement();
+ }
+ }
+
+ /**
+ * Remove a partition. This method assumes that the partition is already
+ * retrieved and is in memory)
+ *
+ * @param partitionId id of a partition to remove
+ */
+ public void removePartition(Integer partitionId) {
+ MetaPartition meta = partitions.remove(partitionId);
+ checkState(!meta.isOnDisk());
+ numInMemoryPartitions.getAndDecrement();
+ }
+
+ /**
+ * Pops an entry from the specified set.
+ *
+ * @param set set to pop an entry from
+ * @param <T> Type of entries in the set
+ * @return popped entry from the given set
+ */
+ private static <T> T popFromSet(Set<T> set) {
+ if (!set.isEmpty()) {
+ Iterator<T> it = set.iterator();
+ T entry = it.next();
+ it.remove();
+ return entry;
+ }
+ return null;
+ }
+
+ /**
+ * Peeks an entry from the specified set.
+ *
+ * @param set set to peek an entry from
+ * @param <T> Type of entries in the set
+ * @return peeked entry from the given set
+ */
+ private static <T> T peekFromSet(Set<T> set) {
+ if (!set.isEmpty()) {
+ return set.iterator().next();
+ }
+ return null;
+ }
+
+ /**
+ * Get id of a partition to offload on disk
+ *
+ * @param threadId id of the thread who is going to store the partition on
+ * disk
+ * @return id of the partition to offload on disk
+ */
+ public Integer getOffloadPartitionId(int threadId) {
+ Set<MetaPartition> partitionSet = perThreadPartitions.get(threadId)
+ .getInMemoryProcessed();
+ synchronized (partitionSet) {
+ MetaPartition meta = peekFromSet(partitionSet);
+ if (meta != null) {
+ return meta.getPartitionId();
+ }
+ }
+ partitionSet = perThreadPartitions.get(threadId).getInMemoryUnprocessed();
+ synchronized (partitionSet) {
+ MetaPartition meta = peekFromSet(partitionSet);
+ if (meta != null) {
+ return meta.getPartitionId();
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get id of a partition to offload its vertex/edge buffers on disk
+ *
+ * @param threadId id of the thread who is going to store the buffers on disk
+ * @return id of the partition to offload its vertex/edge buffers on disk
+ */
+ public Integer getOffloadPartitionBufferId(int threadId) {
+ if (oocEngine.getServiceWorker().getSuperstep() ==
+ BspServiceWorker.INPUT_SUPERSTEP) {
+ Integer partitionId =
+ popFromSet(perThreadVertexEdgeBuffers.get(threadId));
+ if (partitionId == null) {
+ DiskBackedPartitionStore<?, ?, ?> partitionStore =
+ (DiskBackedPartitionStore<?, ?, ?>) (oocEngine.getServerData()
+ .getPartitionStore());
+ perThreadVertexEdgeBuffers.get(threadId)
+ .addAll(partitionStore.getCandidateBuffersToOffload());
+ DiskBackedEdgeStore<?, ?, ?> edgeStore =
+ (DiskBackedEdgeStore<?, ?, ?>) (oocEngine.getServerData())
+ .getEdgeStore();
+ perThreadVertexEdgeBuffers.get(threadId)
+ .addAll(edgeStore.getCandidateBuffersToOffload());
+ partitionId = popFromSet(perThreadVertexEdgeBuffers.get(threadId));
+ }
+ return partitionId;
+ }
+ return null;
+ }
+
+ /**
+ * Get id of a partition to offload its incoming message buffers on disk
+ *
+ * @param threadId id of the thread who is going to store the buffers on disk
+ * @return id of the partition to offload its message buffer on disk
+ */
+ public Integer getOffloadMessageBufferId(int threadId) {
+ if (oocEngine.getServiceWorker().getSuperstep() !=
+ BspServiceWorker.INPUT_SUPERSTEP) {
+ Integer partitionId =
+ popFromSet(perThreadMessageBuffers.get(threadId));
+ if (partitionId == null) {
+ DiskBackedMessageStore<?, ?> messageStore =
+ (DiskBackedMessageStore<?, ?>) (oocEngine.getServerData()
+ .getIncomingMessageStore());
+ if (messageStore != null) {
+ perThreadMessageBuffers.get(threadId)
+ .addAll(messageStore.getCandidateBuffersToOffload());
+ partitionId = popFromSet(perThreadMessageBuffers.get(threadId));
+ }
+ }
+ return partitionId;
+ }
+ return null;
+ }
+
+ /**
+ * Get id of a partition to offload its incoming message on disk
+ *
+ * @param threadId id of the thread who is going to store the incoming
+ * messages on disk
+ * @return id of the partition to offload its message on disk
+ */
+ public Integer getOffloadMessageId(int threadId) {
+ Set<MetaPartition> partitionSet = perThreadPartitions.get(threadId)
+ .getInDiskProcessed();
+ synchronized (partitionSet) {
+ for (MetaPartition meta : partitionSet) {
+ if (meta.getIncomingMessagesState() == StorageState.IN_MEM) {
+ return meta.getPartitionId();
+ }
+ }
+ }
+ partitionSet = perThreadPartitions.get(threadId).getInDiskUnprocessed();
+ synchronized (partitionSet) {
+ for (MetaPartition meta : partitionSet) {
+ if (meta.getIncomingMessagesState() == StorageState.IN_MEM) {
+ return meta.getPartitionId();
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get id of a partition to prefetch its data to memory
+ *
+ * @param threadId id of the thread who is going to load the partition data
+ * @return id of the partition to load its data to memory
+ */
+ public Integer getPrefetchPartitionId(int threadId) {
+ Set<MetaPartition> partitionSet =
+ perThreadPartitions.get(threadId).getInDiskUnprocessed();
+ synchronized (partitionSet) {
+ MetaPartition meta = peekFromSet(partitionSet);
+ return (meta != null) ? meta.getPartitionId() : null;
+ }
+ }
+
+ /**
+ * Mark a partition inaccessible to IO and compute threads
+ *
+ * @param partitionId id of the partition to mark
+ */
+ public void makePartitionInaccessible(int partitionId) {
+ MetaPartition meta = partitions.get(partitionId);
+ perThreadPartitions.get(oocEngine.getIOScheduler()
+ .getOwnerThreadId(partitionId))
+ .remove(meta);
+ synchronized (meta) {
+ meta.setProcessingState(ProcessingState.IN_PROCESS);
+ }
+ }
+
+ /**
+ * Mark a partition as 'PROCESSED'
+ *
+ * @param partitionId id of the partition to mark
+ */
+ public void setPartitionIsProcessed(int partitionId) {
+ MetaPartition meta = partitions.get(partitionId);
+ synchronized (meta) {
+ meta.setProcessingState(ProcessingState.PROCESSED);
+ }
+ Set<MetaPartition> partitionSet = perThreadPartitions
+ .get(oocEngine.getIOScheduler().getOwnerThreadId(partitionId))
+ .getInMemoryProcessed();
+ synchronized (partitionSet) {
+ partitionSet.add(meta);
+ }
+ numPartitionsProcessed.getAndIncrement();
+ }
+
+ /**
+ * Notify this meta store that load of a partition for a specific superstep
+ * is about to start.
+ *
+ * @param partitionId id of the partition to load to memory
+ * @param superstep superstep in which the partition is needed for
+ * @return true iff load of the given partition is viable
+ */
+ public boolean startLoadingPartition(int partitionId, long superstep) {
+ MetaPartition meta = partitions.get(partitionId);
+ synchronized (meta) {
+ boolean shouldLoad = meta.getPartitionState() == StorageState.ON_DISK;
+ if (superstep == oocEngine.getServiceWorker().getSuperstep()) {
+ shouldLoad |= meta.getCurrentMessagesState() == StorageState.ON_DISK;
+ } else {
+ shouldLoad |= meta.getIncomingMessagesState() == StorageState.ON_DISK;
+ }
+ return shouldLoad;
+ }
+ }
+
+ /**
+ * Notify this meta store that load of a partition for a specific superstep
+ * is completed
+ *
+ * @param partitionId id of a the partition that load is completed
+ * @param superstep superstep in which the partition is loaded for
+ */
+ public void doneLoadingPartition(int partitionId, long superstep) {
+ MetaPartition meta = partitions.get(partitionId);
+ numInMemoryPartitions.getAndIncrement();
+ int owner = oocEngine.getIOScheduler().getOwnerThreadId(partitionId);
+ boolean removed = perThreadPartitions.get(owner)
+ .remove(meta, StorageState.ON_DISK);
+ if (removed || meta.getProcessingState() == ProcessingState.IN_PROCESS) {
+ synchronized (meta) {
+ meta.setPartitionState(StorageState.IN_MEM);
+ if (superstep == oocEngine.getServiceWorker().getSuperstep()) {
+ meta.setCurrentMessagesState(StorageState.IN_MEM);
+ } else {
+ meta.setIncomingMessagesState(StorageState.IN_MEM);
+ }
+ }
+ perThreadPartitions.get(owner).add(meta, StorageState.IN_MEM);
+ }
+ }
+
+ /**
+ * Notify this meta store that offload of messages for a particular partition
+ * is about to start.
+ *
+ * @param partitionId id of the partition that its messages is being offloaded
+ * @return true iff offload of messages of the given partition is viable
+ */
+ public boolean startOffloadingMessages(int partitionId) {
+ MetaPartition meta = partitions.get(partitionId);
+ synchronized (meta) {
+ if (meta.getIncomingMessagesState() == StorageState.IN_MEM) {
+ meta.setIncomingMessagesState(StorageState.IN_TRANSIT);
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ /**
+ * Notify this meta store that offload of messages for a particular partition
+ * is complete.
+ *
+ * @param partitionId id of the partition that its messages is offloaded to
+ * disk
+ */
+ public void doneOffloadingMessages(int partitionId) {
+ MetaPartition meta = partitions.get(partitionId);
+ synchronized (meta) {
+ meta.setIncomingMessagesState(StorageState.ON_DISK);
+ }
+ }
+
+ /**
+ * Notify this meta store that offload of raw data buffers (vertex/edges/
+ * messages) of a particular partition is about to start.
+ *
+ * @param partitionId id of the partition that its buffer is being offloaded
+ * @return true iff offload of buffers of the given partition is viable
+ */
+ public boolean startOffloadingBuffer(int partitionId) {
+ // Do nothing
+ return true;
+ }
+
+ /**
+ * Notify this meta store that offload of raw data buffers (vertex/edges/
+ * messages) of a particular partition is completed.
+ *
+ * @param partitionId id of the partition that its buffer is offloaded
+ */
+ public void doneOffloadingBuffer(int partitionId) {
+ // Do nothing
+ }
+
+ /**
+ * Notify this meta store that offload of a partition (partition data and its
+ * current messages) is about to start.
+ *
+ * @param partitionId id of the partition that its data is being offloaded
+ * @return true iff offload of the given partition is viable
+ */
+ public boolean startOffloadingPartition(int partitionId) {
+ MetaPartition meta = partitions.get(partitionId);
+ int owner = oocEngine.getIOScheduler().getOwnerThreadId(partitionId);
+ boolean removed = perThreadPartitions.get(owner)
+ .remove(meta, StorageState.IN_MEM);
+ if (removed) {
+ synchronized (meta) {
+ meta.setPartitionState(StorageState.IN_TRANSIT);
+ meta.setCurrentMessagesState(StorageState.IN_TRANSIT);
+ }
+ perThreadPartitions.get(owner).add(meta, StorageState.IN_TRANSIT);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Notify this meta store that offload of a partition (partition data and its
+ * current messages) is completed.
+ *
+ * @param partitionId id of the partition that its data is offloaded
+ */
+ public void doneOffloadingPartition(int partitionId) {
+ numInMemoryPartitions.getAndDecrement();
+ MetaPartition meta = partitions.get(partitionId);
+ int owner = oocEngine.getIOScheduler().getOwnerThreadId(partitionId);
+ boolean removed = perThreadPartitions.get(owner)
+ .remove(meta, StorageState.IN_TRANSIT);
+ if (removed) {
+ synchronized (meta) {
+ meta.setPartitionState(StorageState.ON_DISK);
+ meta.setCurrentMessagesState(StorageState.ON_DISK);
+ }
+ perThreadPartitions.get(owner).add(meta, StorageState.ON_DISK);
+ }
+ }
+
+ /**
+ * Reset the meta store for a new iteration cycle over all partitions.
+ * Note: this is not thread-safe and should be called from a single thread.
+ */
+ public void resetPartition() {
+ for (MetaPartition meta : partitions.values()) {
+ meta.resetPartition();
+ }
+ int numPartition = 0;
+ for (PerThreadPartitionStatus status : perThreadPartitions) {
+ numPartition += status.reset();
+ }
+ checkState(numPartition == partitions.size());
+ numPartitionsProcessed.set(0);
+ }
+
+ /**
+ * Reset messages in the meta store.
+ * Note: this is not thread-safe and should be called from a single thread.
+ */
+ public void resetMessages() {
+ for (MetaPartition meta : partitions.values()) {
+ meta.resetMessages();
+ }
+ // After swapping incoming messages and current messages, it may be the case
+ // that a partition has data in memory (partitionState == IN_MEM), but now
+ // its current messages are on disk (currentMessageState == ON_DISK). So, we
+ // have to mark the partition as ON_DISK, and load its messages once it is
+ // about to be processed.
+ for (PerThreadPartitionStatus status : perThreadPartitions) {
+ Set<MetaPartition> partitionSet = status.getInMemoryUnprocessed();
+ Iterator<MetaPartition> it = partitionSet.iterator();
+ while (it.hasNext()) {
+ MetaPartition meta = it.next();
+ if (meta.getCurrentMessagesState() == StorageState.ON_DISK) {
+ it.remove();
+ status.getInDiskUnprocessed().add(meta);
+ numInMemoryPartitions.getAndDecrement();
+ }
+ }
+ partitionSet = status.getInMemoryProcessed();
+ it = partitionSet.iterator();
+ while (it.hasNext()) {
+ MetaPartition meta = it.next();
+ if (meta.getCurrentMessagesState() == StorageState.ON_DISK) {
+ it.remove();
+ status.getInDiskProcessed().add(meta);
+ numInMemoryPartitions.getAndDecrement();
+ }
+ }
+ }
+ }
+
+ /**
+ * Return the id of an unprocessed partition in memory. If all partitions are
+ * processed, return an appropriate 'finisher signal'. If there are
+ * unprocessed partitions, but none are is memory, return null.
+ *
+ * @return id of the partition to be processed next.
+ */
+ public Integer getNextPartition() {
+ if (numPartitionsProcessed.get() >= partitions.size()) {
+ return NO_PARTITION_TO_PROCESS;
+ }
+ int numThreads = perThreadPartitions.size();
+ int index = randomGenerator.nextInt(numThreads);
+ int startIndex = index;
+ do {
+ Set<MetaPartition> partitionSet =
+ perThreadPartitions.get(index).getInMemoryUnprocessed();
+ MetaPartition meta;
+ synchronized (partitionSet) {
+ meta = popFromSet(partitionSet);
+ }
+ if (meta != null) {
+ synchronized (meta) {
+ meta.setProcessingState(ProcessingState.IN_PROCESS);
+ return meta.getPartitionId();
+ }
+ }
+ index = (index + 1) % numThreads;
+ } while (index != startIndex);
+ return null;
+ }
+
+ /**
+ * Whether a partition is on disk (both its data and its current messages)
+ *
+ * @param partitionId id of the partition to check if it is on disk
+ * @return true if partition data or its current messages are on disk, false
+ * otherwise
+ */
+ public boolean isPartitionOnDisk(int partitionId) {
+ MetaPartition meta = partitions.get(partitionId);
+ return meta.isOnDisk();
+ }
+
+ /**
+ * Representation of meta information of a partition
+ */
+ private static class MetaPartition {
+ /** Id of the partition */
+ private int partitionId;
+ /** Storage state of incoming messages */
+ private StorageState incomingMessagesState;
+ /** Storage state of current messages */
+ private StorageState currentMessagesState;
+ /** Storage state of partition data */
+ private StorageState partitionState;
+ /** Processing state of a partition */
+ private volatile ProcessingState processingState;
+
+ /**
+ * Constructor
+ *
+ * @param partitionId id of the partition
+ */
+ MetaPartition(int partitionId) {
+ this.partitionId = partitionId;
+ this.processingState = ProcessingState.PROCESSED;
+ this.partitionState = StorageState.IN_MEM;
+ this.currentMessagesState = StorageState.IN_MEM;
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append("\nMetaData: {");
+ sb.append("ID: " + partitionId + "; ");
+ sb.append("Partition: " + partitionState + "; ");
+ sb.append("Current Messages: " + currentMessagesState + "; ");
+ sb.append("Incoming Messages: " + incomingMessagesState + "; ");
+ sb.append("Processed? : " + processingState + "}");
+ return sb.toString();
+ }
+
+ /**
+ * Get id of the partition
+ *
+ * @return id of the partition
+ */
+ public int getPartitionId() {
+ return partitionId;
+ }
+
+ /**
+ * Get storage state of incoming messages of the partition
+ *
+ * @return storage state of incoming messages
+ */
+ public StorageState getIncomingMessagesState() {
+ return incomingMessagesState;
+ }
+
+ /**
+ * Set storage state of incoming messages of the partition
+ *
+ * @param incomingMessagesState storage state of incoming messages
+ */
+ public void setIncomingMessagesState(StorageState incomingMessagesState) {
+ this.incomingMessagesState = incomingMessagesState;
+ }
+
+ /**
+ * Get storage state of current messages of the partition
+ *
+ * @return storage state of current messages
+ */
+ public StorageState getCurrentMessagesState() {
+ return currentMessagesState;
+ }
+
+ /**
+ * Set storage state of current messages of the partition
+ *
+ * @param currentMessagesState storage state of current messages
+ */
+ public void setCurrentMessagesState(StorageState currentMessagesState) {
+ this.currentMessagesState = currentMessagesState;
+ }
+
+ /**
+ * Get storage state of the partition
+ *
+ * @return storage state of the partition
+ */
+ public StorageState getPartitionState() {
+ return partitionState;
+ }
+
+ /**
+ * Set storage state of the partition
+ *
+ * @param state storage state of the partition
+ */
+ public void setPartitionState(StorageState state) {
+ this.partitionState = state;
+ }
+
+ public ProcessingState getProcessingState() {
+ return processingState;
+ }
+
+ public void setProcessingState(ProcessingState processingState) {
+ this.processingState = processingState;
+ }
+
+ /**
+ * Whether the partition is on disk (either its data or its current
+ * messages)
+ *
+ * @return true if the partition is on disk, false otherwise
+ */
+ public boolean isOnDisk() {
+ return partitionState == StorageState.ON_DISK ||
+ currentMessagesState == StorageState.ON_DISK;
+ }
+
+ /**
+ * Reset the partition meta information for the next iteration cycle
+ */
+ public void resetPartition() {
+ processingState = ProcessingState.UNPROCESSED;
+ }
+
+ /**
+ * Reset messages meta information for the next iteration cycle
+ */
+ public void resetMessages() {
+ currentMessagesState = incomingMessagesState;
+ incomingMessagesState = StorageState.IN_MEM;
+ }
+ }
+
+ /**
+ * Representation of partitions' state per IO thread
+ */
+ private static class PerThreadPartitionStatus {
+ /**
+ * Contains partitions that has been processed in the current iteration
+ * cycle, and are not in use by any thread.
+ */
+ private Map<StorageState, Set<MetaPartition>>
+ processedPartitions = Maps.newConcurrentMap();
+ /**
+ * Contains partitions that has *NOT* been processed in the current
+ * iteration cycle, and are not in use by any thread.
+ */
+ private Map<StorageState, Set<MetaPartition>>
+ unprocessedPartitions = Maps.newConcurrentMap();
+
+ /**
+ * Constructor
+ */
+ public PerThreadPartitionStatus() {
+ processedPartitions.put(StorageState.IN_MEM,
+ Sets.<MetaPartition>newLinkedHashSet());
+ processedPartitions.put(StorageState.ON_DISK,
+ Sets.<MetaPartition>newLinkedHashSet());
+ processedPartitions.put(StorageState.IN_TRANSIT,
+ Sets.<MetaPartition>newLinkedHashSet());
+
+ unprocessedPartitions.put(StorageState.IN_MEM,
+ Sets.<MetaPartition>newLinkedHashSet());
+ unprocessedPartitions.put(StorageState.ON_DISK,
+ Sets.<MetaPartition>newLinkedHashSet());
+ unprocessedPartitions.put(StorageState.IN_TRANSIT,
+ Sets.<MetaPartition>newLinkedHashSet());
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append("\nProcessed Partitions: " + processedPartitions + "; ");
+ sb.append("\nUnprocessedPartitions: " + unprocessedPartitions);
+ return sb.toString();
+ }
+
+ /**
+ * Get set of partitions that are in memory and are processed
+ *
+ * @return set of partition that are in memory are are processed
+ */
+ public Set<MetaPartition> getInMemoryProcessed() {
+ return processedPartitions.get(StorageState.IN_MEM);
+ }
+
+ /**
+ * Get set of partitions that are in memory are are not processed
+ *
+ * @return set of partitions that are in memory and are not processed
+ */
+ public Set<MetaPartition> getInMemoryUnprocessed() {
+ return unprocessedPartitions.get(StorageState.IN_MEM);
+ }
+
+ /**
+ * Get set of partitions that are on disk and are processed
+ *
+ * @return set of partitions that are on disk and are processed
+ */
+ public Set<MetaPartition> getInDiskProcessed() {
+ return processedPartitions.get(StorageState.ON_DISK);
+ }
+
+ /**
+ * Get set of partitions that are on disk and are not processed
+ *
+ * @return set of partitions that are on disk and are not processed
+ */
+ public Set<MetaPartition> getInDiskUnprocessed() {
+ return unprocessedPartitions.get(StorageState.ON_DISK);
+ }
+
+ /**
+ * Remove a partition from meta information
+ *
+ * @param meta meta-information of a partition to be removed
+ */
+ public void remove(MetaPartition meta) {
+ Set<MetaPartition> partitionSet;
+ partitionSet = processedPartitions.get(StorageState.IN_MEM);
+ synchronized (partitionSet) {
+ if (partitionSet.remove(meta)) {
+ return;
+ }
+ }
+ partitionSet = unprocessedPartitions.get(StorageState.IN_MEM);
+ synchronized (partitionSet) {
+ if (partitionSet.remove(meta)) {
+ return;
+ }
+ }
+ partitionSet = processedPartitions.get(StorageState.IN_TRANSIT);
+ synchronized (partitionSet) {
+ if (partitionSet.remove(meta)) {
+ return;
+ }
+ }
+ partitionSet = unprocessedPartitions.get(StorageState.IN_TRANSIT);
+ synchronized (partitionSet) {
+ if (partitionSet.remove(meta)) {
+ return;
+ }
+ }
+ partitionSet = processedPartitions.get(StorageState.ON_DISK);
+ synchronized (partitionSet) {
+ if (partitionSet.remove(meta)) {
+ return;
+ }
+ }
+ partitionSet = unprocessedPartitions.get(StorageState.ON_DISK);
+ synchronized (partitionSet) {
+ partitionSet.remove(meta);
+ }
+ }
+
+ /**
+ * Reset meta-information for the next iteration cycle over all partitions
+ *
+ * @return total number of partitions kept for this thread
+ */
+ public int reset() {
+ checkState(unprocessedPartitions.get(StorageState.IN_MEM).size() == 0);
+ checkState(unprocessedPartitions.get(StorageState.IN_TRANSIT).size() ==
+ 0);
+ checkState(unprocessedPartitions.get(StorageState.ON_DISK).size() == 0);
+ unprocessedPartitions.clear();
+ unprocessedPartitions.putAll(processedPartitions);
+ processedPartitions.clear();
+ processedPartitions.put(StorageState.IN_MEM,
+ Sets.<MetaPartition>newLinkedHashSet());
+ processedPartitions.put(StorageState.ON_DISK,
+ Sets.<MetaPartition>newLinkedHashSet());
+ processedPartitions.put(StorageState.IN_TRANSIT,
+ Sets.<MetaPartition>newLinkedHashSet());
+ return unprocessedPartitions.get(StorageState.IN_MEM).size() +
+ unprocessedPartitions.get(StorageState.IN_TRANSIT).size() +
+ unprocessedPartitions.get(StorageState.ON_DISK).size();
+ }
+
+ /**
+ * Remove a partition from partition set of a given state
+ *
+ * @param meta meta partition to remove
+ * @param state state from which the partition should be removed
+ * @return true iff the partition is actually removed
+ */
+ public boolean remove(MetaPartition meta, StorageState state) {
+ boolean removed = false;
+ Set<MetaPartition> partitionSet = null;
+ if (meta.getProcessingState() == ProcessingState.UNPROCESSED) {
+ partitionSet = unprocessedPartitions.get(state);
+ } else if (meta.getProcessingState() == ProcessingState.PROCESSED) {
+ partitionSet = processedPartitions.get(state);
+ } else {
+ LOG.info("remove: partition " + meta.getPartitionId() + " is " +
+ "already being processed! This should happen only if partition " +
+ "removal is done before start of an iteration over all partitions");
+ }
+ if (partitionSet != null) {
+ synchronized (partitionSet) {
+ removed = partitionSet.remove(meta);
+ }
+ }
+ return removed;
+ }
+
+ /**
+ * Add a partition to partition set of a given state
+ *
+ * @param meta meta partition to add
+ * @param state state to which the partition should be added
+ */
+ public void add(MetaPartition meta, StorageState state) {
+ Set<MetaPartition> partitionSet = null;
+ if (meta.getProcessingState() == ProcessingState.UNPROCESSED) {
+ partitionSet = unprocessedPartitions.get(state);
+ } else if (meta.getProcessingState() == ProcessingState.PROCESSED) {
+ partitionSet = processedPartitions.get(state);
+ } else {
+ LOG.info("add: partition " + meta.getPartitionId() + " is already " +
+ "being processed!");
+ }
+ if (partitionSet != null) {
+ synchronized (partitionSet) {
+ partitionSet.add(meta);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/data/OutOfCoreDataManager.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/data/OutOfCoreDataManager.java b/giraph-core/src/main/java/org/apache/giraph/ooc/data/OutOfCoreDataManager.java
new file mode 100644
index 0000000..7d97e51
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/data/OutOfCoreDataManager.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.data;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.conf.IntConfOption;
+import org.apache.log4j.Logger;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static org.apache.giraph.conf.GiraphConstants.ONE_MB;
+
+/**
+ * This class provides basic operations for data structures that have to
+ * participate in out-of-core mechanism. Essential subclasses of this class are:
+ * - DiskBackedPartitionStore (for partition data)
+ * - DiskBackedMessageStore (for messages)
+ * - DiskBackedEdgeStore (for edges read in INPUT_SUPERSTEP)
+ * Basically, any data structure that may cause OOM to happen can be implemented
+ * as a subclass of this class.
+ *
+ * There are two different terms used in the rest of this class:
+ * - "data store" refers to in-memory representation of data. Usually this is
+ * stored per-partition in in-memory implementations of data structures. For
+ * instance, "data store" of a DiskBackedPartitionStore would collection of
+ * all partitions kept in the in-memory partition store within the
+ * DiskBackedPartitionStore.
+ * - "raw data buffer" refers to raw data which were supposed to be
+ * de-serialized and added to the data store, but they remain 'as is' in the
+ * memory because their corresponding partition is offloaded to disk and is
+ * not available in the data store.
+ *
+ * @param <T> raw data format of the data store subclassing this class
+ */
+public abstract class OutOfCoreDataManager<T> {
+ /**
+ * Minimum size of a buffer (in bytes) to flush to disk. This is used to
+ * decide whether vertex/edge buffers are large enough to flush to disk.
+ */
+ public static final IntConfOption MINIMUM_BUFFER_SIZE_TO_FLUSH =
+ new IntConfOption("giraph.flushBufferSize", 8 * ONE_MB,
+ "Minimum size of a buffer (in bytes) to flush to disk.");
+
+ /** Class logger. */
+ private static final Logger LOG = Logger.getLogger(
+ OutOfCoreDataManager.class);
+ /** Cached value for MINIMUM_BUFFER_SIZE_TO_FLUSH */
+ private final int minBufferSizeToOffload;
+ /** Set containing ids of all out-of-core partitions */
+ private final Set<Integer> hasPartitionDataOnDisk =
+ Sets.newConcurrentHashSet();
+ /**
+ * Map of partition ids to list of raw data buffers. The map will have entries
+ * only for partitions that their in-memory data structures are currently
+ * offloaded to disk. We keep the aggregate size of buffers for each partition
+ * as part of the values in the map to estimate how much memory we can free up
+ * if we offload data buffers of a particular partition to disk.
+ */
+ private final ConcurrentMap<Integer, Pair<Integer, List<T>>> dataBuffers =
+ Maps.newConcurrentMap();
+ /**
+ * Map of partition ids to number of raw data buffers offloaded to disk for
+ * each partition. The map will have entries only for partitions that their
+ * in-memory data structures are currently out of core. It is necessary to
+ * know the number of data buffers on disk for a particular partition when we
+ * are loading all these buffers back in memory.
+ */
+ private final ConcurrentMap<Integer, Integer> numDataBuffersOnDisk =
+ Maps.newConcurrentMap();
+ /**
+ * Lock to avoid overlapping of read and write on data associated with each
+ * partition.
+ * */
+ private final ConcurrentMap<Integer, ReadWriteLock> locks =
+ Maps.newConcurrentMap();
+
+ /**
+ * Constructor.
+ *
+ * @param conf Configuration
+ */
+ OutOfCoreDataManager(ImmutableClassesGiraphConfiguration conf) {
+ this.minBufferSizeToOffload = MINIMUM_BUFFER_SIZE_TO_FLUSH.get(conf);
+ }
+
+ /**
+ * Retrieves a lock for a given partition. If the lock for the given partition
+ * does not exist, creates a new lock.
+ *
+ * @param partitionId id of the partition the lock is needed for
+ * @return lock for a given partition
+ */
+ private ReadWriteLock getPartitionLock(int partitionId) {
+ ReadWriteLock readWriteLock = locks.get(partitionId);
+ if (readWriteLock == null) {
+ readWriteLock = new ReentrantReadWriteLock();
+ ReadWriteLock temp = locks.putIfAbsent(partitionId, readWriteLock);
+ if (temp != null) {
+ readWriteLock = temp;
+ }
+ }
+ return readWriteLock;
+ }
+
+ /**
+ * Adds a data entry for a given partition to the current data store. If data
+ * of a given partition in data store is already offloaded to disk, adds the
+ * data entry to appropriate raw data buffer list.
+ *
+ * @param partitionId id of the partition to add the data entry to
+ * @param entry data entry to add
+ */
+ protected void addEntry(int partitionId, T entry) {
+ // Addition of data entries to a data store is much more common than
+ // out-of-core operations. Besides, in-memory data store implementations
+ // existing in the code base already account for parallel addition to data
+ // stores. Therefore, using read lock would optimize for parallel addition
+ // to data stores, specially for cases where the addition should happen for
+ // partitions that are entirely in memory.
+ ReadWriteLock rwLock = getPartitionLock(partitionId);
+ rwLock.readLock().lock();
+ if (hasPartitionDataOnDisk.contains(partitionId)) {
+ List<T> entryList = new ArrayList<>();
+ entryList.add(entry);
+ int entrySize = entrySerializedSize(entry);
+ MutablePair<Integer, List<T>> newPair =
+ new MutablePair<>(entrySize, entryList);
+ Pair<Integer, List<T>> oldPair =
+ dataBuffers.putIfAbsent(partitionId, newPair);
+ if (oldPair != null) {
+ synchronized (oldPair) {
+ newPair = (MutablePair<Integer, List<T>>) oldPair;
+ newPair.setLeft(oldPair.getLeft() + entrySize);
+ newPair.getRight().add(entry);
+ }
+ }
+ } else {
+ addEntryToImMemoryPartitionData(partitionId, entry);
+ }
+ rwLock.readLock().unlock();
+ }
+
+ /**
+ * Loads and assembles all data for a given partition, and put it into the
+ * data store.
+ *
+ * @param partitionId id of the partition to load ana assemble all data for
+ * @param basePath path to load the data from
+ * @throws IOException
+ */
+ public void loadPartitionData(int partitionId, String basePath)
+ throws IOException {
+ ReadWriteLock rwLock = getPartitionLock(partitionId);
+ rwLock.writeLock().lock();
+ if (hasPartitionDataOnDisk.contains(partitionId)) {
+ loadInMemoryPartitionData(partitionId, getPath(basePath, partitionId));
+ hasPartitionDataOnDisk.remove(partitionId);
+ // Loading raw data buffers from disk if there is any and applying those
+ // to already loaded in-memory data.
+ Integer numBuffers = numDataBuffersOnDisk.remove(partitionId);
+ if (numBuffers != null) {
+ checkState(numBuffers > 0);
+ File file = new File(getBuffersPath(basePath, partitionId));
+ checkState(file.exists());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("loadPartitionData: loading " + numBuffers + " buffers of" +
+ " partition " + partitionId + " from " + file.getAbsolutePath());
+ }
+ FileInputStream fis = new FileInputStream(file);
+ BufferedInputStream bis = new BufferedInputStream(fis);
+ DataInputStream dis = new DataInputStream(bis);
+ for (int i = 0; i < numBuffers; ++i) {
+ T entry = readNextEntry(dis);
+ addEntryToImMemoryPartitionData(partitionId, entry);
+ }
+ dis.close();
+ checkState(file.delete(), "loadPartitionData: failed to delete %s.",
+ file.getAbsoluteFile());
+ }
+ // Applying in-memory raw data buffers to in-memory partition data.
+ Pair<Integer, List<T>> pair = dataBuffers.remove(partitionId);
+ if (pair != null) {
+ for (T entry : pair.getValue()) {
+ addEntryToImMemoryPartitionData(partitionId, entry);
+ }
+ }
+ }
+ rwLock.writeLock().unlock();
+ }
+
+ /**
+ * Offloads partition data of a given partition in the data store to disk
+ *
+ * @param partitionId id of the partition to offload its data
+ * @param basePath path to offload the data to
+ * @throws IOException
+ */
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(
+ "UL_UNRELEASED_LOCK_EXCEPTION_PATH")
+ public void offloadPartitionData(int partitionId, String basePath)
+ throws IOException {
+ ReadWriteLock rwLock = getPartitionLock(partitionId);
+ rwLock.writeLock().lock();
+ hasPartitionDataOnDisk.add(partitionId);
+ rwLock.writeLock().unlock();
+ offloadInMemoryPartitionData(partitionId, getPath(basePath, partitionId));
+ }
+
+ /**
+ * Offloads raw data buffers of a given partition to disk
+ *
+ * @param partitionId id of the partition to offload its raw data buffers
+ * @param basePath path to offload the data to
+ * @throws IOException
+ */
+ public void offloadBuffers(int partitionId, String basePath)
+ throws IOException {
+ Pair<Integer, List<T>> pair = dataBuffers.get(partitionId);
+ if (pair == null || pair.getLeft() < minBufferSizeToOffload) {
+ return;
+ }
+ ReadWriteLock rwLock = getPartitionLock(partitionId);
+ rwLock.writeLock().lock();
+ pair = dataBuffers.remove(partitionId);
+ rwLock.writeLock().unlock();
+ checkNotNull(pair);
+ checkState(!pair.getRight().isEmpty());
+ File file = new File(getBuffersPath(basePath, partitionId));
+ FileOutputStream fos = new FileOutputStream(file, true);
+ BufferedOutputStream bos = new BufferedOutputStream(fos);
+ DataOutputStream dos = new DataOutputStream(bos);
+ for (T entry : pair.getRight()) {
+ writeEntry(entry, dos);
+ }
+ dos.close();
+ int numBuffers = pair.getRight().size();
+ Integer oldNumBuffersOnDisk =
+ numDataBuffersOnDisk.putIfAbsent(partitionId, numBuffers);
+ if (oldNumBuffersOnDisk != null) {
+ numDataBuffersOnDisk.replace(partitionId,
+ oldNumBuffersOnDisk + numBuffers);
+ }
+ }
+
+ /**
+ * Looks through all partitions that their data is not in the data store (is
+ * offloaded to disk), and sees if any of them has enough raw data buffer in
+ * memory. If so, puts that partition in a list to return.
+ *
+ * @return Set of partition ids of all partition raw buffers where the
+ * aggregate size of buffers are large enough and it is worth flushing
+ * those buffers to disk
+ */
+ public Set<Integer> getCandidateBuffersToOffload() {
+ Set<Integer> result = new HashSet<>();
+ for (Map.Entry<Integer, Pair<Integer, List<T>>> entry :
+ dataBuffers.entrySet()) {
+ if (entry.getValue().getLeft() > minBufferSizeToOffload) {
+ result.add(entry.getKey());
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Creates the path to read/write partition data from/to for a given
+ * partition.
+ *
+ * @param basePath path prefix to create the actual path from
+ * @param partitionId id of the partition
+ * @return path to read/write data from/to
+ */
+ private static String getPath(String basePath, int partitionId) {
+ return basePath + "-P" + partitionId;
+ }
+
+ /**
+ * Creates the path to read/write raw data buffers of a given partition
+ * from/to.
+ *
+ * @param basePath path prefix to create the actual path from
+ * @param partitionId id of the partition
+ * @return path to read/write raw data buffer to/from
+ */
+ private static String getBuffersPath(String basePath, int partitionId) {
+ return getPath(basePath, partitionId) + "_buffers";
+ }
+
+ /**
+ * Writes a single raw entry to a given output stream.
+ *
+ * @param entry entry to write to output
+ * @param out output stream to write the entry to
+ * @throws IOException
+ */
+ protected abstract void writeEntry(T entry, DataOutput out)
+ throws IOException;
+
+ /**
+ * Reads the next available raw entry from a given input stream.
+ *
+ * @param in input stream to read the entry from
+ * @return entry read from an input stream
+ * @throws IOException
+ */
+ protected abstract T readNextEntry(DataInput in) throws IOException;
+
+ /**
+ * Loads data of a partition into data store.
+ *
+ * @param partitionId id of the partition to load its data
+ * @param path path from which data should be loaded
+ * @throws IOException
+ */
+ protected abstract void loadInMemoryPartitionData(int partitionId,
+ String path)
+ throws IOException;
+
+ /**
+ * Offloads data of a partition in data store to disk.
+ *
+ * @param partitionId id of the partition to offload to disk
+ * @param path path to which data should be offloaded
+ * @throws IOException
+ */
+ protected abstract void offloadInMemoryPartitionData(int partitionId,
+ String path)
+ throws IOException;
+
+ /**
+ * Gets the size of a given entry in bytes.
+ *
+ * @param entry input entry to find its size
+ * @return size of given input entry in bytes
+ */
+ protected abstract int entrySerializedSize(T entry);
+
+ /**
+ * Adds a single entry for a given partition to the in-memory data store.
+ *
+ * @param partitionId id of the partition to add the data to
+ * @param entry input entry to add to the data store
+ */
+ protected abstract void addEntryToImMemoryPartitionData(int partitionId,
+ T entry);
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/data/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/data/package-info.java b/giraph-core/src/main/java/org/apache/giraph/ooc/data/package-info.java
new file mode 100644
index 0000000..5260f2c
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/data/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Package of classes related to data structures used for an out-of-core
+ * mechanism
+ */
+package org.apache.giraph.ooc.data;
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/io/IOCommand.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/io/IOCommand.java b/giraph-core/src/main/java/org/apache/giraph/ooc/io/IOCommand.java
new file mode 100644
index 0000000..eb6d2c9
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/io/IOCommand.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.io;
+
+import org.apache.giraph.ooc.OutOfCoreEngine;
+
+import java.io.IOException;
+
+/**
+ * Representation of an IO command (moving data to disk/memory) used in
+ * out-of-core mechanism.
+ */
+public abstract class IOCommand {
+ /** Id of the partition involved for the IO */
+ protected final int partitionId;
+ /** Out-of-core engine */
+ protected final OutOfCoreEngine oocEngine;
+
+ /**
+ * Constructor
+ *
+ * @param oocEngine Out-of-core engine
+ * @param partitionId Id of the partition involved in the IO
+ */
+ public IOCommand(OutOfCoreEngine oocEngine, int partitionId) {
+ this.oocEngine = oocEngine;
+ this.partitionId = partitionId;
+ }
+
+ /**
+ * GEt the id of the partition involved in the IO
+ *
+ * @return id of the partition
+ */
+ public int getPartitionId() {
+ return partitionId;
+ }
+
+ /**
+ * Execute (load/store of data) the IO command, and change the data stores
+ * appropriately based on the data loaded/stored.
+ *
+ * @param basePath the base path (prefix) to the files/folders IO command
+ * should read/write data from/to
+ * @throws IOException
+ */
+ public abstract void execute(String basePath) throws IOException;
+}
+
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/io/LoadPartitionIOCommand.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/io/LoadPartitionIOCommand.java b/giraph-core/src/main/java/org/apache/giraph/ooc/io/LoadPartitionIOCommand.java
new file mode 100644
index 0000000..c28a0da
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/io/LoadPartitionIOCommand.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.io;
+
+import com.google.common.base.Preconditions;
+import org.apache.giraph.bsp.BspService;
+import org.apache.giraph.comm.messages.MessageStore;
+import org.apache.giraph.ooc.OutOfCoreEngine;
+import org.apache.giraph.ooc.data.DiskBackedEdgeStore;
+import org.apache.giraph.ooc.data.DiskBackedMessageStore;
+import org.apache.giraph.ooc.data.DiskBackedPartitionStore;
+
+import java.io.IOException;
+
+/**
+ * IOCommand to load partition data, edge data (if in INPUT_SUPERSTEP), and
+ * message data (if in compute supersteps). Also, this command can be used to
+ * prefetch a partition to be processed in the next superstep.
+ */
+public class LoadPartitionIOCommand extends IOCommand {
+ /**
+ * Which superstep this partition should be loaded for? (can be current
+ * superstep or next superstep -- in case of prefetching).
+ */
+ private final long superstep;
+
+ /**
+ * Constructor
+ *
+ * @param oocEngine out-of-core engine
+ * @param partitionId id of the partition to be loaded
+ * @param superstep superstep to load the partition for
+ */
+ public LoadPartitionIOCommand(OutOfCoreEngine oocEngine, int partitionId,
+ long superstep) {
+ super(oocEngine, partitionId);
+ this.superstep = superstep;
+ }
+
+ @Override
+ public void execute(String basePath) throws IOException {
+ if (oocEngine.getMetaPartitionManager()
+ .startLoadingPartition(partitionId, superstep)) {
+ long currentSuperstep = oocEngine.getServiceWorker().getSuperstep();
+ DiskBackedPartitionStore partitionStore =
+ (DiskBackedPartitionStore)
+ oocEngine.getServerData().getPartitionStore();
+ partitionStore.loadPartitionData(partitionId, basePath);
+ if (currentSuperstep == BspService.INPUT_SUPERSTEP &&
+ superstep == currentSuperstep) {
+ DiskBackedEdgeStore edgeStore =
+ (DiskBackedEdgeStore) oocEngine.getServerData().getEdgeStore();
+ edgeStore.loadPartitionData(partitionId, basePath);
+ }
+ MessageStore messageStore;
+ if (currentSuperstep == superstep) {
+ messageStore = oocEngine.getServerData().getCurrentMessageStore();
+ } else {
+ Preconditions.checkState(superstep == currentSuperstep + 1);
+ messageStore = oocEngine.getServerData().getIncomingMessageStore();
+ }
+ if (messageStore != null) {
+ ((DiskBackedMessageStore) messageStore)
+ .loadPartitionData(partitionId, basePath);
+ }
+ oocEngine.getMetaPartitionManager()
+ .doneLoadingPartition(partitionId, superstep);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "LoadPartitionIOCommand: (partitionId = " + partitionId + ", " +
+ "superstep = " + superstep + ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/io/StoreDataBufferIOCommand.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/io/StoreDataBufferIOCommand.java b/giraph-core/src/main/java/org/apache/giraph/ooc/io/StoreDataBufferIOCommand.java
new file mode 100644
index 0000000..41a0682
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/io/StoreDataBufferIOCommand.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.io;
+
+import org.apache.giraph.ooc.OutOfCoreEngine;
+import org.apache.giraph.ooc.data.DiskBackedEdgeStore;
+import org.apache.giraph.ooc.data.DiskBackedMessageStore;
+import org.apache.giraph.ooc.data.DiskBackedPartitionStore;
+
+import java.io.IOException;
+
+/**
+ * IOCommand to store raw data buffers on disk.
+ */
+public class StoreDataBufferIOCommand extends IOCommand {
+ /**
+ * Types of raw data buffer to offload to disk (either vertices/edges buffer
+ * in INPUT_SUPERSTEP or incoming message buffer).
+ */
+ public enum DataBufferType { PARTITION, MESSAGE };
+ /**
+ * Type of the buffer to store on disk.
+ */
+ private final DataBufferType type;
+
+ /**
+ * Constructor
+ *
+ * @param oocEngine out-of-core engine
+ * @param partitionId id of the partition to offload its buffers
+ * @param type type of the buffer to store on disk
+ */
+ public StoreDataBufferIOCommand(OutOfCoreEngine oocEngine,
+ int partitionId,
+ DataBufferType type) {
+ super(oocEngine, partitionId);
+ this.type = type;
+ }
+
+ @Override
+ public void execute(String basePath) throws IOException {
+ if (oocEngine.getMetaPartitionManager()
+ .startOffloadingBuffer(partitionId)) {
+ switch (type) {
+ case PARTITION:
+ DiskBackedPartitionStore partitionStore =
+ (DiskBackedPartitionStore)
+ oocEngine.getServerData().getPartitionStore();
+ partitionStore.offloadBuffers(partitionId, basePath);
+ DiskBackedEdgeStore edgeStore =
+ (DiskBackedEdgeStore) oocEngine.getServerData().getEdgeStore();
+ edgeStore.offloadBuffers(partitionId, basePath);
+ break;
+ case MESSAGE:
+ DiskBackedMessageStore messageStore =
+ (DiskBackedMessageStore)
+ oocEngine.getServerData().getIncomingMessageStore();
+ messageStore.offloadBuffers(partitionId, basePath);
+ break;
+ default:
+ throw new IllegalStateException("execute: requested data buffer type " +
+ "does not exist!");
+ }
+ oocEngine.getMetaPartitionManager().doneOffloadingBuffer(partitionId);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "StoreDataBufferIOCommand: (partitionId = " + partitionId + ", " +
+ "type = " + type.name() + ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/io/StoreIncomingMessageIOCommand.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/io/StoreIncomingMessageIOCommand.java b/giraph-core/src/main/java/org/apache/giraph/ooc/io/StoreIncomingMessageIOCommand.java
new file mode 100644
index 0000000..9c1c0a2
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/io/StoreIncomingMessageIOCommand.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.io;
+
+import org.apache.giraph.ooc.OutOfCoreEngine;
+import org.apache.giraph.ooc.data.DiskBackedMessageStore;
+
+import java.io.IOException;
+
+import static com.google.common.base.Preconditions.checkState;
+
+/**
+ * IOCommand to store incoming message of a particular partition.
+ */
+public class StoreIncomingMessageIOCommand extends IOCommand {
+ /**
+ * Constructor
+ *
+ * @param oocEngine out-of-core engine
+ * @param partitionId id of the partition to store its incoming messages
+ */
+ public StoreIncomingMessageIOCommand(OutOfCoreEngine oocEngine,
+ int partitionId) {
+ super(oocEngine, partitionId);
+ }
+
+ @Override
+ public void execute(String basePath) throws IOException {
+ if (oocEngine.getMetaPartitionManager()
+ .startOffloadingMessages(partitionId)) {
+ DiskBackedMessageStore messageStore =
+ (DiskBackedMessageStore)
+ oocEngine.getServerData().getIncomingMessageStore();
+ checkState(messageStore != null);
+ messageStore.offloadPartitionData(partitionId, basePath);
+ oocEngine.getMetaPartitionManager().doneOffloadingMessages(partitionId);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "StoreIncomingMessageIOCommand: (partitionId = " + partitionId + ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/io/StorePartitionIOCommand.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/io/StorePartitionIOCommand.java b/giraph-core/src/main/java/org/apache/giraph/ooc/io/StorePartitionIOCommand.java
new file mode 100644
index 0000000..77955dc
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/io/StorePartitionIOCommand.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.io;
+
+import org.apache.giraph.bsp.BspService;
+import org.apache.giraph.comm.messages.MessageStore;
+import org.apache.giraph.ooc.data.DiskBackedEdgeStore;
+import org.apache.giraph.ooc.data.DiskBackedMessageStore;
+import org.apache.giraph.ooc.data.DiskBackedPartitionStore;
+import org.apache.giraph.ooc.OutOfCoreEngine;
+
+import java.io.IOException;
+
+/**
+ * IOCommand to store partition data, edge data (if in INPUT_SUPERSTEP), and
+ * message data (if in compute supersteps).
+ */
+public class StorePartitionIOCommand extends IOCommand {
+ /**
+ * Constructor
+ *
+ * @param oocEngine out-of-core engine
+ * @param partitionId id of the partition to store its data
+ */
+ public StorePartitionIOCommand(OutOfCoreEngine oocEngine,
+ int partitionId) {
+ super(oocEngine, partitionId);
+ }
+
+ @Override
+ public void execute(String basePath) throws IOException {
+ if (oocEngine.getMetaPartitionManager()
+ .startOffloadingPartition(partitionId)) {
+ DiskBackedPartitionStore partitionStore =
+ (DiskBackedPartitionStore)
+ oocEngine.getServerData().getPartitionStore();
+ partitionStore.offloadPartitionData(partitionId, basePath);
+ if (oocEngine.getServiceWorker().getSuperstep() !=
+ BspService.INPUT_SUPERSTEP) {
+ MessageStore messageStore =
+ oocEngine.getServerData().getCurrentMessageStore();
+ if (messageStore != null) {
+ ((DiskBackedMessageStore) messageStore)
+ .offloadPartitionData(partitionId, basePath);
+ }
+ } else {
+ DiskBackedEdgeStore edgeStore =
+ (DiskBackedEdgeStore)
+ oocEngine.getServerData().getEdgeStore();
+ edgeStore.offloadPartitionData(partitionId, basePath);
+ }
+ oocEngine.getMetaPartitionManager().doneOffloadingPartition(partitionId);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "StorePartitionIOCommand: (partitionId = " + partitionId + ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/io/WaitIOCommand.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/io/WaitIOCommand.java b/giraph-core/src/main/java/org/apache/giraph/ooc/io/WaitIOCommand.java
new file mode 100644
index 0000000..b6e0546
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/io/WaitIOCommand.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.io;
+
+import org.apache.giraph.ooc.OutOfCoreEngine;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * IOCommand to do nothing regarding moving data to/from disk.
+ */
+public class WaitIOCommand extends IOCommand {
+ /** How long should the disk be idle? (in milliseconds) */
+ private final long waitDuration;
+
+ /**
+ * Constructor
+ *
+ * @param oocEngine out-of-core engine
+ * @param waitDuration duration of wait
+ */
+ public WaitIOCommand(OutOfCoreEngine oocEngine, long waitDuration) {
+ super(oocEngine, -1);
+ this.waitDuration = waitDuration;
+ }
+
+ @Override
+ public void execute(String basePath) throws IOException {
+ try {
+ TimeUnit.MILLISECONDS.sleep(waitDuration);
+ } catch (InterruptedException e) {
+ throw new IllegalStateException("execute: caught InterruptedException " +
+ "while IO thread is waiting!");
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "WaitIOCommand: (duration = " + waitDuration + "ms)";
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/io/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/io/package-info.java b/giraph-core/src/main/java/org/apache/giraph/ooc/io/package-info.java
new file mode 100644
index 0000000..2230ec4
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/io/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Package of classes related to IO operations in out-of-core mechanism
+ */
+package org.apache.giraph.ooc.io;
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/partition/PartitionData.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/partition/PartitionData.java b/giraph-core/src/main/java/org/apache/giraph/partition/PartitionData.java
deleted file mode 100644
index d6e3a70..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/partition/PartitionData.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.partition;
-
-import org.apache.giraph.utils.ExtendedDataOutput;
-import org.apache.giraph.utils.VertexIdEdges;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-/**
- * Structure that keeps partition information.
- *
- * @param <I> Vertex id
- * @param <V> Vertex value
- * @param <E> Edge value
- */
-public interface PartitionData<I extends WritableComparable,
- V extends Writable, E extends Writable> {
- /**
- * Add a *new* partition to the store. If the partition is already existed,
- * it does not add the partition and returns false.
- *
- * @param partition Partition to add
- * @return Whether the addition made any change in the partition store
- */
- boolean addPartition(Partition<I, V, E> partition);
-
- /**
- * Remove a partition and return it. Called from a single thread, *not* from
- * within a scheduling cycle, and after INPUT_SUPERSTEP is complete.
- *
- * @param partitionId Partition id
- * @return The removed partition
- */
- Partition<I, V, E> removePartition(Integer partitionId);
-
- /**
- * Whether a specific partition is present in the store.
- *
- * @param partitionId Partition id
- * @return True iff the partition is present
- */
- boolean hasPartition(Integer partitionId);
-
- /**
- * Return the ids of all the stored partitions as an Iterable.
- *
- * @return The partition ids
- */
- Iterable<Integer> getPartitionIds();
-
- /**
- * Return the number of stored partitions.
- *
- * @return The number of partitions
- */
- int getNumPartitions();
-
- /**
- * Return the number of vertices in a partition.
- *
- * @param partitionId Partition id
- * @return The number of vertices in the specified partition
- */
- long getPartitionVertexCount(Integer partitionId);
-
- /**
- * Return the number of edges in a partition.
- *
- * @param partitionId Partition id
- * @return The number of edges in the specified partition
- */
- long getPartitionEdgeCount(Integer partitionId);
-
- /**
- * Whether the partition store is empty.
- *
- * @return True iff there are no partitions in the store
- */
- boolean isEmpty();
-
- /**
- * Add vertices to a given partition from a given DataOutput instance. This
- * method is called right after receipt of vertex request in INPUT_SUPERSTEP.
- *
- * @param partitionId Partition id
- * @param extendedDataOutput Output containing serialized vertex data
- */
- void addPartitionVertices(Integer partitionId,
- ExtendedDataOutput extendedDataOutput);
-
- /**
- * Add edges to a given partition from a given send edge request. This
- * method is called right after receipt of edge request in INPUT_SUPERSTEP.
- *
- * @param partitionId Partition id
- * @param edges Edges in the request
- */
- void addPartitionEdges(Integer partitionId, VertexIdEdges<I, E> edges);
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/partition/PartitionStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/partition/PartitionStore.java b/giraph-core/src/main/java/org/apache/giraph/partition/PartitionStore.java
index 2facff8..5ff15eb 100644
--- a/giraph-core/src/main/java/org/apache/giraph/partition/PartitionStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/partition/PartitionStore.java
@@ -18,152 +18,106 @@
package org.apache.giraph.partition;
-import org.apache.giraph.bsp.CentralizedServiceWorker;
-import org.apache.giraph.comm.messages.MessageData;
-import org.apache.giraph.comm.messages.MessageStore;
-import org.apache.giraph.comm.messages.MessageStoreFactory;
-import org.apache.giraph.comm.messages.queue.AsyncMessageStoreWrapper;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.edge.EdgeStore;
-import org.apache.giraph.edge.EdgeStoreFactory;
-import org.apache.giraph.utils.ReflectionUtils;
+import org.apache.giraph.utils.ExtendedDataOutput;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapreduce.Mapper;
-
-import java.io.IOException;
-
-import static org.apache.giraph.conf.GiraphConstants.MESSAGE_STORE_FACTORY_CLASS;
/**
* Structure that stores partitions for a worker. PartitionStore does not allow
* random accesses to partitions except upon removal.
- * This structure is thread-safe.
+ * This structure is thread-safe, unless otherwise specified.
*
* @param <I> Vertex id
* @param <V> Vertex data
* @param <E> Edge data
*/
-public abstract class PartitionStore<I extends WritableComparable,
- V extends Writable, E extends Writable>
- implements PartitionData<I, V, E>, MessageData<I> {
- /** Configuration. */
- protected final ImmutableClassesGiraphConfiguration<I, V, E> conf;
- /** Context used to report progress */
- protected final Mapper<?, ?, ?, ?>.Context context;
- /** service worker reference */
- protected final CentralizedServiceWorker<I, V, E> serviceWorker;
-
- /** Edge store for this worker */
- protected final EdgeStore<I, V, E> edgeStore;
-
- /** Message store factory */
- protected MessageStoreFactory<I, Writable, MessageStore<I, Writable>>
- messageStoreFactory;
+public interface PartitionStore<I extends WritableComparable,
+ V extends Writable, E extends Writable> {
+ /**
+ * Add a *new* partition to the store. If the partition is already existed,
+ * it does not add the partition and returns false.
+ * Note: this method is not thread-safe and should be called by a single
+ * thread.
+ *
+ * @param partition Partition to add
+ * @return Whether the addition made any change in the partition store
+ */
+ boolean addPartition(Partition<I, V, E> partition);
+
+ /**
+ * Remove a partition and return it. Called from a single thread, *not* from
+ * within a scheduling cycle. This method should *not* be called in
+ * INPUT_SUPERSTEP.
+ *
+ * @param partitionId Partition id
+ * @return The removed partition
+ */
+ Partition<I, V, E> removePartition(Integer partitionId);
+
/**
- * Message store for incoming messages (messages which will be consumed
- * in the next super step)
+ * Whether a specific partition is present in the store.
+ *
+ * @param partitionId Partition id
+ * @return True iff the partition is present
*/
- protected volatile MessageStore<I, Writable> incomingMessageStore;
+ boolean hasPartition(Integer partitionId);
+
/**
- * Message store for current messages (messages which we received in
- * previous super step and which will be consumed in current super step)
+ * Return the ids of all the stored partitions as an Iterable.
+ *
+ * @return The partition ids
*/
- protected volatile MessageStore<I, Writable> currentMessageStore;
+ Iterable<Integer> getPartitionIds();
/**
- * Constructor for abstract partition store
+ * Return the number of stored partitions.
*
- * @param conf Job configuration
- * @param context Mapper context
- * @param serviceWorker Worker service
+ * @return The number of partitions
*/
- public PartitionStore(ImmutableClassesGiraphConfiguration<I, V, E> conf,
- Mapper<?, ?, ?, ?>.Context context,
- CentralizedServiceWorker<I, V, E> serviceWorker) {
- this.conf = conf;
- this.context = context;
- this.serviceWorker = serviceWorker;
- this.messageStoreFactory = createMessageStoreFactory();
- EdgeStoreFactory<I, V, E> edgeStoreFactory = conf.createEdgeStoreFactory();
- edgeStoreFactory.initialize(serviceWorker, conf, context);
- this.edgeStore = edgeStoreFactory.newStore();
- }
+ int getNumPartitions();
/**
- * Decide which message store should be used for current application,
- * and create the factory for that store
+ * Return the number of vertices in a partition.
*
- * @return Message store factory
+ * @param partitionId Partition id
+ * @return The number of vertices in the specified partition
*/
- private MessageStoreFactory<I, Writable, MessageStore<I, Writable>>
- createMessageStoreFactory() {
- Class<? extends MessageStoreFactory> messageStoreFactoryClass =
- MESSAGE_STORE_FACTORY_CLASS.get(conf);
-
- MessageStoreFactory messageStoreFactoryInstance =
- ReflectionUtils.newInstance(messageStoreFactoryClass);
- messageStoreFactoryInstance.initialize(serviceWorker, conf);
-
- return messageStoreFactoryInstance;
- }
-
- @Override
- public boolean isEmpty() {
- return getNumPartitions() == 0;
- }
-
- @Override
- public <M extends Writable> MessageStore<I, M> getIncomingMessageStore() {
- return (MessageStore<I, M>) incomingMessageStore;
- }
-
- @Override
- public <M extends Writable> MessageStore<I, M> getCurrentMessageStore() {
- return (MessageStore<I, M>) currentMessageStore;
- }
-
- @Override
- public void resetMessageStores() throws IOException {
- if (currentMessageStore != null) {
- currentMessageStore.clearAll();
- currentMessageStore = null;
- }
- if (incomingMessageStore != null) {
- incomingMessageStore.clearAll();
- incomingMessageStore = null;
- }
- prepareSuperstep();
- }
-
- /** Prepare for next super step */
- public void prepareSuperstep() {
- if (currentMessageStore != null) {
- try {
- currentMessageStore.clearAll();
- } catch (IOException e) {
- throw new IllegalStateException(
- "Failed to clear previous message store");
- }
- }
- currentMessageStore = incomingMessageStore != null ?
- incomingMessageStore :
- messageStoreFactory.newStore(conf.getIncomingMessageClasses());
- incomingMessageStore =
- messageStoreFactory.newStore(conf.getOutgoingMessageClasses());
- // finalize current message-store before resolving mutations
- currentMessageStore.finalizeStore();
- }
+ long getPartitionVertexCount(Integer partitionId);
+
+ /**
+ * Return the number of edges in a partition.
+ *
+ * @param partitionId Partition id
+ * @return The number of edges in the specified partition
+ */
+ long getPartitionEdgeCount(Integer partitionId);
+
+ /**
+ * Whether the partition store is empty.
+ *
+ * @return True iff there are no partitions in the store
+ */
+ boolean isEmpty();
+
+ /**
+ * Add vertices to a given partition from a given DataOutput instance. This
+ * method is called right after receipt of vertex request in INPUT_SUPERSTEP.
+ *
+ * @param partitionId Partition id
+ * @param extendedDataOutput Output containing serialized vertex data
+ */
+ void addPartitionVertices(Integer partitionId,
+ ExtendedDataOutput extendedDataOutput);
/**
* Called at the end of the computation. Called from a single thread.
*/
- public void shutdown() { }
+ void shutdown();
/**
* Called at the beginning of the computation. Called from a single thread.
*/
- public void initialize() { }
+ void initialize();
/**
* Start the iteration cycle to iterate over partitions. Note that each
@@ -194,7 +148,7 @@ public abstract class PartitionStore<I extends WritableComparable,
*
* Called from a single thread.
*/
- public abstract void startIteration();
+ void startIteration();
/**
* Return the next partition in iteration for the current superstep.
@@ -204,7 +158,7 @@ public abstract class PartitionStore<I extends WritableComparable,
*
* @return The next partition to process
*/
- public abstract Partition<I, V, E> getNextPartition();
+ Partition<I, V, E> getNextPartition();
/**
* Put a partition back to the store. Use this method to put a partition
@@ -213,21 +167,5 @@ public abstract class PartitionStore<I extends WritableComparable,
*
* @param partition Partition
*/
- public abstract void putPartition(Partition<I, V, E> partition);
-
- /**
- * Move edges from edge store to partitions. This method is called from a
- * *single thread* once all vertices and edges are read in INPUT_SUPERSTEP.
- */
- public abstract void moveEdgesToVertices();
-
- /**
- * In case of async message store we have to wait for all messages
- * to be processed before going into next superstep.
- */
- public void waitForComplete() {
- if (incomingMessageStore instanceof AsyncMessageStoreWrapper) {
- ((AsyncMessageStoreWrapper) incomingMessageStore).waitToComplete();
- }
- }
+ void putPartition(Partition<I, V, E> partition);
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/partition/SimplePartitionStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/partition/SimplePartitionStore.java b/giraph-core/src/main/java/org/apache/giraph/partition/SimplePartitionStore.java
index 9f0c408..4846702 100644
--- a/giraph-core/src/main/java/org/apache/giraph/partition/SimplePartitionStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/partition/SimplePartitionStore.java
@@ -19,18 +19,13 @@
package org.apache.giraph.partition;
import com.google.common.collect.Maps;
-import org.apache.giraph.bsp.CentralizedServiceWorker;
-import org.apache.giraph.comm.messages.MessageStore;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.utils.ExtendedDataOutput;
-import org.apache.giraph.utils.VertexIdEdges;
-import org.apache.giraph.utils.VertexIdMessages;
import org.apache.giraph.utils.VertexIterator;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Mapper;
-import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentMap;
@@ -46,7 +41,11 @@ import static com.google.common.base.Preconditions.checkState;
*/
public class SimplePartitionStore<I extends WritableComparable,
V extends Writable, E extends Writable>
- extends PartitionStore<I, V, E> {
+ implements PartitionStore<I, V, E> {
+ /** Configuration */
+ private final ImmutableClassesGiraphConfiguration<I, V, E> conf;
+ /** Job context (for progress) */
+ private final Mapper<?, ?, ?, ?>.Context context;
/** Map of stored partitions. */
private final ConcurrentMap<Integer, Partition<I, V, E>> partitions =
Maps.newConcurrentMap();
@@ -57,12 +56,11 @@ public class SimplePartitionStore<I extends WritableComparable,
* Constructor.
* @param conf Configuration
* @param context Mapper context
- * @param serviceWorker Service worker
*/
public SimplePartitionStore(ImmutableClassesGiraphConfiguration<I, V, E> conf,
- Mapper<?, ?, ?, ?>.Context context,
- CentralizedServiceWorker<I, V, E> serviceWorker) {
- super(conf, context, serviceWorker);
+ Mapper<?, ?, ?, ?>.Context context) {
+ this.conf = conf;
+ this.context = context;
}
@Override
@@ -111,6 +109,11 @@ public class SimplePartitionStore<I extends WritableComparable,
}
@Override
+ public boolean isEmpty() {
+ return partitions.size() == 0;
+ }
+
+ @Override
public void startIteration() {
checkState(partitionQueue == null || partitionQueue.isEmpty(),
"startIteration: It seems that some of " +
@@ -162,27 +165,8 @@ public class SimplePartitionStore<I extends WritableComparable,
}
@Override
- public void addPartitionEdges(Integer partitionId,
- VertexIdEdges<I, E> edges) {
- edgeStore.addPartitionEdges(partitionId, edges);
- }
+ public void shutdown() { }
@Override
- public void moveEdgesToVertices() {
- edgeStore.moveEdgesToVertices();
- }
-
- @Override
- public <M extends Writable> void addPartitionCurrentMessages(
- int partitionId, VertexIdMessages<I, M> messages) throws IOException {
- ((MessageStore<I, M>) currentMessageStore)
- .addPartitionMessages(partitionId, messages);
- }
-
- @Override
- public <M extends Writable> void addPartitionIncomingMessages(
- int partitionId, VertexIdMessages<I, M> messages) throws IOException {
- ((MessageStore<I, M>) incomingMessageStore)
- .addPartitionMessages(partitionId, messages);
- }
+ public void initialize() { }
}
[3/5] git commit: updated refs/heads/trunk to fafecee
Posted by ed...@apache.org.
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/FixedOutOfCoreIOScheduler.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/FixedOutOfCoreIOScheduler.java b/giraph-core/src/main/java/org/apache/giraph/ooc/FixedOutOfCoreIOScheduler.java
new file mode 100644
index 0000000..2cb002f
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/FixedOutOfCoreIOScheduler.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.ooc.io.IOCommand;
+import org.apache.giraph.ooc.io.LoadPartitionIOCommand;
+import org.apache.giraph.ooc.io.StoreDataBufferIOCommand;
+import org.apache.giraph.ooc.io.StoreIncomingMessageIOCommand;
+import org.apache.giraph.ooc.io.StorePartitionIOCommand;
+import org.apache.giraph.ooc.io.WaitIOCommand;
+import org.apache.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static com.google.common.base.Preconditions.checkState;
+
+/**
+ * IO Scheduler for out-of-core mechanism with fixed number of partitions in
+ * memory
+ */
+public class FixedOutOfCoreIOScheduler extends OutOfCoreIOScheduler {
+ /** Class logger */
+ private static final Logger LOG =
+ Logger.getLogger(FixedOutOfCoreIOScheduler.class);
+ /** Maximum number of partitions to be kept in memory */
+ private final int maxPartitionsInMemory;
+ /**
+ * Number of partitions to be added (loaded) or removed (stored) to/from
+ * memory. Each outstanding load partition counts +1 and each outstanding
+ * store partition counts -1 toward this counter.
+ */
+ private final AtomicInteger deltaNumPartitionsInMemory =
+ new AtomicInteger(0);
+ /** Queue of IO commands for loading partitions to memory */
+ private final List<Queue<IOCommand>> threadLoadCommandQueue;
+ /** Queue of IO commands for storing partition on disk */
+ private final List<Queue<IOCommand>> threadStoreCommandQueue;
+ /** Whether IO threads should terminate */
+ private volatile boolean shouldTerminate;
+
+ /**
+ * Constructor
+ * @param maxPartitionsInMemory maximum number of partitions can be kept in
+ * memory
+ * @param numThreads number of available IO threads (i.e. disks)
+ * @param oocEngine out-of-core engine
+ * @param conf configuration
+ */
+ public FixedOutOfCoreIOScheduler(int maxPartitionsInMemory, int numThreads,
+ OutOfCoreEngine oocEngine,
+ ImmutableClassesGiraphConfiguration conf) {
+ super(conf, oocEngine, numThreads);
+ this.maxPartitionsInMemory = maxPartitionsInMemory;
+ threadLoadCommandQueue = new ArrayList<>(numThreads);
+ threadStoreCommandQueue = new ArrayList<>(numThreads);
+ for (int i = 0; i < numThreads; ++i) {
+ threadLoadCommandQueue.add(
+ new ConcurrentLinkedQueue<IOCommand>());
+ threadStoreCommandQueue.add(
+ new ConcurrentLinkedQueue<IOCommand>());
+ }
+ shouldTerminate = false;
+ }
+
+ @Override
+ public IOCommand getNextIOCommand(int threadId) {
+ if (shouldTerminate) {
+ return null;
+ }
+ int numPartitionsInMemory =
+ oocEngine.getMetaPartitionManager().getNumInMemoryPartitions();
+ IOCommand command = null;
+ if (LOG.isInfoEnabled()) {
+ LOG.info("getNextIOCommand with " + numPartitionsInMemory +
+ " partitions in memory, " + deltaNumPartitionsInMemory.get() +
+ " on the fly");
+ }
+ // Check if we have to store a partition on disk
+ if (numPartitionsInMemory + deltaNumPartitionsInMemory.getAndDecrement() >
+ maxPartitionsInMemory) {
+ command = threadStoreCommandQueue.get(threadId).poll();
+ if (command == null) {
+ Integer partitionId = oocEngine.getMetaPartitionManager()
+ .getOffloadPartitionId(threadId);
+ if (partitionId != null) {
+ command = new StorePartitionIOCommand(oocEngine, partitionId);
+ } else {
+ deltaNumPartitionsInMemory.getAndIncrement();
+ }
+ } else {
+ checkState(command instanceof StorePartitionIOCommand,
+ "getNextIOCommand: Illegal command type in store command queue!");
+ }
+ } else {
+ // Roll back the decrement in delta counter.
+ deltaNumPartitionsInMemory.getAndIncrement();
+ }
+
+ // Check if there is any buffers/messages of current out-of-core partitions
+ // in memory
+ if (command == null) {
+ Integer partitionId = oocEngine.getMetaPartitionManager()
+ .getOffloadPartitionBufferId(threadId);
+ if (partitionId != null) {
+ command = new StoreDataBufferIOCommand(oocEngine, partitionId,
+ StoreDataBufferIOCommand.DataBufferType.PARTITION);
+ } else {
+ partitionId = oocEngine.getMetaPartitionManager()
+ .getOffloadMessageBufferId(threadId);
+ if (partitionId != null) {
+ command = new StoreDataBufferIOCommand(oocEngine, partitionId,
+ StoreDataBufferIOCommand.DataBufferType.MESSAGE);
+ } else {
+ partitionId = oocEngine.getMetaPartitionManager()
+ .getOffloadMessageId(threadId);
+ if (partitionId != null) {
+ command = new StoreIncomingMessageIOCommand(oocEngine, partitionId);
+ }
+ }
+ }
+ }
+
+ // Check if we can load/prefetch a partition to memory
+ if (command == null) {
+ if (numPartitionsInMemory +
+ deltaNumPartitionsInMemory.getAndIncrement() <=
+ maxPartitionsInMemory) {
+ command = threadLoadCommandQueue.get(threadId).poll();
+ if (command == null) {
+ Integer partitionId = oocEngine.getMetaPartitionManager()
+ .getPrefetchPartitionId(threadId);
+ if (partitionId != null) {
+ command = new LoadPartitionIOCommand(oocEngine, partitionId,
+ oocEngine.getServiceWorker().getSuperstep());
+ } else {
+ deltaNumPartitionsInMemory.getAndDecrement();
+ }
+ }
+ } else {
+ // Roll back the increment in delta counter.
+ deltaNumPartitionsInMemory.getAndDecrement();
+ }
+ }
+
+ // Check if no appropriate IO command is found
+ if (command == null) {
+ command = new WaitIOCommand(oocEngine, waitInterval);
+ }
+ return command;
+ }
+
+ @Override
+ public void ioCommandCompleted(IOCommand command) {
+ if (command instanceof LoadPartitionIOCommand) {
+ deltaNumPartitionsInMemory.getAndDecrement();
+ } else if (command instanceof StorePartitionIOCommand) {
+ deltaNumPartitionsInMemory.getAndIncrement();
+ }
+ oocEngine.ioCommandCompleted(command);
+ }
+
+ @Override
+ public void addIOCommand(IOCommand ioCommand) {
+ int ownerThread = getOwnerThreadId(ioCommand.getPartitionId());
+ if (ioCommand instanceof LoadPartitionIOCommand) {
+ threadLoadCommandQueue.get(ownerThread).offer(ioCommand);
+ } else if (ioCommand instanceof StorePartitionIOCommand) {
+ threadStoreCommandQueue.get(ownerThread).offer(ioCommand);
+ } else {
+ throw new IllegalStateException("addIOCommand: IO command type is not " +
+ "supported for addition");
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ super.shutdown();
+ shouldTerminate = true;
+ }
+
+ /**
+ * Clear store command queue (should happen at the beginning of each iteration
+ * to eliminate eager store commands generated by OOC engine)
+ */
+ public void clearStoreCommandQueue() {
+ for (int i = 0; i < threadStoreCommandQueue.size(); ++i) {
+ threadStoreCommandQueue.get(i).clear();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/JVMMemoryEstimator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/JVMMemoryEstimator.java b/giraph-core/src/main/java/org/apache/giraph/ooc/JVMMemoryEstimator.java
deleted file mode 100644
index 46d989a..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/JVMMemoryEstimator.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.ooc;
-
-import org.apache.giraph.bsp.CentralizedServiceWorker;
-import org.apache.giraph.utils.MemoryUtils;
-
-/**
- * Memory estimator class using JVM runtime methods to estimate the
- * free/available memory.
- */
-public class JVMMemoryEstimator implements MemoryEstimator {
- /**
- * Constructor for reflection
- */
- public JVMMemoryEstimator() { }
-
- @Override
- public void initialize(CentralizedServiceWorker serviceWorker) { }
-
- @Override
- public double freeMemoryMB() {
- return MemoryUtils.freePlusUnallocatedMemoryMB();
- }
-
- @Override public double maxMemoryMB() {
- return MemoryUtils.maxMemoryMB();
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/MemoryEstimator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/MemoryEstimator.java b/giraph-core/src/main/java/org/apache/giraph/ooc/MemoryEstimator.java
deleted file mode 100644
index 74a2f7b..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/MemoryEstimator.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.ooc;
-
-import org.apache.giraph.bsp.CentralizedServiceWorker;
-
-/**
- * Interface for memory estimator. Estimated memory is used in adaptive
- * out-of-core mechanism.
- */
-public interface MemoryEstimator {
- /**
- * Initialize the memory estimator.
- *
- * @param serviceWorker Worker service
- */
- void initialize(CentralizedServiceWorker serviceWorker);
-
- /**
- * @return amount of free memory in MB
- */
- double freeMemoryMB();
-
- /**
- * @return amount of available memory in MB
- */
- double maxMemoryMB();
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreEngine.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreEngine.java b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreEngine.java
index 218a10b..bc0ece4 100644
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreEngine.java
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreEngine.java
@@ -18,26 +18,189 @@
package org.apache.giraph.ooc;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
+import org.apache.giraph.bsp.CentralizedServiceWorker;
+import org.apache.giraph.comm.ServerData;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.ooc.data.MetaPartitionManager;
+import org.apache.giraph.ooc.io.IOCommand;
+import org.apache.log4j.Logger;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static com.google.common.base.Preconditions.checkState;
/**
- * Engine for out-of-core mechanism. The engine may attach to a partition store
- * capable of handling disk transfer of partitions.
- *
- * @param <I> Vertex id
- * @param <V> Vertex data
- * @param <E> Edge data
+ * Class to represent an out-of-core engine.
*/
-public interface OutOfCoreEngine<I extends WritableComparable,
- V extends Writable, E extends Writable> {
+public abstract class OutOfCoreEngine {
+ /** Class logger. */
+ private static final Logger LOG = Logger.getLogger(OutOfCoreEngine.class);
+ /** Service worker */
+ protected final CentralizedServiceWorker<?, ?, ?> service;
+ /** Scheduler for IO threads */
+ protected OutOfCoreIOScheduler ioScheduler;
+ /** Data structure to keep meta partition information */
+ protected final MetaPartitionManager metaPartitionManager;
+ /** How many disk (i.e. IO threads) do we have? */
+ protected final int numIOThreads;
+ /**
+ * Whether the job should fail due to IO threads terminating because of
+ * exceptions
+ */
+ protected volatile boolean jobFailed = false;
+ /** Whether the out-of-core engine has initialized */
+ private final AtomicBoolean isInitialized = new AtomicBoolean(false);
+ /**
+ * Global lock for entire superstep. This lock helps to avoid overlapping of
+ * out-of-core decisions (what to do next to help the out-of-core mechanism)
+ * with out-of-core operations (actual IO operations).
+ */
+ private final ReadWriteLock superstepLock = new ReentrantReadWriteLock();
+ /** Callable factory for IO threads */
+ private final OutOfCoreIOCallableFactory oocIOCallableFactory;
+
+ /**
+ * Constructor
+ *
+ * @param conf Configuration
+ * @param service Service worker
+ */
+ public OutOfCoreEngine(ImmutableClassesGiraphConfiguration<?, ?, ?> conf,
+ CentralizedServiceWorker<?, ?, ?> service) {
+ this.service = service;
+ this.oocIOCallableFactory = new OutOfCoreIOCallableFactory(conf, this);
+ this.numIOThreads = oocIOCallableFactory.getNumDisks();
+ this.metaPartitionManager = new MetaPartitionManager(numIOThreads, this);
+ oocIOCallableFactory.createCallable();
+ }
+
/**
* Initialize/Start the out-of-core engine.
*/
- void initialize();
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(
+ "JLM_JSR166_UTILCONCURRENT_MONITORENTER")
+ public void initialize() {
+ synchronized (isInitialized) {
+ isInitialized.set(true);
+ isInitialized.notifyAll();
+ }
+ }
/**
* Shutdown/Stop the out-of-core engine.
*/
- void shutdown();
+ public void shutdown() {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("shutdown: out-of-core engine shutting down, signalling IO " +
+ "threads to shutdown");
+ }
+ ioScheduler.shutdown();
+ oocIOCallableFactory.shutdown();
+ }
+
+ /**
+ * Get a reference to the server data
+ *
+ * @return ServerData
+ */
+ public ServerData getServerData() {
+ return service.getServerData();
+ }
+
+ /**
+ * Get a reference to the service worker
+ *
+ * @return CentralizedServiceWorker
+ */
+ public CentralizedServiceWorker getServiceWorker() {
+ return service;
+ }
+
+ /**
+ * Get a reference to IO scheduler
+ *
+ * @return OutOfCoreIOScheduler
+ */
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(
+ "JLM_JSR166_UTILCONCURRENT_MONITORENTER")
+ public OutOfCoreIOScheduler getIOScheduler() {
+ synchronized (isInitialized) {
+ while (!isInitialized.get()) {
+ try {
+ isInitialized.wait();
+ } catch (InterruptedException e) {
+ throw new IllegalStateException("getIOScheduler: " +
+ "InterruptedException while waiting for out-of-core engine to " +
+ "be initialized!");
+ }
+ }
+ }
+ return ioScheduler;
+ }
+
+ /**
+ * Get a reference to meta partition information
+ *
+ * @return MetaPartitionManager
+ */
+ public MetaPartitionManager getMetaPartitionManager() {
+ checkState(isInitialized.get());
+ return metaPartitionManager;
+ }
+
+ /**
+ * Get a refernce to superstep lock
+ *
+ * @return read/write lock used for global superstep lock
+ */
+ public ReadWriteLock getSuperstepLock() {
+ return superstepLock;
+ }
+
+ /**
+ * Get the id of the next partition to process in the current iteration over
+ * all the partitions. If all partitions are already processed, this method
+ * returns null.
+ *
+ * @return id of a partition to process. 'null' if all partitions are
+ * processed in current iteration over partitions.
+ */
+ public abstract Integer getNextPartition();
+
+ /**
+ * Notify out-of-core engine that processing of a particular partition is done
+ *
+ * @param partitionId id of the partition that its processing is done
+ */
+ public abstract void doneProcessingPartition(int partitionId);
+
+ /**
+ * Notify out=of-core engine that iteration cycle over all partitions is about
+ * to begin.
+ */
+ public abstract void startIteration();
+
+ /**
+ * Retrieve a particular partition. After this method is complete the
+ * requested partition should be in memory.
+ *
+ * @param partitionId id of the partition to retrieve
+ */
+ public abstract void retrievePartition(int partitionId);
+
+ /**
+ * Notify out-of-core engine that an IO command is competed by an IO thread
+ *
+ * @param command the IO command that is completed
+ */
+ public abstract void ioCommandCompleted(IOCommand command);
+
+ /**
+ * Set a flag to fail the job.
+ */
+ public void failTheJob() {
+ jobFailed = true;
+ }
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallable.java b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallable.java
new file mode 100644
index 0000000..6c6e4ff
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallable.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc;
+
+import org.apache.giraph.ooc.io.IOCommand;
+import org.apache.giraph.ooc.io.WaitIOCommand;
+import org.apache.log4j.Logger;
+
+import java.util.concurrent.Callable;
+
+/**
+ * IO threads for out-of-core mechanism.
+ */
+public class OutOfCoreIOCallable implements Callable<Void> {
+ /** Class logger. */
+ private static final Logger LOG = Logger.getLogger(OutOfCoreIOCallable.class);
+ /** Out-of-core engine */
+ private final OutOfCoreEngine oocEngine;
+ /** Base path that this thread will write to/read from */
+ private final String basePath;
+ /** Thread id/Disk id */
+ private final int diskId;
+
+ /**
+ * Constructor
+ *
+ * @param oocEngine out-of-core engine
+ * @param basePath base path this thread will be using
+ * @param diskId thread id/disk id
+ */
+ public OutOfCoreIOCallable(OutOfCoreEngine oocEngine, String basePath,
+ int diskId) {
+ this.oocEngine = oocEngine;
+ this.basePath = basePath;
+ this.diskId = diskId;
+ }
+
+ @Override
+ public Void call() {
+ while (true) {
+ oocEngine.getSuperstepLock().readLock().lock();
+ IOCommand command = oocEngine.getIOScheduler().getNextIOCommand(diskId);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("call: thread " + diskId + "'s next IO command is: " +
+ command);
+ }
+ if (command == null) {
+ oocEngine.getSuperstepLock().readLock().unlock();
+ break;
+ }
+ if (command instanceof WaitIOCommand) {
+ oocEngine.getSuperstepLock().readLock().unlock();
+ }
+ // CHECKSTYLE: stop IllegalCatch
+ try {
+ command.execute(basePath);
+ } catch (Exception e) {
+ oocEngine.failTheJob();
+ LOG.info("call: execution of IO command " + command + " failed!");
+ throw new RuntimeException(e);
+ }
+ // CHECKSTYLE: resume IllegalCatch
+ if (!(command instanceof WaitIOCommand)) {
+ oocEngine.getSuperstepLock().readLock().unlock();
+ }
+ oocEngine.getIOScheduler().ioCommandCompleted(command);
+ }
+ if (LOG.isInfoEnabled()) {
+ LOG.info("call: out-of-core IO thread " + diskId + " terminating!");
+ }
+ return null;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallableFactory.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallableFactory.java b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallableFactory.java
new file mode 100644
index 0000000..d4fea22
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallableFactory.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.utils.CallableFactory;
+import org.apache.giraph.utils.LogStacktraceCallable;
+import org.apache.giraph.utils.ThreadUtils;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import static com.google.common.base.Preconditions.checkState;
+import static org.apache.giraph.conf.GiraphConstants.PARTITIONS_DIRECTORY;
+
+/**
+ * Factory class to create IO threads for out-of-core engine.
+ */
+public class OutOfCoreIOCallableFactory {
+ /** Class logger. */
+ private static final Logger LOG =
+ Logger.getLogger(OutOfCoreIOCallableFactory.class);
+ /** Out-of-core engine */
+ private final OutOfCoreEngine oocEngine;
+ /** Result of IO threads at the end of the computation */
+ private final List<Future> results;
+ /** How many disks (i.e. IO threads) do we have? */
+ private int numDisks;
+ /** Path prefix for different disks */
+ private final String[] basePaths;
+ /** Executor service for IO threads */
+ private ExecutorService outOfCoreIOExecutor;
+ /**
+ * Constructor
+ *
+ * @param conf Configuration
+ * @param oocEngine Out-of-core engine
+ */
+ public OutOfCoreIOCallableFactory(
+ ImmutableClassesGiraphConfiguration<?, ?, ?> conf,
+ OutOfCoreEngine oocEngine) {
+ this.oocEngine = oocEngine;
+ this.results = new ArrayList<>();
+ // Take advantage of multiple disks
+ String[] userPaths = PARTITIONS_DIRECTORY.getArray(conf);
+ this.numDisks = userPaths.length;
+ this.basePaths = new String[numDisks];
+ int ptr = 0;
+ for (String path : userPaths) {
+ File file = new File(path);
+ if (!file.exists()) {
+ checkState(file.mkdirs(), "OutOfCoreIOCallableFactory: cannot create " +
+ "directory " + file.getAbsolutePath());
+ }
+ basePaths[ptr] = path + "/" + conf.get("mapred.job.id", "Unknown Job");
+ ptr++;
+ }
+ }
+
+ /**
+ * Creates/Launches IO threads
+ */
+ public void createCallable() {
+ CallableFactory<Void> outOfCoreIOCallableFactory =
+ new CallableFactory<Void>() {
+ @Override
+ public Callable<Void> newCallable(int callableId) {
+ return new OutOfCoreIOCallable(oocEngine, basePaths[callableId],
+ callableId);
+ }
+ };
+ outOfCoreIOExecutor = new ThreadPoolExecutor(numDisks, numDisks, 0L,
+ TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
+ ThreadUtils.createThreadFactory("ooc-io-%d")) {
+ @Override
+ protected void afterExecute(Runnable r, Throwable t) {
+ super.afterExecute(r, t);
+ if (t == null && r instanceof Future<?>) {
+ try {
+ Future<?> future = (Future<?>) r;
+ if (future.isDone()) {
+ future.get();
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } catch (ExecutionException e) {
+ t = e;
+ }
+ if (t != null) {
+ LOG.info("afterExecute: an out-of-core thread terminated " +
+ "unexpectedly with " + t);
+ oocEngine.failTheJob();
+ }
+ }
+ }
+ };
+
+ for (int i = 0; i < numDisks; ++i) {
+ Future<Void> future = outOfCoreIOExecutor.submit(
+ new LogStacktraceCallable<>(
+ outOfCoreIOCallableFactory.newCallable(i)));
+ results.add(future);
+ }
+ // Notify executor to not accept any more tasks
+ outOfCoreIOExecutor.shutdown();
+ }
+
+ /**
+ * How many disks do we have?
+ *
+ * @return number of disks (IO threads)
+ */
+ public int getNumDisks() {
+ return numDisks;
+ }
+
+ /**
+ * Check whether all IO threads terminated gracefully.
+ */
+ public void shutdown() {
+ boolean threadsTerminated = false;
+ while (!threadsTerminated) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("shutdown: waiting for IO threads to finish!");
+ }
+ try {
+ threadsTerminated =
+ outOfCoreIOExecutor.awaitTermination(1000, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ throw new IllegalStateException("shutdown: caught " +
+ "InterruptedException while waiting for IO threads to finish");
+ }
+ }
+ for (int i = 0; i < numDisks; ++i) {
+ try {
+ // Check whether the tread terminated gracefully
+ results.get(i).get();
+ } catch (InterruptedException e) {
+ LOG.error("shutdown: IO thread " + i + " was interrupted during its " +
+ "execution");
+ throw new IllegalStateException(e);
+ } catch (ExecutionException e) {
+ LOG.error("shutdown: IO thread " + i + " threw an exception during " +
+ "its execution");
+ throw new IllegalStateException(e);
+ }
+ }
+ for (String path : basePaths) {
+ File file = new File(path).getParentFile();
+ for (String subFileName : file.list()) {
+ File subFile = new File(file.getPath(), subFileName);
+ checkState(subFile.delete(), "shutdown: cannot delete file %s",
+ subFile.getAbsoluteFile());
+ }
+ checkState(file.delete(), "shutdown: cannot delete directory %s",
+ file.getAbsoluteFile());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOScheduler.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOScheduler.java b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOScheduler.java
new file mode 100644
index 0000000..dee632d
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOScheduler.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc;
+
+import com.google.common.hash.Hashing;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.conf.IntConfOption;
+import org.apache.giraph.ooc.io.IOCommand;
+import org.apache.log4j.Logger;
+
+/**
+ * Representation of IO thread scheduler for out-of-core mechanism
+ */
+public abstract class OutOfCoreIOScheduler {
+ /**
+ * If an IO thread does not have any command to do, it waits for certain a
+ * period and check back again to see if there exist any command to perform.
+ * This constant determines this wait period in milliseconds.
+ */
+ public static final IntConfOption OOC_WAIT_INTERVAL =
+ new IntConfOption("giraph.oocWaitInterval", 1000,
+ "Duration (in milliseconds) which IO threads in out-of-core " +
+ "mechanism would wait until a command becomes available");
+ /** Class logger. */
+ private static final Logger LOG =
+ Logger.getLogger(OutOfCoreIOScheduler.class);
+ /** Out-of-core engine */
+ protected final OutOfCoreEngine oocEngine;
+ /** How much an IO thread should wait if there is no IO command */
+ protected final int waitInterval;
+ /** How many disks (i.e. IO threads) do we have? */
+ private final int numDisks;
+
+ /**
+ * Constructor
+ *
+ * @param conf configuration
+ * @param oocEngine out-of-core engine
+ * @param numDisks number of disks (IO threads)
+ */
+ OutOfCoreIOScheduler(final ImmutableClassesGiraphConfiguration conf,
+ OutOfCoreEngine oocEngine, int numDisks) {
+ this.oocEngine = oocEngine;
+ this.numDisks = numDisks;
+ this.waitInterval = OOC_WAIT_INTERVAL.get(conf);
+ }
+
+ /**
+ * Get the thread id that is responsible for a particular partition
+ *
+ * @param partitionId id of the given partition
+ * @return id of the thread responsible for the given partition
+ */
+ public int getOwnerThreadId(int partitionId) {
+ int result = Hashing.murmur3_32().hashInt(partitionId).asInt() % numDisks;
+ return (result >= 0) ? result : (result + numDisks);
+ }
+
+ /**
+ * Generate and return the next appropriate IO command for a given thread
+ *
+ * @param threadId id of the thread ready to execute the next IO command
+ * @return next IO command to be executed by the given thread
+ */
+ public abstract IOCommand getNextIOCommand(int threadId);
+
+ /**
+ * Notify IO scheduler that the IO command is completed
+ *
+ * @param command completed command
+ */
+ public abstract void ioCommandCompleted(IOCommand command);
+
+ /**
+ * Add an IO command to the scheduling queue of the IO scheduler
+ *
+ * @param ioCommand IO command to add to the scheduler
+ */
+ public abstract void addIOCommand(IOCommand ioCommand);
+
+ /**
+ * Shutdown/Terminate the IO scheduler, and notify all IO threads to halt
+ */
+ public void shutdown() {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("shutdown: OutOfCoreIOScheduler shutting down!");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreProcessorCallable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreProcessorCallable.java b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreProcessorCallable.java
deleted file mode 100644
index 7c4d8df..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreProcessorCallable.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.ooc;
-
-import org.apache.giraph.bsp.CentralizedServiceWorker;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.log4j.Logger;
-
-import java.io.IOException;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Class to implement slaves for adaptive out-of-core brain. Basically the brain
- * decides on when to offload and what data to offload to disk and generates
- * commands for offloading. Slaves just execute the commands. Commands can be:
- * 1) offloading vertex buffer of partitions in INPUT_SUPERSTEP,
- * 2) offloading edge buffer of partitions in INPUT_SUPERSTEP,
- * 3) offloading partitions.
- *
- * @param <I> Vertex id
- * @param <V> Vertex value
- * @param <E> Edge value
- */
-public class OutOfCoreProcessorCallable<I extends WritableComparable,
- V extends Writable, E extends Writable> implements Callable<Void> {
- /** Class logger */
- private static final Logger LOG =
- Logger.getLogger(OutOfCoreProcessorCallable.class);
- /** Partition store */
- private final DiskBackedPartitionStore<I, V, E> partitionStore;
- /** Adaptive out-of-core engine */
- private final AdaptiveOutOfCoreEngine<I, V, E> oocEngine;
-
- /**
- * Constructor for out-of-core processor threads.
- *
- * @param oocEngine out-of-core engine
- * @param serviceWorker worker service
- */
- public OutOfCoreProcessorCallable(AdaptiveOutOfCoreEngine<I, V, E> oocEngine,
- CentralizedServiceWorker<I, V, E> serviceWorker) {
- this.oocEngine = oocEngine;
- this.partitionStore =
- (DiskBackedPartitionStore<I, V, E>) serviceWorker.getPartitionStore();
- }
-
- @Override
- public Void call() {
- while (true) {
- // First wait on a gate to be opened by memory-check thread. Memory-check
- // thread opens the gate once there are data available to be spilled to
- // disk.
- try {
- oocEngine.waitOnGate();
- } catch (InterruptedException e) {
- throw new IllegalStateException("call: Caught InterruptedException " +
- "while waiting on memory check thread signal on available " +
- "partitions to put on disk");
- } catch (BrokenBarrierException e) {
- throw new IllegalStateException("call Caught BrokenBarrierException. " +
- "Looks like some other threads broke while waiting on barrier");
- }
-
- // The value of 'done' is true iff it is set right before the gate
- // at end of check-memory thread. In such case, the computation is done
- // and OOC processing threads should terminate gracefully.
- if (oocEngine.isDone()) {
- break;
- }
-
- BlockingQueue<Integer> partitionsWithInputVertices =
- oocEngine.getPartitionsWithInputVertices();
- BlockingQueue<Integer> partitionsWithInputEdges =
- oocEngine.getPartitionsWithInputEdges();
- BlockingQueue<Integer> partitionsWithPendingMessages =
- oocEngine.getPartitionsWithPendingMessages();
- AtomicInteger numPartitionsToSpill =
- oocEngine.getNumPartitionsToSpill();
-
- while (!partitionsWithInputVertices.isEmpty()) {
- Integer partitionId = partitionsWithInputVertices.poll();
- if (partitionId == null) {
- break;
- }
- if (LOG.isInfoEnabled()) {
- LOG.info("call: spilling vertex buffer of partition " + partitionId);
- }
- try {
- partitionStore.spillPartitionInputVertexBuffer(partitionId);
- } catch (IOException e) {
- throw new IllegalStateException("call: caught IOException while " +
- "spilling vertex buffers to disk");
- }
- }
-
- while (!partitionsWithInputEdges.isEmpty()) {
- Integer partitionId = partitionsWithInputEdges.poll();
- if (partitionId == null) {
- break;
- }
- if (LOG.isInfoEnabled()) {
- LOG.info("call: spilling edge buffer of partition " + partitionId);
- }
- try {
- partitionStore.spillPartitionInputEdgeStore(partitionId);
- } catch (IOException e) {
- throw new IllegalStateException("call: caught IOException while " +
- "spilling edge buffers/store to disk");
- }
- }
-
- while (!partitionsWithPendingMessages.isEmpty()) {
- Integer partitionId = partitionsWithPendingMessages.poll();
- if (partitionId == null) {
- break;
- }
- if (LOG.isInfoEnabled()) {
- LOG.info(
- "call: spilling message buffers of partition " + partitionId);
- }
- try {
- partitionStore.spillPartitionMessages(partitionId);
- } catch (IOException e) {
- throw new IllegalStateException("call: caught IOException while " +
- "spilling edge buffers/store to disk");
- }
- }
-
- // Put partitions on disk
- while (numPartitionsToSpill.getAndDecrement() > 0) {
- if (LOG.isInfoEnabled()) {
- LOG.info("call: start offloading a partition");
- }
- partitionStore.spillOnePartition();
- }
-
- // Signal memory check thread that I am done putting data on disk
- try {
- oocEngine.waitOnOocSignal();
- } catch (InterruptedException e) {
- throw new IllegalStateException("call: Caught InterruptedException " +
- "while waiting to notify memory check thread that I am done");
- } catch (BrokenBarrierException e) {
- throw new IllegalStateException("call: Caught BrokenBarrierException " +
- "while waiting to notify memory check thread that I am done");
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedEdgeStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedEdgeStore.java b/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedEdgeStore.java
new file mode 100644
index 0000000..7909100
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedEdgeStore.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.data;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.edge.EdgeStore;
+import org.apache.giraph.ooc.OutOfCoreEngine;
+import org.apache.giraph.utils.ByteArrayVertexIdEdges;
+import org.apache.giraph.utils.VertexIdEdges;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static com.google.common.base.Preconditions.checkState;
+
+/**
+ * Implementation of an edge-store used for out-of-core mechanism.
+ *
+ * @param <I> Vertex id
+ * @param <V> Vertex data
+ * @param <E> Edge data
+ */
+public class DiskBackedEdgeStore<I extends WritableComparable,
+ V extends Writable, E extends Writable>
+ extends OutOfCoreDataManager<VertexIdEdges<I, E>>
+ implements EdgeStore<I, V, E> {
+ /** Class logger. */
+ private static final Logger LOG = Logger.getLogger(DiskBackedEdgeStore.class);
+ /** In-memory message store */
+ private final EdgeStore<I, V, E> edgeStore;
+ /** Configuration */
+ private final ImmutableClassesGiraphConfiguration<I, V, E> conf;
+ /** Out-of-core engine */
+ private final OutOfCoreEngine oocEngine;
+
+ /**
+ * Constructor
+ *
+ * @param edgeStore In-memory edge store for which out-of-core edge store
+ * would be a wrapper
+ * @param conf Configuration
+ * @param oocEngine Out-of-core engine
+ */
+ public DiskBackedEdgeStore(
+ EdgeStore<I, V, E> edgeStore,
+ ImmutableClassesGiraphConfiguration<I, V, E> conf,
+ OutOfCoreEngine oocEngine) {
+ super(conf);
+ this.edgeStore = edgeStore;
+ this.conf = conf;
+ this.oocEngine = oocEngine;
+ }
+
+ @Override
+ public void addPartitionEdges(int partitionId, VertexIdEdges<I, E> edges) {
+ addEntry(partitionId, edges);
+ }
+
+ @Override
+ public void moveEdgesToVertices() {
+ edgeStore.moveEdgesToVertices();
+ }
+
+ @Override
+ public void writePartitionEdgeStore(int partitionId, DataOutput output)
+ throws IOException {
+ // This method is only called (should only be called) on in-memory edge
+ // stores
+ throw new IllegalStateException("writePartitionEdgeStore: this method " +
+ "should not be called for DiskBackedEdgeStore!");
+ }
+
+ @Override
+ public void readPartitionEdgeStore(int partitionId, DataInput input)
+ throws IOException {
+ // This method is only called (should only be called) on in-memory edge
+ // stores
+ throw new IllegalStateException("readPartitionEdgeStore: this method " +
+ "should not be called for DiskBackedEdgeStore!");
+ }
+
+ @Override
+ public boolean hasEdgesForPartition(int partitionId) {
+ // This method is only called (should only be called) on in-memory edge
+ // stores
+ throw new IllegalStateException("hasEdgesForPartition: this method " +
+ "should not be called for DiskBackedEdgeStore!");
+ }
+
+ /**
+ * Gets the path that should be used specifically for edge data.
+ *
+ * @param basePath path prefix to build the actual path from
+ * @return path to files specific for edge data
+ */
+ private static String getPath(String basePath) {
+ return basePath + "_edge_store";
+ }
+
+ @Override
+ public void loadPartitionData(int partitionId, String basePath)
+ throws IOException {
+ super.loadPartitionData(partitionId, getPath(basePath));
+ }
+
+ @Override
+ public void offloadPartitionData(int partitionId, String basePath)
+ throws IOException {
+ super.offloadPartitionData(partitionId, getPath(basePath));
+ }
+
+ @Override
+ public void offloadBuffers(int partitionId, String basePath)
+ throws IOException {
+ super.offloadBuffers(partitionId, getPath(basePath));
+ }
+
+ @Override
+ protected void writeEntry(VertexIdEdges<I, E> edges, DataOutput out)
+ throws IOException {
+ edges.write(out);
+ }
+
+ @Override
+ protected VertexIdEdges<I, E> readNextEntry(DataInput in) throws IOException {
+ VertexIdEdges<I, E> vertexIdEdges = new ByteArrayVertexIdEdges<>();
+ vertexIdEdges.setConf(conf);
+ vertexIdEdges.readFields(in);
+ return vertexIdEdges;
+ }
+
+ @Override
+ protected void loadInMemoryPartitionData(int partitionId, String path)
+ throws IOException {
+ File file = new File(path);
+ if (file.exists()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("loadInMemoryPartitionData: loading edge data for " +
+ "partition " + partitionId + " from " + file.getAbsolutePath());
+ }
+ FileInputStream fis = new FileInputStream(file);
+ BufferedInputStream bis = new BufferedInputStream(fis);
+ DataInputStream dis = new DataInputStream(bis);
+ edgeStore.readPartitionEdgeStore(partitionId, dis);
+ dis.close();
+ checkState(file.delete(), "loadInMemoryPartitionData: failed to delete " +
+ "%s.", file.getAbsoluteFile());
+ }
+ }
+
+ @Override
+ protected void offloadInMemoryPartitionData(int partitionId, String path)
+ throws IOException {
+ if (edgeStore.hasEdgesForPartition(partitionId)) {
+ File file = new File(path);
+ checkState(!file.exists(), "offloadInMemoryPartitionData: edge store " +
+ "file %s already exist", file.getAbsoluteFile());
+ checkState(file.createNewFile(),
+ "offloadInMemoryPartitionData: cannot create edge store file %s",
+ file.getAbsoluteFile());
+ FileOutputStream fos = new FileOutputStream(file);
+ BufferedOutputStream bos = new BufferedOutputStream(fos);
+ DataOutputStream dos = new DataOutputStream(bos);
+ edgeStore.writePartitionEdgeStore(partitionId, dos);
+ dos.close();
+ }
+ }
+
+ @Override
+ protected int entrySerializedSize(VertexIdEdges<I, E> edges) {
+ return edges.getSerializedSize();
+ }
+
+ @Override
+ protected void addEntryToImMemoryPartitionData(int partitionId,
+ VertexIdEdges<I, E> edges) {
+ oocEngine.getMetaPartitionManager().addPartition(partitionId);
+ edgeStore.addPartitionEdges(partitionId, edges);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedMessageStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedMessageStore.java b/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedMessageStore.java
new file mode 100644
index 0000000..2a40a58
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedMessageStore.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.data;
+
+import org.apache.giraph.comm.messages.MessageStore;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.utils.ByteArrayOneMessageToManyIds;
+import org.apache.giraph.utils.ByteArrayVertexIdMessages;
+import org.apache.giraph.utils.VertexIdMessages;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static com.google.common.base.Preconditions.checkState;
+
+/**
+ * Implementation of a message store used for out-of-core mechanism.
+ *
+ * @param <I> Vertex id
+ * @param <M> Message data
+ */
+public class DiskBackedMessageStore<I extends WritableComparable,
+ M extends Writable> extends OutOfCoreDataManager<VertexIdMessages<I, M>>
+ implements MessageStore<I, M> {
+ /** Class logger. */
+ private static final Logger LOG =
+ Logger.getLogger(DiskBackedMessageStore.class);
+ /** Configuration */
+ private final ImmutableClassesGiraphConfiguration<I, ?, ?> config;
+ /** In-memory message store */
+ private final MessageStore<I, M> messageStore;
+ /** Whether the message store uses message combiner or not */
+ private final boolean useMessageCombiner;
+ /** Which superstep this message store is used for */
+ private final long superstep;
+
+ /**
+ * Type of VertexIdMessage class (container for serialized messages) received
+ * for a particular message. If we write the received messages to disk before
+ * adding them to message store, we need this type when we want to read the
+ * messages back from disk (so that we know how to read the messages from
+ * disk).
+ */
+ private enum SerializedMessageClass {
+ /** ByteArrayVertexIdMessages */
+ BYTE_ARRAY_VERTEX_ID_MESSAGES,
+ /** ByteArrayOneMEssageToManyIds */
+ BYTE_ARRAY_ONE_MESSAGE_TO_MANY_IDS
+ }
+
+ /**
+ * Constructor
+ *
+ * @param config Configuration
+ * @param messageStore In-memory message store for which out-of-core message
+ * store would be wrapper
+ * @param useMessageCombiner Whether message combiner is used for this message
+ * store
+ * @param superstep superstep number this messages store is used for
+ */
+ public DiskBackedMessageStore(ImmutableClassesGiraphConfiguration<I, ?, ?>
+ config,
+ MessageStore<I, M> messageStore,
+ boolean useMessageCombiner, long superstep) {
+ super(config);
+ this.config = config;
+ this.messageStore = messageStore;
+ this.useMessageCombiner = useMessageCombiner;
+ this.superstep = superstep;
+ }
+
+ @Override
+ public boolean isPointerListEncoding() {
+ return messageStore.isPointerListEncoding();
+ }
+
+ @Override
+ public Iterable<M> getVertexMessages(I vertexId) throws IOException {
+ return messageStore.getVertexMessages(vertexId);
+ }
+
+ @Override
+ public void clearVertexMessages(I vertexId) throws IOException {
+ messageStore.clearVertexMessages(vertexId);
+ }
+
+ @Override
+ public void clearAll() throws IOException {
+ messageStore.clearAll();
+ }
+
+ @Override
+ public boolean hasMessagesForVertex(I vertexId) {
+ return messageStore.hasMessagesForVertex(vertexId);
+ }
+
+ @Override
+ public boolean hasMessagesForPartition(int partitionId) {
+ return messageStore.hasMessagesForPartition(partitionId);
+ }
+
+ @Override
+ public void addPartitionMessages(
+ int partitionId, VertexIdMessages<I, M> messages) throws IOException {
+ if (useMessageCombiner) {
+ messageStore.addPartitionMessages(partitionId, messages);
+ } else {
+ addEntry(partitionId, messages);
+ }
+ }
+
+ /**
+ * Gets the path that should be used specifically for message data.
+ *
+ * @param basePath path prefix to build the actual path from
+ * @param superstep superstep for which message data should be stored
+ * @return path to files specific for message data
+ */
+ private static String getPath(String basePath, long superstep) {
+ return basePath + "_messages-S" + superstep;
+ }
+
+ @Override
+ public void loadPartitionData(int partitionId, String basePath)
+ throws IOException {
+ if (!useMessageCombiner) {
+ super.loadPartitionData(partitionId, getPath(basePath, superstep));
+ }
+ }
+
+ @Override
+ public void offloadPartitionData(int partitionId, String basePath)
+ throws IOException {
+ if (!useMessageCombiner) {
+ super.offloadPartitionData(partitionId, getPath(basePath, superstep));
+ }
+ }
+
+ @Override
+ public void offloadBuffers(int partitionId, String basePath)
+ throws IOException {
+ if (!useMessageCombiner) {
+ super.offloadBuffers(partitionId, getPath(basePath, superstep));
+ }
+ }
+
+ @Override
+ public void finalizeStore() {
+ messageStore.finalizeStore();
+ }
+
+ @Override
+ public Iterable<I> getPartitionDestinationVertices(int partitionId) {
+ return messageStore.getPartitionDestinationVertices(partitionId);
+ }
+
+ @Override
+ public void clearPartition(int partitionId) throws IOException {
+ messageStore.clearPartition(partitionId);
+ }
+
+ @Override
+ public void writePartition(DataOutput out, int partitionId)
+ throws IOException {
+ messageStore.writePartition(out, partitionId);
+ }
+
+ @Override
+ public void readFieldsForPartition(DataInput in, int partitionId)
+ throws IOException {
+ messageStore.readFieldsForPartition(in, partitionId);
+ }
+
+ @Override
+ protected void writeEntry(VertexIdMessages<I, M> messages, DataOutput out)
+ throws IOException {
+ SerializedMessageClass messageClass;
+ if (messages instanceof ByteArrayVertexIdMessages) {
+ messageClass = SerializedMessageClass.BYTE_ARRAY_VERTEX_ID_MESSAGES;
+ } else if (messages instanceof ByteArrayOneMessageToManyIds) {
+ messageClass = SerializedMessageClass.BYTE_ARRAY_ONE_MESSAGE_TO_MANY_IDS;
+ } else {
+ throw new IllegalStateException("writeEntry: serialized message " +
+ "type is not supported");
+ }
+ out.writeInt(messageClass.ordinal());
+ messages.write(out);
+ }
+
+ @Override
+ protected VertexIdMessages<I, M> readNextEntry(DataInput in)
+ throws IOException {
+ int messageType = in.readInt();
+ SerializedMessageClass messageClass =
+ SerializedMessageClass.values()[messageType];
+ VertexIdMessages<I, M> vim;
+ switch (messageClass) {
+ case BYTE_ARRAY_VERTEX_ID_MESSAGES:
+ vim = new ByteArrayVertexIdMessages<>(
+ config.<M>createOutgoingMessageValueFactory());
+ vim.setConf(config);
+ break;
+ case BYTE_ARRAY_ONE_MESSAGE_TO_MANY_IDS:
+ vim = new ByteArrayOneMessageToManyIds<>(
+ config.<M>createOutgoingMessageValueFactory());
+ vim.setConf(config);
+ break;
+ default:
+ throw new IllegalStateException("readNextEntry: unsupported " +
+ "serialized message type!");
+ }
+ vim.readFields(in);
+ return vim;
+ }
+
+ @Override
+ protected void loadInMemoryPartitionData(int partitionId, String basePath)
+ throws IOException {
+ File file = new File(basePath);
+ if (file.exists()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("loadInMemoryPartitionData: loading message data for " +
+ "partition " + partitionId + " from " + file.getAbsolutePath());
+ }
+ FileInputStream fis = new FileInputStream(file);
+ BufferedInputStream bis = new BufferedInputStream(fis);
+ DataInputStream dis = new DataInputStream(bis);
+ messageStore.readFieldsForPartition(dis, partitionId);
+ dis.close();
+ checkState(file.delete(), "loadInMemoryPartitionData: failed to delete " +
+ "%s.", file.getAbsoluteFile());
+ }
+ }
+
+ @Override
+ protected void offloadInMemoryPartitionData(int partitionId, String basePath)
+ throws IOException {
+ if (messageStore.hasMessagesForPartition(partitionId)) {
+ File file = new File(basePath);
+ checkState(!file.exists(), "offloadInMemoryPartitionData: message store" +
+ " file %s already exist", file.getAbsoluteFile());
+ checkState(file.createNewFile(),
+ "offloadInMemoryPartitionData: cannot create message store file %s",
+ file.getAbsoluteFile());
+ FileOutputStream fileout = new FileOutputStream(file);
+ BufferedOutputStream bufferout = new BufferedOutputStream(fileout);
+ DataOutputStream outputStream = new DataOutputStream(bufferout);
+ messageStore.writePartition(outputStream, partitionId);
+ messageStore.clearPartition(partitionId);
+ outputStream.close();
+ }
+ }
+
+ @Override
+ protected int entrySerializedSize(VertexIdMessages<I, M> messages) {
+ return messages.getSerializedSize();
+ }
+
+ @Override
+ protected void addEntryToImMemoryPartitionData(int partitionId,
+ VertexIdMessages<I, M>
+ messages) {
+ try {
+ messageStore.addPartitionMessages(partitionId, messages);
+ } catch (IOException e) {
+ throw new IllegalStateException("Caught IOException while adding a new " +
+ "message to in-memory message store");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedPartitionStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedPartitionStore.java b/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedPartitionStore.java
new file mode 100644
index 0000000..5854c8d
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/data/DiskBackedPartitionStore.java
@@ -0,0 +1,469 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc.data;
+
+import com.google.common.collect.Maps;
+import org.apache.giraph.bsp.BspService;
+import org.apache.giraph.bsp.CentralizedServiceWorker;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.edge.OutEdges;
+import org.apache.giraph.graph.Vertex;
+import org.apache.giraph.ooc.OutOfCoreEngine;
+import org.apache.giraph.partition.Partition;
+import org.apache.giraph.partition.PartitionStore;
+import org.apache.giraph.utils.ExtendedDataOutput;
+import org.apache.giraph.utils.WritableUtils;
+import org.apache.giraph.worker.BspServiceWorker;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.log4j.Logger;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Map;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+/**
+ * Implementation of a partition-store used for out-of-core mechanism.
+ * Partition store is responsible for partition data, as well as data buffers in
+ * INPUT_SUPERSTEP ("raw data buffer" -- defined in OutOfCoreDataManager --
+ * refers to vertex buffers in INPUT_SUPERSTEP).
+ *
+ * @param <I> Vertex id
+ * @param <V> Vertex data
+ * @param <E> Edge data
+ */
+public class DiskBackedPartitionStore<I extends WritableComparable,
+ V extends Writable, E extends Writable>
+ extends OutOfCoreDataManager<ExtendedDataOutput>
+ implements PartitionStore<I, V, E> {
+ /** Class logger. */
+ private static final Logger LOG =
+ Logger.getLogger(DiskBackedPartitionStore.class);
+ /** Configuration */
+ private final ImmutableClassesGiraphConfiguration<I, V, E> conf;
+ /** Job context (for progress) */
+ private final Mapper<?, ?, ?, ?>.Context context;
+ /** Service worker */
+ private final CentralizedServiceWorker<I, V, E> serviceWorker;
+ /** Out-of-core engine */
+ private final OutOfCoreEngine oocEngine;
+ /** In-memory partition store */
+ private final PartitionStore<I, V, E> partitionStore;
+ /**
+ * Keeps number of vertices in partitions, right when they are last spilled
+ * to the disk. This value may be inaccurate for in-memory partitions, but
+ * is accurate for out-of-core partitions.
+ */
+ private final Map<Integer, Long> partitionVertexCount =
+ Maps.newConcurrentMap();
+ /**
+ * Keeps number of edges in partitions, right when they are last spilled
+ * to the disk. This value may be inaccurate for in-memory partitions, but
+ * is accurate for out-of-core partitions.
+ */
+ private final Map<Integer, Long> partitionEdgeCount =
+ Maps.newConcurrentMap();
+
+ /**
+ * Constructor.
+ *
+ * @param partitionStore In-memory partition store for which out-of-code
+ * partition store would be a wrapper
+ * @param conf Configuration
+ * @param context Job context
+ * @param serviceWorker Service worker
+ * @param oocEngine Out-of-core engine
+ */
+ public DiskBackedPartitionStore(
+ PartitionStore<I, V, E> partitionStore,
+ ImmutableClassesGiraphConfiguration<I, V, E> conf,
+ Mapper<?, ?, ?, ?>.Context context,
+ CentralizedServiceWorker<I, V, E> serviceWorker,
+ OutOfCoreEngine oocEngine) {
+ super(conf);
+ this.partitionStore = partitionStore;
+ this.conf = conf;
+ this.context = context;
+ this.serviceWorker = serviceWorker;
+ this.oocEngine = oocEngine;
+ }
+
+ @Override
+ public boolean addPartition(Partition<I, V, E> partition) {
+ boolean added = partitionStore.addPartition(partition);
+ if (added) {
+ oocEngine.getMetaPartitionManager()
+ .addPartition(partition.getId());
+ }
+ return added;
+ }
+
+ @Override
+ public Partition<I, V, E> removePartition(Integer partitionId) {
+ // Set the partition as 'in process' so its data and messages do not get
+ // spilled to disk until the remove is complete.
+ oocEngine.getMetaPartitionManager().makePartitionInaccessible(partitionId);
+ oocEngine.retrievePartition(partitionId);
+ Partition<I, V, E> partition = partitionStore.removePartition(partitionId);
+ checkNotNull(partition, "removePartition: partition " + partitionId +
+ " is not in memory for removal!");
+ oocEngine.getMetaPartitionManager().removePartition(partitionId);
+ return partition;
+ }
+
+ @Override
+ public boolean hasPartition(Integer partitionId) {
+ return oocEngine.getMetaPartitionManager().hasPartition(partitionId);
+ }
+
+ @Override
+ public Iterable<Integer> getPartitionIds() {
+ return oocEngine.getMetaPartitionManager().getPartitionIds();
+ }
+
+ @Override
+ public int getNumPartitions() {
+ return oocEngine.getMetaPartitionManager().getNumPartitions();
+ }
+
+ @Override
+ public long getPartitionVertexCount(Integer partitionId) {
+ if (partitionStore.hasPartition(partitionId)) {
+ return partitionStore.getPartitionVertexCount(partitionId);
+ } else {
+ return partitionVertexCount.get(partitionId);
+ }
+ }
+
+ @Override
+ public long getPartitionEdgeCount(Integer partitionId) {
+ if (partitionStore.hasPartition(partitionId)) {
+ return partitionStore.getPartitionEdgeCount(partitionId);
+ } else {
+ return partitionEdgeCount.get(partitionId);
+ }
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return getNumPartitions() == 0;
+ }
+
+ @Override
+ public void startIteration() {
+ oocEngine.startIteration();
+ }
+
+ @Override
+ public Partition<I, V, E> getNextPartition() {
+ Integer partitionId = oocEngine.getNextPartition();
+ if (partitionId == null) {
+ return null;
+ }
+ Partition<I, V, E> partition = partitionStore.removePartition(partitionId);
+ if (partition == null) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("getNextPartition: partition " + partitionId + " is not in " +
+ "the partition store. Creating an empty partition for it.");
+ }
+ partition = conf.createPartition(partitionId, context);
+ }
+ partitionStore.addPartition(partition);
+ return partition;
+ }
+
+ @Override
+ public void putPartition(Partition<I, V, E> partition) {
+ oocEngine.doneProcessingPartition(partition.getId());
+ }
+
+ @Override
+ public void addPartitionVertices(Integer partitionId,
+ ExtendedDataOutput extendedDataOutput) {
+ addEntry(partitionId, extendedDataOutput);
+ }
+
+ @Override
+ public void shutdown() {
+ oocEngine.shutdown();
+ }
+
+ @Override
+ public void initialize() {
+ oocEngine.initialize();
+ }
+
+ /**
+ * Gets the path that should be used specifically for partition data.
+ *
+ * @param basePath path prefix to build the actual path from
+ * @return path to files specific for partition data
+ */
+ private static String getPath(String basePath) {
+ return basePath + "_partition";
+ }
+
+ /**
+ * Get the path to the file where vertices are stored.
+ *
+ * @param basePath path prefix to build the actual path from
+ * @return The path to the vertices file
+ */
+ private static String getVerticesPath(String basePath) {
+ return basePath + "_vertices";
+ }
+
+ /**
+ * Get the path to the file where edges are stored.
+ *
+ * @param basePath path prefix to build the actual path from
+ * @return The path to the edges file
+ */
+ private static String getEdgesPath(String basePath) {
+ return basePath + "_edges";
+ }
+
+ /**
+ * Read vertex data from an input and initialize the vertex.
+ *
+ * @param in The input stream
+ * @param vertex The vertex to initialize
+ * @throws IOException
+ */
+ private void readVertexData(DataInput in, Vertex<I, V, E> vertex)
+ throws IOException {
+ I id = conf.createVertexId();
+ id.readFields(in);
+ V value = conf.createVertexValue();
+ value.readFields(in);
+ OutEdges<I, E> edges = conf.createAndInitializeOutEdges(0);
+ vertex.initialize(id, value, edges);
+ if (in.readBoolean()) {
+ vertex.voteToHalt();
+ } else {
+ vertex.wakeUp();
+ }
+ }
+
+ /**
+ * Read vertex edges from an input and set them to the vertex.
+ *
+ * @param in The input stream
+ * @param partition The partition owning the vertex
+ * @throws IOException
+ */
+ private void readOutEdges(DataInput in, Partition<I, V, E> partition)
+ throws IOException {
+ I id = conf.createVertexId();
+ id.readFields(in);
+ Vertex<I, V, E> v = partition.getVertex(id);
+ OutEdges<I, E> edges = (OutEdges<I, E>) v.getEdges();
+ edges.readFields(in);
+ partition.saveVertex(v);
+ }
+
+ @Override
+ protected void loadInMemoryPartitionData(int partitionId, String path)
+ throws IOException {
+ // Load vertices
+ File file = new File(getVerticesPath(path));
+ if (file.exists()) {
+ Partition<I, V, E> partition = conf.createPartition(partitionId, context);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("loadInMemoryPartitionData: loading partition vertices " +
+ partitionId + " from " + file.getAbsolutePath());
+ }
+
+ FileInputStream fis = new FileInputStream(file);
+ BufferedInputStream bis = new BufferedInputStream(fis);
+ DataInputStream inputStream = new DataInputStream(bis);
+ long numVertices = inputStream.readLong();
+ for (long i = 0; i < numVertices; ++i) {
+ Vertex<I, V, E> vertex = conf.createVertex();
+ readVertexData(inputStream, vertex);
+ partition.putVertex(vertex);
+ }
+ inputStream.close();
+ checkState(file.delete(), "loadInMemoryPartitionData: failed to delete " +
+ "%s", file.getAbsolutePath());
+
+ // Load edges
+ file = new File(getEdgesPath(path));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("loadInMemoryPartitionData: loading partition edges " +
+ partitionId + " from " + file.getAbsolutePath());
+ }
+
+ fis = new FileInputStream(file);
+ bis = new BufferedInputStream(fis);
+ inputStream = new DataInputStream(bis);
+ for (int i = 0; i < numVertices; ++i) {
+ readOutEdges(inputStream, partition);
+ }
+ inputStream.close();
+ // If the graph is static and it is not INPUT_SUPERSTEP, keep the file
+ // around.
+ if (!conf.isStaticGraph() ||
+ serviceWorker.getSuperstep() == BspService.INPUT_SUPERSTEP) {
+ checkState(file.delete(), "loadPartition: failed to delete %s",
+ file.getAbsolutePath());
+ }
+ partitionStore.addPartition(partition);
+ }
+ }
+
+ @Override
+ protected ExtendedDataOutput readNextEntry(DataInput in) throws IOException {
+ return WritableUtils.readExtendedDataOutput(in, conf);
+ }
+
+ @Override
+ protected void addEntryToImMemoryPartitionData(int partitionId,
+ ExtendedDataOutput vertices) {
+ if (!partitionStore.hasPartition(partitionId)) {
+ oocEngine.getMetaPartitionManager().addPartition(partitionId);
+ }
+ partitionStore.addPartitionVertices(partitionId, vertices);
+ }
+
+ @Override
+ public void loadPartitionData(int partitionId, String basePath)
+ throws IOException {
+ super.loadPartitionData(partitionId, getPath(basePath));
+ }
+
+ @Override
+ public void offloadPartitionData(int partitionId, String basePath)
+ throws IOException {
+ super.offloadPartitionData(partitionId, getPath(basePath));
+ }
+
+ /**
+ * Writes vertex data (Id, value and halted state) to stream.
+ *
+ * @param output The output stream
+ * @param vertex The vertex to serialize
+ * @throws IOException
+ */
+ private void writeVertexData(DataOutput output, Vertex<I, V, E> vertex)
+ throws IOException {
+ vertex.getId().write(output);
+ vertex.getValue().write(output);
+ output.writeBoolean(vertex.isHalted());
+ }
+
+ /**
+ * Writes vertex edges (Id, edges) to stream.
+ *
+ * @param output The output stream
+ * @param vertex The vertex to serialize
+ * @throws IOException
+ */
+ private void writeOutEdges(DataOutput output, Vertex<I, V, E> vertex)
+ throws IOException {
+ vertex.getId().write(output);
+ OutEdges<I, E> edges = (OutEdges<I, E>) vertex.getEdges();
+ edges.write(output);
+ }
+
+ @Override
+ protected void offloadInMemoryPartitionData(int partitionId, String path)
+ throws IOException {
+ if (partitionStore.hasPartition(partitionId)) {
+ partitionVertexCount.put(partitionId,
+ partitionStore.getPartitionVertexCount(partitionId));
+ partitionEdgeCount.put(partitionId,
+ partitionStore.getPartitionEdgeCount(partitionId));
+ Partition<I, V, E> partition =
+ partitionStore.removePartition(partitionId);
+ File file = new File(getVerticesPath(path));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("offloadInMemoryPartitionData: writing partition vertices " +
+ partitionId + " to " + file.getAbsolutePath());
+ }
+ checkState(!file.exists(), "offloadInMemoryPartitionData: partition " +
+ "store file %s already exist", file.getAbsoluteFile());
+ checkState(file.createNewFile(),
+ "offloadInMemoryPartitionData: file %s already exists.",
+ file.getAbsolutePath());
+
+ FileOutputStream fileout = new FileOutputStream(file);
+ BufferedOutputStream bufferout = new BufferedOutputStream(fileout);
+ DataOutputStream outputStream = new DataOutputStream(bufferout);
+ outputStream.writeLong(partition.getVertexCount());
+ for (Vertex<I, V, E> vertex : partition) {
+ writeVertexData(outputStream, vertex);
+ }
+ outputStream.close();
+
+ // Avoid writing back edges if we have already written them once and
+ // the graph is not changing.
+ // If we are in the input superstep, we need to write the files
+ // at least the first time, even though the graph is static.
+ file = new File(getEdgesPath(path));
+ if (serviceWorker.getSuperstep() == BspServiceWorker.INPUT_SUPERSTEP ||
+ partitionVertexCount.get(partitionId) == null ||
+ partitionVertexCount.get(partitionId) != partition.getVertexCount() ||
+ !conf.isStaticGraph() || !file.exists()) {
+ checkState(file.createNewFile(), "offloadInMemoryPartitionData: file " +
+ "%s already exists.", file.getAbsolutePath());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("offloadInMemoryPartitionData: writing partition edges " +
+ partitionId + " to " + file.getAbsolutePath());
+ }
+ fileout = new FileOutputStream(file);
+ bufferout = new BufferedOutputStream(fileout);
+ outputStream = new DataOutputStream(bufferout);
+ for (Vertex<I, V, E> vertex : partition) {
+ writeOutEdges(outputStream, vertex);
+ }
+ outputStream.close();
+ }
+ }
+ }
+
+ @Override
+ protected void writeEntry(ExtendedDataOutput vertices, DataOutput out)
+ throws IOException {
+ WritableUtils.writeExtendedDataOutput(vertices, out);
+ }
+
+ @Override
+ public void offloadBuffers(int partitionId, String basePath)
+ throws IOException {
+ super.offloadBuffers(partitionId, getPath(basePath));
+ }
+
+ @Override
+ protected int entrySerializedSize(ExtendedDataOutput vertices) {
+ return vertices.getPos();
+ }
+}
[4/5] git commit: updated refs/heads/trunk to fafecee
Posted by ed...@apache.org.
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/DiskBackedPartitionStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/DiskBackedPartitionStore.java b/giraph-core/src/main/java/org/apache/giraph/ooc/DiskBackedPartitionStore.java
deleted file mode 100644
index cdafa3f..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/DiskBackedPartitionStore.java
+++ /dev/null
@@ -1,2149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.ooc;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-import org.apache.commons.lang3.tuple.MutablePair;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.giraph.bsp.CentralizedServiceWorker;
-import org.apache.giraph.comm.messages.MessageStore;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.conf.IntConfOption;
-import org.apache.giraph.edge.OutEdges;
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.partition.Partition;
-import org.apache.giraph.partition.PartitionStore;
-import org.apache.giraph.utils.ByteArrayOneMessageToManyIds;
-import org.apache.giraph.utils.ByteArrayVertexIdEdges;
-import org.apache.giraph.utils.ByteArrayVertexIdMessages;
-import org.apache.giraph.utils.ExtendedDataOutput;
-import org.apache.giraph.utils.PairList;
-import org.apache.giraph.utils.VertexIdEdges;
-import org.apache.giraph.utils.VertexIdMessages;
-import org.apache.giraph.utils.VertexIterator;
-import org.apache.giraph.utils.WritableUtils;
-import org.apache.giraph.worker.BspServiceWorker;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.log4j.Logger;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static org.apache.giraph.conf.GiraphConstants.MAX_PARTITIONS_IN_MEMORY;
-import static org.apache.giraph.conf.GiraphConstants.ONE_MB;
-import static org.apache.giraph.conf.GiraphConstants.PARTITIONS_DIRECTORY;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
-
-/**
- * Disk-backed PartitionStore. An instance of this class can be coupled with an
- * out-of-core engine. Out-of-core engine is responsible to determine when to
- * offload and what to offload to disk. The instance of this class handles the
- * interactions with disk.
- *
- * This class provides efficient scheduling mechanism while iterating over
- * partitions. It prefers spilling in-memory processed partitions, but the
- * scheduling can be improved upon further.
- *
- * @param <I> Vertex id
- * @param <V> Vertex data
- * @param <E> Edge data
- */
-@SuppressWarnings("rawtypes")
-public class DiskBackedPartitionStore<I extends WritableComparable,
- V extends Writable, E extends Writable>
- extends PartitionStore<I, V, E> {
- /**
- * Minimum size of a buffer (in bytes) to flush to disk. This is used to
- * decide whether vertex/edge buffers are large enough to flush to disk.
- */
- public static final IntConfOption MINIMUM_BUFFER_SIZE_TO_FLUSH =
- new IntConfOption("giraph.flushBufferSize", 8 * ONE_MB,
- "Minimum size of a buffer (in bytes) to flush to disk. ");
-
- /** Class logger. */
- private static final Logger LOG =
- Logger.getLogger(DiskBackedPartitionStore.class);
-
- /** Cached value for MINIMUM_BUFFER_SIZE_TO_FLUSH */
- private final int minBuffSize;
- /**
- * States the partition can be found in:
- * INIT: the partition has just been created
- * ACTIVE: there is at least one thread who holds a reference to the partition
- * and uses it
- * INACTIVE: the partition is not being used by anyone, but it is in memory
- * IN_TRANSIT: the partition is being transferred to disk, the transfer is
- * not yet complete
- * ON_DISK: the partition resides on disk
- */
- private enum State { INIT, ACTIVE, INACTIVE, IN_TRANSIT, ON_DISK };
-
- /** Hash map containing all the partitions */
- private final ConcurrentMap<Integer, MetaPartition> partitions =
- Maps.newConcurrentMap();
-
- /**
- * Contains partitions that has been processed in the current iteration cycle,
- * and are not in use by any thread. The 'State' of these partitions can only
- * be INACTIVE, IN_TRANSIT, and ON_DISK.
- */
- private final Map<State, Set<Integer>> processedPartitions;
- /**
- * Contains partitions that has *not* been processed in the current iteration
- * cycle. Similar to processedPartitions, 'State' if these partitions can only
- * be INACTIVE, IN_TRANSIT, and ON_DISK.
- */
- private final Map<State, Set<Integer>> unProcessedPartitions;
-
- /**
- * Read/Write lock to avoid interleaving of the process of starting a new
- * iteration cycle and the process of spilling data to disk. This is necessary
- * as starting a new iteration changes the data structure holding data that is
- * being spilled to disk. Spilling of different data can happen at the same
- * time (a read lock used for spilling), and cannot be overlapped with
- * change of data structure holding the data.
- */
- private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
-
- /** Base path where the partition files are written to */
- private final String[] basePaths;
- /** Used to hash partition Ids */
- private final HashFunction hasher = Hashing.murmur3_32();
- /** Maximum number of partition slots in memory */
- private final AtomicInteger maxPartitionsInMem = new AtomicInteger(-1);
- /** Number of slots used */
- private final AtomicInteger numPartitionsInMem = new AtomicInteger(0);
-
- /** Out-of-core engine */
- private final OutOfCoreEngine oocEngine;
- /** If moving of edges to vertices in INPUT_SUPERSTEP has been started */
- private volatile boolean movingEdges;
- /** Whether the partition store is initialized */
- private volatile AtomicBoolean isInitialized;
- /** Whether the number of partitions are fixed as requested by user */
- private final boolean isNumPartitionsFixed;
-
- /**
- * Map of partition ids to list of input vertex buffers. The map will have an
- * entry only for partitions that are currently out-of-core. We keep the
- * aggregate size of buffers in as part of the values of the map to estimate
- * how much memory would be free if we offload this buffer to disk.
- */
- private final ConcurrentMap<Integer, Pair<Integer, List<ExtendedDataOutput>>>
- pendingInputVertices = Maps.newConcurrentMap();
- /**
- * When a partition is out-of-core, and we also offloaded some of its vertex
- * buffers, we have to keep track of how many buffers we offloaded to disk.
- * This contains this value for out-of-core partitions.
- */
- private final ConcurrentMap<Integer, Integer> numPendingInputVerticesOnDisk =
- Maps.newConcurrentMap();
- /** Lock to avoid overlap of addition and removal on pendingInputVertices */
- private final ReadWriteLock vertexBufferRWLock = new ReentrantReadWriteLock();
-
- /**
- * Similar to vertex buffer, but used for input edges (see comments for
- * pendingInputVertices).
- */
- private final ConcurrentMap<Integer, Pair<Integer, List<VertexIdEdges<I, E>>>>
- pendingInputEdges = Maps.newConcurrentMap();
- /** Similar to numPendingInputVerticesOnDisk but used for edge buffers */
- private final ConcurrentMap<Integer, Integer> numPendingInputEdgesOnDisk =
- Maps.newConcurrentMap();
- /** Lock to avoid overlap of addition and removal on pendingInputEdges */
- private final ReadWriteLock edgeBufferRWLock = new ReentrantReadWriteLock();
-
- /**
- * For each out-of-core partitions, whether its edge store is also
- * offloaded to disk in INPUT_SUPERSTEP.
- */
- private final ConcurrentMap<Integer, Boolean> hasEdgeStoreOnDisk =
- Maps.newConcurrentMap();
-
- /**
- * Type of VertexIdMessage class (container for serialized messages) received
- * for a particular message. If we write the received messages to disk before
- * adding them to message store, we need this type when we want to read the
- * messages back from disk (so that we know how to read the messages from
- * disk).
- */
- private enum SerializedMessageClass {
- /** ByteArrayVertexIdMessages */
- BYTE_ARRAY_VERTEX_ID_MESSAGES,
- /** ByteArrayOneMEssageToManyIds */
- BYTE_ARRAY_ONE_MESSAGE_TO_MANY_IDS
- }
-
- /**
- * Similar to vertex buffer and edge buffer, but used for messages (see
- * comments for pendingInputVertices).
- */
- private volatile ConcurrentMap<Integer,
- Pair<Integer, List<VertexIdMessages<I, Writable>>>>
- pendingIncomingMessages = Maps.newConcurrentMap();
- /** Whether a partition has any incoming message buffer on disk */
- private volatile ConcurrentMap<Integer, Boolean> incomingMessagesOnDisk =
- Maps.newConcurrentMap();
-
- /**
- * Similar to pendingIncomingMessages, but is used for messages for current
- * superstep instead.
- */
- private volatile ConcurrentMap<Integer,
- Pair<Integer, List<VertexIdMessages<I, Writable>>>>
- pendingCurrentMessages = Maps.newConcurrentMap();
- /** Similar to incomingMessagesOnDisk for messages for current superstep */
- private volatile ConcurrentMap<Integer, Boolean> currentMessagesOnDisk =
- Maps.newConcurrentMap();
-
- /**
- * Lock to avoid overlap of addition and removal of pending message buffers
- */
- private final ReadWriteLock messageBufferRWLock =
- new ReentrantReadWriteLock();
-
- /**
- * Constructor
- *
- * @param conf Configuration
- * @param context Context
- * @param serviceWorker service worker reference
- */
- public DiskBackedPartitionStore(
- ImmutableClassesGiraphConfiguration<I, V, E> conf,
- Mapper<?, ?, ?, ?>.Context context,
- CentralizedServiceWorker<I, V, E> serviceWorker) {
- super(conf, context, serviceWorker);
- this.minBuffSize = MINIMUM_BUFFER_SIZE_TO_FLUSH.get(conf);
- int userMaxNumPartitions = MAX_PARTITIONS_IN_MEMORY.get(conf);
- if (userMaxNumPartitions > 0) {
- this.isNumPartitionsFixed = true;
- this.maxPartitionsInMem.set(userMaxNumPartitions);
- oocEngine = null;
- } else {
- this.isNumPartitionsFixed = false;
- this.oocEngine =
- new AdaptiveOutOfCoreEngine<I, V, E>(conf, serviceWorker);
- }
- this.movingEdges = false;
- this.isInitialized = new AtomicBoolean(false);
-
- this.processedPartitions = Maps.newHashMap();
- this.processedPartitions
- .put(State.INACTIVE, Sets.<Integer>newLinkedHashSet());
- this.processedPartitions
- .put(State.IN_TRANSIT, Sets.<Integer>newLinkedHashSet());
- this.processedPartitions
- .put(State.ON_DISK, Sets.<Integer>newLinkedHashSet());
-
- this.unProcessedPartitions = Maps.newHashMap();
- this.unProcessedPartitions
- .put(State.INACTIVE, Sets.<Integer>newLinkedHashSet());
- this.unProcessedPartitions
- .put(State.IN_TRANSIT, Sets.<Integer>newLinkedHashSet());
- this.unProcessedPartitions
- .put(State.ON_DISK, Sets.<Integer>newLinkedHashSet());
-
- // Take advantage of multiple disks
- String[] userPaths = PARTITIONS_DIRECTORY.getArray(conf);
- basePaths = new String[userPaths.length];
- int i = 0;
- for (String path : userPaths) {
- basePaths[i++] = path + "/" + conf.get("mapred.job.id", "Unknown Job");
- }
- if (LOG.isInfoEnabled()) {
- LOG.info("DiskBackedPartitionStore with isStaticGraph=" +
- conf.isStaticGraph() + ((userMaxNumPartitions > 0) ?
- (" with maximum " + userMaxNumPartitions + " partitions in memory.") :
- "."));
- }
- }
-
- /**
- * @return maximum number of partitions allowed in memory
- */
- public int getNumPartitionSlots() {
- return maxPartitionsInMem.get();
- }
-
- /**
- * @return number of partitions in memory
- */
- public int getNumPartitionInMemory() {
- return numPartitionsInMem.get();
- }
-
- /**
- * Sets the maximum number of partitions allowed in memory
- *
- * @param numPartitions Number of partitions to allow in memory
- */
- public void setNumPartitionSlots(int numPartitions) {
- maxPartitionsInMem.set(numPartitions);
- }
-
- @Override
- public void initialize() {
- // "initialize" is called right before partition assignment in setup
- // process. However, it might be the case that this worker is a bit slow
- // and other workers start sending vertices/edges (in input superstep)
- // to this worker before the initialize is called. So, we put a guard in
- // necessary places to make sure the 'initialize' is called at a proper time
- // and also only once.
- if (isInitialized.compareAndSet(false, true)) {
- // Set the maximum number of partition slots in memory if unset
- if (maxPartitionsInMem.get() == -1) {
- maxPartitionsInMem.set(serviceWorker.getNumPartitionsOwned());
- // Check if master has not done partition assignment yet (may happen in
- // test codes)
- if (maxPartitionsInMem.get() == 0) {
- LOG.warn("initialize: partitions assigned to this worker is not " +
- "known yet");
- maxPartitionsInMem.set(partitions.size());
- if (maxPartitionsInMem.get() == 0) {
- maxPartitionsInMem.set(Integer.MAX_VALUE);
- }
- }
- if (LOG.isInfoEnabled()) {
- LOG.info("initialize: set the max number of partitions in memory " +
- "to " + maxPartitionsInMem.get());
- }
- oocEngine.initialize();
- }
- }
- }
-
- @Override
- public Iterable<Integer> getPartitionIds() {
- return Iterables.unmodifiableIterable(partitions.keySet());
- }
-
- @Override
- public boolean hasPartition(final Integer id) {
- return partitions.containsKey(id);
- }
-
- @Override
- public int getNumPartitions() {
- return partitions.size();
- }
-
- @Override
- public long getPartitionVertexCount(Integer partitionId) {
- MetaPartition meta = partitions.get(partitionId);
- if (meta == null) {
- return 0;
- } else if (meta.getState() == State.ON_DISK) {
- return meta.getVertexCount();
- } else {
- return meta.getPartition().getVertexCount();
- }
- }
-
- @Override
- public long getPartitionEdgeCount(Integer partitionId) {
- MetaPartition meta = partitions.get(partitionId);
- if (meta == null) {
- return 0;
- } else if (meta.getState() == State.ON_DISK) {
- return meta.getEdgeCount();
- } else {
- return meta.getPartition().getEdgeCount();
- }
- }
-
- /**
- * Spill one partition to disk.
- */
- @edu.umd.cs.findbugs.annotations.SuppressWarnings("TLW_TWO_LOCK_WAIT")
- private void swapOnePartitionToDisk() {
- Integer partitionId;
- // The only partitions in memory are IN_TRANSIT, ACTIVE, and INACTIVE ones.
- // If a partition is currently in transit, it means an OOC thread is
- // pushing the partition to disk, or a compute thread is swapping the
- // partition to open up space for another partition. So, such partitions
- // eventually will free up space in memory. However, this method is usually
- // called at critical points where freeing up space in memory is crucial.
- // So, we should look into a partition to swap amongst other in-memory
- // partitions. An in-memory partition that is not in-transit can be in
- // three states:
- // 1) already processed, which we can simply swap it to disk,
- // 2) non-processed but active (means someone is in the middle of
- // processing the partition). In this case we cannot touch the
- // partition until its processing is done.
- // 3) un-processed and inactive. It is bad to swap this partition to disk
- // as someone will load it again for processing in future. But, this
- // method is called to strictly swap a partition to disk. So, if there
- // is no partition in state 1, we should swap a partition in state 3 to
- // disk.
- rwLock.readLock().lock();
- synchronized (processedPartitions) {
- partitionId = popFromSet(processedPartitions.get(State.INACTIVE));
- }
- if (partitionId == null) {
- synchronized (unProcessedPartitions) {
- partitionId = popFromSet(unProcessedPartitions.get(State.INACTIVE));
- }
- if (partitionId == null) {
- // At this point some partitions are being processed and we should
- // wait until their processing is done
- synchronized (processedPartitions) {
- partitionId = popFromSet(processedPartitions.get(State.INACTIVE));
- while (partitionId == null) {
- try {
- // Here is the only place we wait on 'processedPartition', and
- // this wait is only for INACTIVE entry of the map. So, only at
- // times where a partition is added to INACTIVE entry of this map,
- // we should call '.notifyAll()'. Although this might seem a bad
- // practice, decoupling the INACTIVE entry from this map makes the
- // synchronization mechanism cumbersome and error-prone.
- processedPartitions.wait();
- } catch (InterruptedException e) {
- throw new IllegalStateException("swapOnePartitionToDisk: Caught" +
- "InterruptedException while waiting on a partition to" +
- "become inactive in memory and swapping it to disk");
- }
- partitionId = popFromSet(processedPartitions.get(State.INACTIVE));
- }
- }
- }
- }
-
- if (LOG.isInfoEnabled()) {
- LOG.info("swapOnePartitionToDisk: decided to swap partition " +
- partitionId + " to disk");
- }
- MetaPartition swapOutPartition = partitions.get(partitionId);
- checkNotNull(swapOutPartition,
- "swapOnePartitionToDisk: the partition is not found to spill to disk " +
- "(impossible)");
-
- // Since the partition is popped from the maps, it is not going to be
- // processed (or change its process state) until spilling of the partition
- // is done (the only way to access a partition is through
- // processedPartitions or unProcessedPartitions Map, so once a partition is
- // popped from a map, there is no need for synchronization on that
- // partition).
- Map<State, Set<Integer>> ownerMap = (swapOutPartition.isProcessed()) ?
- processedPartitions :
- unProcessedPartitions;
-
- // Here is the *only* place that holds a lock on an in-transit partition.
- // Anywhere else in the code should call wait() on the in-transit partition
- // to release the lock. This is an important optimization as we are no
- // longer have to keep the lock while partition is being transferred to
- // disk.
- synchronized (swapOutPartition) {
- swapOutPartition.setState(State.IN_TRANSIT);
- synchronized (ownerMap) {
- ownerMap.get(State.IN_TRANSIT).add(partitionId);
- }
- }
-
- try {
- if (LOG.isInfoEnabled()) {
- LOG.info("swapOnePartitionToDisk: start swapping partition " +
- partitionId + " to disk.");
- }
- offloadPartition(swapOutPartition);
- if (LOG.isInfoEnabled()) {
- LOG.info("swapOnePartitionToDisk: done swapping partition " +
- partitionId + " to disk.");
- }
- } catch (IOException e) {
- throw new IllegalStateException(
- "swapOnePartitionToDisk: Failed while offloading partition " +
- partitionId);
- }
-
- synchronized (swapOutPartition) {
- synchronized (ownerMap) {
- boolean stillInMap = ownerMap.get(State.IN_TRANSIT).remove(partitionId);
- swapOutPartition.setOnDisk();
- numPartitionsInMem.getAndDecrement();
- // If a compute thread gets an IN_TRANSIT partition (as the last resort
- // to get the next partition to process), 'swapOutPartition' may no
- // longer be in its map. But, if it is in its own map, we should update
- // the map.
- if (stillInMap) {
- ownerMap.get(State.ON_DISK).add(partitionId);
- }
- }
- // notifying all threads that are waiting for this partition to spill to
- // disk.
- swapOutPartition.notifyAll();
- }
- rwLock.readLock().unlock();
- }
-
- /**
- * Decrement maximum number of partitions allowed in memory by one and pushes
- * one partition to disk if necessary.
- */
- public void spillOnePartition() {
- if (maxPartitionsInMem.getAndDecrement() <= numPartitionsInMem.get()) {
- swapOnePartitionToDisk();
- }
- }
-
- @Override
- public Partition<I, V, E> getNextPartition() {
- Integer partitionId;
- // We prioritize accesses to currently in-memory partitions first. If there
- // is no such partition, we choose amongst on-disk partitions. This is a
- // preferable choice over in-transit partitions since we can start bringing
- // on-disk partitions to memory right away, while if we choose in-transit
- // partitions, we first have to wait for the transit to be complete, and
- // then bring the partition back in memory again.
- synchronized (unProcessedPartitions) {
- partitionId = popFromSet(unProcessedPartitions.get(State.INACTIVE));
- if (partitionId == null) {
- partitionId = popFromSet(unProcessedPartitions.get(State.ON_DISK));
- if (partitionId == null) {
- partitionId = popFromSet(unProcessedPartitions.get(State.IN_TRANSIT));
- }
- }
- }
-
- // Check if we are at the end of the current iteration cycle
- if (partitionId == null) {
- return null;
- }
-
- MetaPartition meta = partitions.get(partitionId);
- checkNotNull(meta, "getNextPartition: partition " + partitionId +
- " does not exist (impossible)");
-
- // The only time we iterate through all partitions in INPUT_SUPERSTEP is
- // when we want to move
- // edges from edge store to vertices. So, we check if we have anything in
- // edge store for the chosen partition, and if there is no edge store for
- // this partition, we skip processing it. This avoids unnecessary loading
- // of on-disk partitions that does not have edge store.
- if (movingEdges) {
- boolean shouldProcess = false;
- synchronized (meta) {
- if (meta.getState() == State.INACTIVE) {
- shouldProcess = edgeStore.hasPartitionEdges(partitionId);
- } else { // either ON_DISK or IN_TRANSIT
- Integer numBuf = numPendingInputEdgesOnDisk.get(partitionId);
- Boolean hasStore = hasEdgeStoreOnDisk.get(partitionId);
- shouldProcess =
- (numBuf != null && numBuf != 0) || (hasStore != null && hasStore);
- }
- if (!shouldProcess) {
- meta.setProcessed(true);
- synchronized (processedPartitions) {
- processedPartitions.get(meta.getState()).add(partitionId);
- if (meta.getState() == State.INACTIVE) {
- processedPartitions.notifyAll();
- }
- }
- }
- }
- if (!shouldProcess) {
- return getNextPartition();
- }
- }
- getPartition(meta);
- return meta.getPartition();
- }
-
- /**
- * Method that gets a partition from the store.
- * The partition is produced as a side effect of the computation and is
- * reflected inside the META object provided as parameter.
- * This function is thread-safe since it locks the whole computation
- * on the MetaPartition provided.
- *
- * When a thread tries to access an element on disk, it waits until space
- * becomes available in memory by swapping partitions to disk.
- *
- * @param meta meta partition container with the partition itself
- */
- private void getPartition(MetaPartition meta) {
- int partitionId = meta.getId();
- synchronized (meta) {
- boolean partitionInMemory = false;
- while (!partitionInMemory) {
- switch (meta.getState()) {
- case INACTIVE:
- // Check if the message store for the current superstep is in memory,
- // and if not, load it from the disk.
- Boolean messagesOnDisk = currentMessagesOnDisk.get(partitionId);
- if (messagesOnDisk != null && messagesOnDisk) {
- try {
- loadMessages(partitionId);
- } catch (IOException e) {
- throw new IllegalStateException("getPartition: failed while " +
- "loading messages of current superstep for partition " +
- partitionId);
- }
- }
- meta.setState(State.ACTIVE);
- partitionInMemory = true;
- break;
- case IN_TRANSIT:
- try {
- // Wait until the partition transfer to disk is complete
- meta.wait();
- } catch (InterruptedException e) {
- throw new IllegalStateException("getPartition: exception " +
- "while waiting on IN_TRANSIT partition " + partitionId + " to" +
- " fully spill to disk.");
- }
- break;
- case ON_DISK:
- boolean spaceAvailable = false;
-
- while (numPartitionsInMem.get() >= maxPartitionsInMem.get()) {
- swapOnePartitionToDisk();
- }
-
- // Reserve the space in memory for the partition
- if (numPartitionsInMem.incrementAndGet() <=
- maxPartitionsInMem.get()) {
- spaceAvailable = true;
- } else {
- numPartitionsInMem.decrementAndGet();
- }
-
- if (spaceAvailable) {
- Partition<I, V, E> partition;
- try {
- if (LOG.isInfoEnabled()) {
- LOG.info("getPartition: start reading partition " +
- partitionId + " from disk");
- }
- partition = loadPartition(meta);
- if (LOG.isInfoEnabled()) {
- LOG.info("getPartition: done reading partition " +
- partitionId + " from disk");
- }
- } catch (IOException e) {
- LOG.error("getPartition: Failed while Loading Partition " +
- "from disk: " + e.getMessage());
- throw new IllegalStateException(e);
- }
- meta.setActive(partition);
- partitionInMemory = true;
- }
- break;
- default:
- throw new IllegalStateException("illegal state " + meta.getState() +
- " for partition " + meta.getId());
- }
- }
- }
- }
-
- @Override
- public void prepareSuperstep() {
- rwLock.writeLock().lock();
- super.prepareSuperstep();
- pendingCurrentMessages = pendingIncomingMessages;
- currentMessagesOnDisk = incomingMessagesOnDisk;
- pendingIncomingMessages = Maps.newConcurrentMap();
- incomingMessagesOnDisk = Maps.newConcurrentMap();
- rwLock.writeLock().unlock();
- }
-
- /**
- * Spill message buffers (either buffers for messages for current superstep,
- * or buffers for incoming messages) of a given partition to disk. Note that
- * the partition should be ON_DISK or IN_TRANSIT.
- *
- * @param partitionId Id of the partition to spill its message buffers
- * @throws IOException
- */
- @edu.umd.cs.findbugs.annotations.SuppressWarnings(
- "UL_UNRELEASED_LOCK_EXCEPTION_PATH")
- public void spillPartitionMessages(Integer partitionId) throws IOException {
- rwLock.readLock().lock();
- spillMessages(partitionId, pendingCurrentMessages,
- serviceWorker.getSuperstep());
- spillMessages(partitionId, pendingIncomingMessages,
- serviceWorker.getSuperstep() + 1);
- rwLock.readLock().unlock();
- }
-
- /**
- * Spill message buffers of a particular type of message (current or incoming
- * buffer) for a partition to disk.
- *
- * @param partitionId Id of the partition to spill the messages for
- * @param pendingMessages The map to get the message buffers from
- * @param superstep Superstep of which we want to offload messages. This is
- * equal to current superstep number if we want to offload
- * buffers for currentMessageStore, and is equal to next
- * superstep number if we want to offload buffer for
- * incomingMessageStore
- * @throws IOException
- */
- private void spillMessages(Integer partitionId,
- ConcurrentMap<Integer, Pair<Integer, List<VertexIdMessages<I, Writable>>>>
- pendingMessages, long superstep) throws IOException {
- Pair<Integer, List<VertexIdMessages<I, Writable>>> entry;
- messageBufferRWLock.writeLock().lock();
- entry = pendingMessages.remove(partitionId);
- if (entry != null && entry.getLeft() < minBuffSize) {
- pendingMessages.put(partitionId, entry);
- entry = null;
- }
- messageBufferRWLock.writeLock().unlock();
-
- if (entry == null) {
- return;
- }
-
- // Sanity check
- checkState(!entry.getRight().isEmpty(),
- "spillMessages: the message buffer that is supposed to be flushed to " +
- "disk does not exist.");
-
- File file = new File(getPendingMessagesBufferPath(partitionId, superstep));
-
- FileOutputStream fos = new FileOutputStream(file, true);
- BufferedOutputStream bos = new BufferedOutputStream(fos);
- DataOutputStream dos = new DataOutputStream(bos);
- for (VertexIdMessages<I, Writable> messages : entry.getRight()) {
- SerializedMessageClass messageClass;
- if (messages instanceof ByteArrayVertexIdMessages) {
- messageClass = SerializedMessageClass.BYTE_ARRAY_VERTEX_ID_MESSAGES;
- } else if (messages instanceof ByteArrayOneMessageToManyIds) {
- messageClass =
- SerializedMessageClass.BYTE_ARRAY_ONE_MESSAGE_TO_MANY_IDS;
- } else {
- throw new IllegalStateException("spillMessages: serialized message " +
- "type is not supported");
- }
- dos.writeInt(messageClass.ordinal());
- messages.write(dos);
- }
- dos.close();
- }
-
- /**
- * Looks through all partitions already on disk, and see if any of them has
- * enough pending message in its buffer in memory. This can be message buffer
- * of current superstep, or incoming superstep. If so, put that partition
- * along with an approximate amount of memory it took (in bytes) in a list to
- * return.
-
- * @return List of pairs (partitionId, sizeInByte) of the partitions where
- * their pending messages are worth flushing to disk
- */
- public PairList<Integer, Integer> getOocPartitionIdsWithPendingMessages() {
- PairList<Integer, Integer> pairList = new PairList<>();
- pairList.initialize();
- Set<Integer> partitionIds = Sets.newHashSet();
- // First, iterating over pending incoming messages
- if (pendingIncomingMessages != null) {
- for (Entry<Integer, Pair<Integer, List<VertexIdMessages<I, Writable>>>>
- entry : pendingIncomingMessages.entrySet()) {
- if (entry.getValue().getLeft() > minBuffSize) {
- pairList.add(entry.getKey(), entry.getValue().getLeft());
- partitionIds.add(entry.getKey());
- }
- }
- }
- // Second, iterating over pending current messages (i.e. pending incoming
- // messages of previous superstep)
- if (pendingCurrentMessages != null) {
- for (Entry<Integer, Pair<Integer, List<VertexIdMessages<I, Writable>>>>
- entry : pendingCurrentMessages.entrySet()) {
- if (entry.getValue().getLeft() > minBuffSize &&
- !partitionIds.contains(entry.getKey())) {
- pairList.add(entry.getKey(), entry.getValue().getLeft());
- }
- }
- }
- return pairList;
- }
-
- @Override
- public <M extends Writable> void addPartitionCurrentMessages(
- int partitionId, VertexIdMessages<I, M> messages) throws IOException {
- // Current messages are only added to the store in the event of partition
- // migration. Presumably the partition has just migrated and its data is
- // still available in memory. Note that partition migration only happens at
- // the beginning of a superstep.
- ((MessageStore<I, M>) currentMessageStore)
- .addPartitionMessages(partitionId, messages);
- }
-
- @Override
- public <M extends Writable> void addPartitionIncomingMessages(
- int partitionId, VertexIdMessages<I, M> messages) throws IOException {
- if (conf.getIncomingMessageClasses().useMessageCombiner()) {
- ((MessageStore<I, M>) incomingMessageStore)
- .addPartitionMessages(partitionId, messages);
- } else {
- MetaPartition meta = partitions.get(partitionId);
- checkNotNull(meta, "addPartitionIncomingMessages: trying to add " +
- "messages to partition " + partitionId + " which does not exist " +
- "in the partition set of this worker!");
-
- synchronized (meta) {
- switch (meta.getState()) {
- case INACTIVE:
- case ACTIVE:
- // A partition might be in memory, but its message store might still
- // be on disk. This happens because while we are loading the partition
- // to memory, we only load its current messages, not the incoming
- // messages. If a new superstep has been started, while the partition
- // is still in memory, the incoming message store in the previous
- // superstep (which is the current messages in the current superstep)
- // is on disk.
- // This may also happen when a partition is offloaded to disk while
- // it was unprocessed, and then again loaded in the same superstep for
- // processing.
- Boolean isMsgOnDisk = incomingMessagesOnDisk.get(partitionId);
- if (isMsgOnDisk == null || !isMsgOnDisk) {
- ((MessageStore<I, M>) incomingMessageStore)
- .addPartitionMessages(partitionId, messages);
- break;
- }
- // Continue to IN_TRANSIT and ON_DISK cases as the partition is in
- // memory, but it's messages are not yet loaded
- // CHECKSTYLE: stop FallThrough
- case IN_TRANSIT:
- case ON_DISK:
- // CHECKSTYLE: resume FallThrough
- List<VertexIdMessages<I, Writable>> newMessages =
- new ArrayList<VertexIdMessages<I, Writable>>();
- newMessages.add((VertexIdMessages<I, Writable>) messages);
- int length = messages.getSerializedSize();
- Pair<Integer, List<VertexIdMessages<I, Writable>>> newPair =
- new MutablePair<>(length, newMessages);
- messageBufferRWLock.readLock().lock();
- Pair<Integer, List<VertexIdMessages<I, Writable>>> oldPair =
- pendingIncomingMessages.putIfAbsent(partitionId, newPair);
- if (oldPair != null) {
- synchronized (oldPair) {
- MutablePair<Integer, List<VertexIdMessages<I, Writable>>> pair =
- (MutablePair<Integer, List<VertexIdMessages<I, Writable>>>)
- oldPair;
- pair.setLeft(pair.getLeft() + length);
- pair.getRight().add((VertexIdMessages<I, Writable>) messages);
- }
- }
- messageBufferRWLock.readLock().unlock();
- // In the case that the number of partitions is asked to be fixed by
- // the user, we should offload the message buffers as necessary.
- if (isNumPartitionsFixed &&
- pendingIncomingMessages.get(partitionId).getLeft() >
- minBuffSize) {
- try {
- spillPartitionMessages(partitionId);
- } catch (IOException e) {
- throw new IllegalStateException("addPartitionIncomingMessages: " +
- "spilling message buffers for partition " + partitionId +
- " failed!");
- }
- }
- break;
- default:
- throw new IllegalStateException("addPartitionIncomingMessages: " +
- "illegal state " + meta.getState() + " for partition " +
- meta.getId());
- }
- }
- }
- }
-
- /**
- * Spills edge store generated for specified partition in INPUT_SUPERSTEP
- * Note that the partition should be ON_DISK or IN_TRANSIT.
- *
- * @param partitionId Id of partition to spill its edge buffer
- */
- public void spillPartitionInputEdgeStore(Integer partitionId)
- throws IOException {
- rwLock.readLock().lock();
- if (movingEdges) {
- rwLock.readLock().unlock();
- return;
- }
- Pair<Integer, List<VertexIdEdges<I, E>>> entry;
-
- // Look at the comment for the similar logic in
- // 'spillPartitionInputVertexBuffer' for why this lock is necessary.
- edgeBufferRWLock.writeLock().lock();
- entry = pendingInputEdges.remove(partitionId);
- edgeBufferRWLock.writeLock().unlock();
-
- // Check if the intermediate edge store has already been moved to vertices
- if (entry == null) {
- rwLock.readLock().unlock();
- return;
- }
-
- // Sanity check
- checkState(!entry.getRight().isEmpty(),
- "spillPartitionInputEdgeStore: the edge buffer that is supposed to " +
- "be flushed to disk does not exist.");
-
- List<VertexIdEdges<I, E>> bufferList = entry.getRight();
- Integer numBuffers = numPendingInputEdgesOnDisk.putIfAbsent(partitionId,
- bufferList.size());
- if (numBuffers != null) {
- numPendingInputEdgesOnDisk.replace(partitionId,
- numBuffers + bufferList.size());
- }
-
- File file = new File(getPendingEdgesBufferPath(partitionId));
- FileOutputStream fos = new FileOutputStream(file, true);
- BufferedOutputStream bos = new BufferedOutputStream(fos);
- DataOutputStream dos = new DataOutputStream(bos);
- for (VertexIdEdges<I, E> edges : entry.getRight()) {
- edges.write(dos);
- }
- dos.close();
- rwLock.readLock().unlock();
- }
-
- /**
- * Looks through all partitions already on disk, and see if any of them has
- * enough pending edges in its buffer in memory. If so, put that
- * partition along with an approximate amount of memory it took (in bytes) in
- * a list to return.
-
- * @return List of pairs (partitionId, sizeInByte) of the partitions where
- * their pending edge store in input superstep in worth flushing to
- * disk
- */
- public PairList<Integer, Integer> getOocPartitionIdsWithPendingInputEdges() {
- PairList<Integer, Integer> pairList = new PairList<>();
- pairList.initialize();
- if (!movingEdges) {
- for (Entry<Integer, Pair<Integer, List<VertexIdEdges<I, E>>>> entry :
- pendingInputEdges.entrySet()) {
- if (entry.getValue().getLeft() > minBuffSize) {
- pairList.add(entry.getKey(), entry.getValue().getLeft());
- }
- }
- }
- return pairList;
- }
-
- @Override
- @edu.umd.cs.findbugs.annotations.SuppressWarnings("SF_SWITCH_FALLTHROUGH")
- public void addPartitionEdges(Integer partitionId,
- VertexIdEdges<I, E> edges) {
- if (!isInitialized.get()) {
- initialize();
- }
-
- MetaPartition meta = new MetaPartition(partitionId);
- MetaPartition temp = partitions.putIfAbsent(partitionId, meta);
- if (temp != null) {
- meta = temp;
- }
-
- boolean createPartition = false;
- synchronized (meta) {
- switch (meta.getState()) {
- case INIT:
- Partition<I, V, E> partition =
- conf.createPartition(partitionId, context);
- meta.setPartition(partition);
- // This is set to processed so that in the very next iteration cycle,
- // when startIteration is called, all partitions seem to be processed
- // and ready for the next iteration cycle. Otherwise, startIteration
- // fails in its sanity check due to finding an unprocessed partition.
- meta.setProcessed(true);
- numPartitionsInMem.getAndIncrement();
- meta.setState(State.INACTIVE);
- synchronized (processedPartitions) {
- processedPartitions.get(State.INACTIVE).add(partitionId);
- processedPartitions.notifyAll();
- }
- createPartition = true;
- // Continue to INACTIVE case to add the edges to the partition
- // CHECKSTYLE: stop FallThrough
- case INACTIVE:
- // CHECKSTYLE: resume FallThrough
- edgeStore.addPartitionEdges(partitionId, edges);
- break;
- case IN_TRANSIT:
- case ON_DISK:
- // Adding edges to in-memory buffer of the partition
- List<VertexIdEdges<I, E>> newEdges =
- new ArrayList<VertexIdEdges<I, E>>();
- newEdges.add(edges);
- int length = edges.getSerializedSize();
- Pair<Integer, List<VertexIdEdges<I, E>>> newPair =
- new MutablePair<>(length, newEdges);
- edgeBufferRWLock.readLock().lock();
- Pair<Integer, List<VertexIdEdges<I, E>>> oldPair =
- pendingInputEdges.putIfAbsent(partitionId, newPair);
- if (oldPair != null) {
- synchronized (oldPair) {
- MutablePair<Integer, List<VertexIdEdges<I, E>>> pair =
- (MutablePair<Integer, List<VertexIdEdges<I, E>>>) oldPair;
- pair.setLeft(pair.getLeft() + length);
- pair.getRight().add(edges);
- }
- }
- edgeBufferRWLock.readLock().unlock();
- // In the case that the number of partitions is asked to be fixed by the
- // user, we should offload the edge store as necessary.
- if (isNumPartitionsFixed &&
- pendingInputEdges.get(partitionId).getLeft() > minBuffSize) {
- try {
- spillPartitionInputEdgeStore(partitionId);
- } catch (IOException e) {
- throw new IllegalStateException("addPartitionEdges: spilling " +
- "edge store for partition " + partitionId + " failed!");
- }
- }
- break;
- default:
- throw new IllegalStateException("illegal state " + meta.getState() +
- " for partition " + meta.getId());
- }
- }
- // If creation of a new partition is violating the policy of maximum number
- // of partitions in memory, we should spill a partition to disk.
- if (createPartition &&
- numPartitionsInMem.get() > maxPartitionsInMem.get()) {
- swapOnePartitionToDisk();
- }
- }
-
- /**
- * Spills vertex buffer generated for specified partition in INPUT_SUPERSTEP
- * Note that the partition should be ON_DISK or IN_TRANSIT.
- *
- * @param partitionId Id of partition to spill its vertex buffer
- */
- public void spillPartitionInputVertexBuffer(Integer partitionId)
- throws IOException {
- rwLock.readLock().lock();
- if (movingEdges) {
- rwLock.readLock().unlock();
- return;
- }
- Pair<Integer, List<ExtendedDataOutput>> entry;
- // Synchronization on the concurrent map is necessary to avoid inconsistent
- // structure while execution of this method is interleaved with the
- // execution of addPartitionVertices. For instance, consider
- // the case where a thread wants to modify the value of an entry in
- // addPartitionVertices while another thread is running this
- // method removing the entry from the map. If removal and offloading the
- // entry's value to disk happens first, the modification by former thread
- // would be lost.
- vertexBufferRWLock.writeLock().lock();
- entry = pendingInputVertices.remove(partitionId);
- vertexBufferRWLock.writeLock().unlock();
-
- // Check if vertex buffer has already been merged with the partition
- if (entry == null) {
- rwLock.readLock().unlock();
- return;
- }
- // Sanity check
- checkState(!entry.getRight().isEmpty(),
- "spillPartitionInputVertexBuffer: the vertex buffer that is " +
- "supposed to be flushed to disk does not exist.");
-
- List<ExtendedDataOutput> bufferList = entry.getRight();
- Integer numBuffers = numPendingInputVerticesOnDisk.putIfAbsent(partitionId,
- bufferList.size());
- if (numBuffers != null) {
- numPendingInputVerticesOnDisk.replace(partitionId,
- numBuffers + bufferList.size());
- }
-
- File file = new File(getPendingVerticesBufferPath(partitionId));
- FileOutputStream fos = new FileOutputStream(file, true);
- BufferedOutputStream bos = new BufferedOutputStream(fos);
- DataOutputStream dos = new DataOutputStream(bos);
- for (ExtendedDataOutput extendedDataOutput : bufferList) {
- WritableUtils.writeExtendedDataOutput(extendedDataOutput, dos);
- }
- dos.close();
- rwLock.readLock().unlock();
- }
-
- /**
- * Looks through all partitions already on disk, and see if any of them has
- * enough pending vertices in its buffer in memory. If so, put that
- * partition along with an approximate amount of memory it took (in bytes) in
- * a list to return.
- *
- * @return List of pairs (partitionId, sizeInByte) of the partitions where
- * their pending vertex buffer in input superstep is worth flushing to
- * disk
- */
- public PairList<Integer, Integer>
- getOocPartitionIdsWithPendingInputVertices() {
- PairList<Integer, Integer> pairList = new PairList<>();
- pairList.initialize();
- if (!movingEdges) {
- for (Entry<Integer, Pair<Integer, List<ExtendedDataOutput>>> entry :
- pendingInputVertices.entrySet()) {
- if (entry.getValue().getLeft() > minBuffSize) {
- pairList.add(entry.getKey(), entry.getValue().getLeft());
- }
- }
- }
- return pairList;
- }
-
- @Override
- @edu.umd.cs.findbugs.annotations.SuppressWarnings("SF_SWITCH_FALLTHROUGH")
- public void addPartitionVertices(Integer partitionId,
- ExtendedDataOutput extendedDataOutput) {
- if (!isInitialized.get()) {
- initialize();
- }
-
- MetaPartition meta = new MetaPartition(partitionId);
- MetaPartition temp = partitions.putIfAbsent(partitionId, meta);
- if (temp != null) {
- meta = temp;
- }
-
- boolean createPartition = false;
- synchronized (meta) {
- switch (meta.getState()) {
- case INIT:
- Partition<I, V, E> partition =
- conf.createPartition(partitionId, context);
- meta.setPartition(partition);
- // Look at the comments in 'addPartitionVertices' for why we set the
- // this to true.
- meta.setProcessed(true);
- numPartitionsInMem.getAndIncrement();
- meta.setState(State.INACTIVE);
- synchronized (processedPartitions) {
- processedPartitions.get(State.INACTIVE).add(partitionId);
- processedPartitions.notifyAll();
- }
- createPartition = true;
- // Continue to INACTIVE case to add the vertices to the partition
- // CHECKSTYLE: stop FallThrough
- case INACTIVE:
- // CHECKSTYLE: resume FallThrough
- meta.getPartition().addPartitionVertices(
- new VertexIterator<I, V, E>(extendedDataOutput, conf));
- break;
- case IN_TRANSIT:
- case ON_DISK:
- // Adding vertices to in-memory buffer of the partition
- List<ExtendedDataOutput> vertices = new ArrayList<ExtendedDataOutput>();
- vertices.add(extendedDataOutput);
- int length = extendedDataOutput.getPos();
- Pair<Integer, List<ExtendedDataOutput>> newPair =
- new MutablePair<>(length, vertices);
- vertexBufferRWLock.readLock().lock();
- Pair<Integer, List<ExtendedDataOutput>> oldPair =
- pendingInputVertices.putIfAbsent(partitionId, newPair);
- if (oldPair != null) {
- synchronized (oldPair) {
- MutablePair<Integer, List<ExtendedDataOutput>> pair =
- (MutablePair<Integer, List<ExtendedDataOutput>>) oldPair;
- pair.setLeft(pair.getLeft() + length);
- pair.getRight().add(extendedDataOutput);
- }
- }
- vertexBufferRWLock.readLock().unlock();
- // In the case that the number of partitions is asked to be fixed by the
- // user, we should offload the edge store as necessary.
- if (isNumPartitionsFixed &&
- pendingInputVertices.get(partitionId).getLeft() > minBuffSize) {
- try {
- spillPartitionInputVertexBuffer(partitionId);
- } catch (IOException e) {
- throw new IllegalStateException("addPartitionVertices: spilling " +
- "vertex buffer for partition " + partitionId + " failed!");
- }
- }
- break;
- default:
- throw new IllegalStateException("illegal state " + meta.getState() +
- " for partition " + meta.getId());
- }
- }
- // If creation of a new partition is violating the policy of maximum number
- // of partitions in memory, we should spill a partition to disk.
- if (createPartition &&
- numPartitionsInMem.get() > maxPartitionsInMem.get()) {
- swapOnePartitionToDisk();
- }
- }
-
- @Override
- public void putPartition(Partition<I, V, E> partition) {
- checkArgument(partition != null);
-
- Integer id = partition.getId();
- MetaPartition meta = partitions.get(id);
- checkNotNull(meta, "putPartition: partition to put does " +
- "not exist in the store (impossible)");
- synchronized (meta) {
- checkState(meta.getState() == State.ACTIVE,
- "It is not possible to put back a partition which is not ACTIVE. " +
- "meta = " + meta.toString());
-
- meta.setState(State.INACTIVE);
- meta.setProcessed(true);
- synchronized (processedPartitions) {
- processedPartitions.get(State.INACTIVE).add(id);
- // Notify OOC threads waiting for a partition to become available to put
- // on disk.
- processedPartitions.notifyAll();
- }
- }
- }
-
- @Override
- public Partition<I, V, E> removePartition(Integer partitionId) {
- if (hasPartition(partitionId)) {
- MetaPartition meta = partitions.remove(partitionId);
- // Since this method is called outside of the iteration cycle, all
- // partitions in the store should be in the processed state.
- checkState(processedPartitions.get(meta.getState()).remove(partitionId),
- "removePartition: partition that is about to remove is not in " +
- "processed list (impossible)");
-
- getPartition(meta);
- numPartitionsInMem.getAndDecrement();
- return meta.getPartition();
- }
- return null;
- }
-
- @Override
- public boolean addPartition(Partition<I, V, E> partition) {
- if (!isInitialized.get()) {
- initialize();
- }
-
- Integer id = partition.getId();
- MetaPartition meta = new MetaPartition(id);
- MetaPartition temp = partitions.putIfAbsent(id, meta);
- if (temp != null) {
- return false;
- }
-
- if (LOG.isInfoEnabled()) {
- LOG.info("addPartition: partition " + id + " is added to the store.");
- }
-
- meta.setPartition(partition);
- meta.setState(State.INACTIVE);
- meta.setProcessed(true);
- synchronized (processedPartitions) {
- processedPartitions.get(State.INACTIVE).add(id);
- processedPartitions.notifyAll();
- }
- numPartitionsInMem.getAndIncrement();
- // Swapping partitions to disk if addition of this partition violates the
- // requirement on the number of partitions.
- if (numPartitionsInMem.get() > maxPartitionsInMem.get()) {
- swapOnePartitionToDisk();
- }
- return true;
- }
-
- @Override
- public void shutdown() {
- // Sanity check to check there is nothing left from previous superstep
- checkState(unProcessedPartitions.get(State.INACTIVE).isEmpty() &&
- unProcessedPartitions.get(State.IN_TRANSIT).isEmpty() &&
- unProcessedPartitions.get(State.ON_DISK).isEmpty(),
- "shutdown: There are some unprocessed partitions left from the " +
- "previous superstep. This should not be possible.");
-
- for (MetaPartition meta : partitions.values()) {
- synchronized (meta) {
- while (meta.getState() == State.IN_TRANSIT) {
- try {
- meta.wait();
- } catch (InterruptedException e) {
- throw new IllegalStateException("shutdown: exception while" +
- "waiting on an IN_TRANSIT partition to be written on disk");
- }
- }
- if (meta.getState() == State.ON_DISK) {
- deletePartitionFiles(meta.getId());
- }
- }
- }
-
- if (oocEngine != null) {
- oocEngine.shutdown();
- }
- }
-
- @Override
- public void startIteration() {
- if (!isInitialized.get()) {
- initialize();
- }
- if (LOG.isInfoEnabled()) {
- LOG.info("startIteration: with " + numPartitionsInMem.get() +
- " partitions in memory, there can be maximum " + maxPartitionsInMem +
- " partitions in memory out of " + partitions.size() + " that " +
- "belongs to this worker.");
- }
- // Sanity check to make sure nothing left unprocessed from previous
- // superstep
- checkState(unProcessedPartitions.get(State.INACTIVE).isEmpty() &&
- unProcessedPartitions.get(State.IN_TRANSIT).isEmpty() &&
- unProcessedPartitions.get(State.ON_DISK).isEmpty(),
- "startIteration: There are some unprocessed and/or " +
- "in-transition partitions left from the previous superstep. " +
- "This should not be possible.");
-
- rwLock.writeLock().lock();
- for (MetaPartition meta : partitions.values()) {
- // Sanity check
- checkState(meta.isProcessed(), "startIteration: " +
- "meta-partition " + meta + " has not been processed in the " +
- "previous superstep.");
-
- // The only case where a partition can be IN_TRANSIT is where it is still
- // being offloaded to disk, and it happens only in swapOnePartitionToDisk,
- // where we at least hold a read lock while transfer is in progress. Since
- // the write lock is held in this section, no partition should be
- // IN_TRANSIT.
- checkState(meta.getState() != State.IN_TRANSIT,
- "startIteration: meta-partition " + meta + " is still IN_TRANSIT " +
- "(impossible)");
-
- meta.setProcessed(false);
- }
-
- unProcessedPartitions.clear();
- unProcessedPartitions.putAll(processedPartitions);
- processedPartitions.clear();
- processedPartitions
- .put(State.INACTIVE, Sets.<Integer>newLinkedHashSet());
- processedPartitions
- .put(State.IN_TRANSIT, Sets.<Integer>newLinkedHashSet());
- processedPartitions
- .put(State.ON_DISK, Sets.<Integer>newLinkedHashSet());
- rwLock.writeLock().unlock();
- LOG.info("startIteration: done preparing the iteration");
- }
-
- @Override
- public void moveEdgesToVertices() {
- movingEdges = true;
- edgeStore.moveEdgesToVertices();
- movingEdges = false;
- }
-
- /**
- * Pops an entry from the specified set. This is guaranteed that the set is
- * being accessed from within a lock.
- *
- * @param set set to pop an entry from
- * @return popped entry from the given set
- */
- private Integer popFromSet(Set<Integer> set) {
- if (!set.isEmpty()) {
- Iterator<Integer> it = set.iterator();
- Integer id = it.next();
- it.remove();
- return id;
- }
- return null;
- }
-
- /**
- * Allows more partitions to be stored in memory.
- *
- * @param numPartitionsToIncrease How many more partitions to allow in the
- * store
- */
- public void increasePartitionSlots(Integer numPartitionsToIncrease) {
- maxPartitionsInMem.getAndAdd(numPartitionsToIncrease);
- if (LOG.isInfoEnabled()) {
- LOG.info("increasePartitionSlots: allowing partition store to have " +
- numPartitionsToIncrease + " more partitions. Now, partition store " +
- "can have up to " + maxPartitionsInMem.get() + " partitions.");
- }
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- for (MetaPartition e : partitions.values()) {
- sb.append(e.toString() + "\n");
- }
- return sb.toString();
- }
-
- /**
- * Writes vertex data (Id, value and halted state) to stream.
- *
- * @param output The output stream
- * @param vertex The vertex to serialize
- * @throws IOException
- */
- private void writeVertexData(DataOutput output, Vertex<I, V, E> vertex)
- throws IOException {
-
- vertex.getId().write(output);
- vertex.getValue().write(output);
- output.writeBoolean(vertex.isHalted());
- }
-
- /**
- * Writes vertex edges (Id, edges) to stream.
- *
- * @param output The output stream
- * @param vertex The vertex to serialize
- * @throws IOException
- */
- @SuppressWarnings("unchecked")
- private void writeOutEdges(DataOutput output, Vertex<I, V, E> vertex)
- throws IOException {
-
- vertex.getId().write(output);
- OutEdges<I, E> edges = (OutEdges<I, E>) vertex.getEdges();
- edges.write(output);
- }
-
- /**
- * Read vertex data from an input and initialize the vertex.
- *
- * @param in The input stream
- * @param vertex The vertex to initialize
- * @throws IOException
- */
- private void readVertexData(DataInput in, Vertex<I, V, E> vertex)
- throws IOException {
-
- I id = conf.createVertexId();
- id.readFields(in);
- V value = conf.createVertexValue();
- value.readFields(in);
- OutEdges<I, E> edges = conf.createAndInitializeOutEdges(0);
- vertex.initialize(id, value, edges);
- if (in.readBoolean()) {
- vertex.voteToHalt();
- } else {
- vertex.wakeUp();
- }
- }
-
- /**
- * Read vertex edges from an input and set them to the vertex.
- *
- * @param in The input stream
- * @param partition The partition owning the vertex
- * @throws IOException
- */
- @SuppressWarnings("unchecked")
- private void readOutEdges(DataInput in, Partition<I, V, E> partition)
- throws IOException {
-
- I id = conf.createVertexId();
- id.readFields(in);
- Vertex<I, V, E> v = partition.getVertex(id);
- OutEdges<I, E> edges = (OutEdges<I, E>) v.getEdges();
- edges.readFields(in);
- partition.saveVertex(v);
- }
-
- /**
- * Load messages for a given partition for the current superstep to memory.
- *
- * @param partitionId Id of the partition to load the messages for
- * @throws IOException
- */
- private void loadMessages(int partitionId) throws IOException {
- // Messages for current superstep
- if (currentMessageStore != null &&
- !conf.getOutgoingMessageClasses().useMessageCombiner()) {
- checkState(!currentMessageStore.hasMessagesForPartition(partitionId),
- "loadMessages: partition " + partitionId + " is on disk, " +
- "but its message store is in memory (impossible)");
- // First, reading the message store for the partition if there is any
- File file = new File(
- getMessagesPath(partitionId, serviceWorker.getSuperstep()));
- if (file.exists()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("loadMessages: loading message store of partition " +
- partitionId);
- }
- FileInputStream filein = new FileInputStream(file);
- BufferedInputStream bufferin = new BufferedInputStream(filein);
- DataInputStream inputStream = new DataInputStream(bufferin);
- currentMessageStore.readFieldsForPartition(inputStream, partitionId);
- inputStream.close();
- checkState(file.delete(), "loadMessages: failed to delete %s.",
- file.getAbsolutePath());
- }
-
- messageBufferRWLock.writeLock().lock();
- Pair<Integer, List<VertexIdMessages<I, Writable>>> pendingMessages =
- pendingCurrentMessages.remove(partitionId);
- messageBufferRWLock.writeLock().unlock();
-
- // Second, reading message buffers (incoming messages in previous
- // superstep)
- file = new File(getPendingMessagesBufferPath(partitionId,
- serviceWorker.getSuperstep()));
- if (file.exists()) {
- FileInputStream filein = new FileInputStream(file);
- BufferedInputStream bufferin = new BufferedInputStream(filein);
- DataInputStream inputStream = new DataInputStream(bufferin);
- while (true) {
- int type;
- try {
- type = inputStream.readInt();
- } catch (EOFException e) {
- // Reached end of file, so all the records are read.
- break;
- }
- SerializedMessageClass messageClass =
- SerializedMessageClass.values()[type];
- VertexIdMessages<I, Writable> vim;
- switch (messageClass) {
- case BYTE_ARRAY_VERTEX_ID_MESSAGES:
- vim = new ByteArrayVertexIdMessages<>(
- conf.createOutgoingMessageValueFactory());
- vim.setConf(conf);
- break;
- case BYTE_ARRAY_ONE_MESSAGE_TO_MANY_IDS:
- vim = new ByteArrayOneMessageToManyIds<>(
- conf.createOutgoingMessageValueFactory());
- vim.setConf(conf);
- break;
- default:
- throw new IllegalStateException("loadMessages: unsupported " +
- "serialized message type!");
- }
- vim.readFields(inputStream);
- currentMessageStore.addPartitionMessages(partitionId, vim);
- }
- inputStream.close();
- checkState(!file.delete(), "loadMessages: failed to delete %s",
- file.getAbsolutePath());
- }
-
- // Third, applying message buffers already in memory
- if (pendingMessages != null) {
- for (VertexIdMessages<I, Writable> vim : pendingMessages.getRight()) {
- currentMessageStore.addPartitionMessages(partitionId, vim);
- }
- }
- currentMessagesOnDisk.put(partitionId, false);
- }
- }
-
- /**
- * Load a partition from disk. It deletes the files after the load,
- * except for the edges, if the graph is static.
- *
- * @param meta meta partition to load the partition of
- * @return The partition
- * @throws IOException
- */
- @SuppressWarnings("unchecked")
- private Partition<I, V, E> loadPartition(MetaPartition meta)
- throws IOException {
- Integer partitionId = meta.getId();
- long numVertices = meta.getVertexCount();
- Partition<I, V, E> partition = conf.createPartition(partitionId, context);
-
- // Vertices
- File file = new File(getVerticesPath(partitionId));
- if (LOG.isDebugEnabled()) {
- LOG.debug("loadPartition: loading partition vertices " +
- partition.getId() + " from " + file.getAbsolutePath());
- }
-
- FileInputStream filein = new FileInputStream(file);
- BufferedInputStream bufferin = new BufferedInputStream(filein);
- DataInputStream inputStream = new DataInputStream(bufferin);
- for (int i = 0; i < numVertices; ++i) {
- Vertex<I, V , E> vertex = conf.createVertex();
- readVertexData(inputStream, vertex);
- partition.putVertex(vertex);
- }
- inputStream.close();
- checkState(file.delete(), "loadPartition: failed to delete %s",
- file.getAbsolutePath());
-
- // Edges
- file = new File(getEdgesPath(partitionId));
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("loadPartition: loading partition edges " +
- partition.getId() + " from " + file.getAbsolutePath());
- }
-
- filein = new FileInputStream(file);
- bufferin = new BufferedInputStream(filein);
- inputStream = new DataInputStream(bufferin);
- for (int i = 0; i < numVertices; ++i) {
- readOutEdges(inputStream, partition);
- }
- inputStream.close();
- // If the graph is static and it is not INPUT_SUPERSTEP, keep the file
- // around.
- if (!conf.isStaticGraph() ||
- serviceWorker.getSuperstep() == BspServiceWorker.INPUT_SUPERSTEP) {
- checkState(file.delete(), "loadPartition: failed to delete %s",
- file.getAbsolutePath());
- }
-
- // Load message for the current superstep
- loadMessages(partitionId);
-
- // Input vertex buffers
- // First, applying vertex buffers on disk (since they came earlier)
- Integer numBuffers = numPendingInputVerticesOnDisk.remove(partitionId);
- if (numBuffers != null) {
- file = new File(getPendingVerticesBufferPath(partitionId));
- if (LOG.isDebugEnabled()) {
- LOG.debug("loadPartition: loading " + numBuffers + " input vertex " +
- "buffers of partition " + partitionId + " from " +
- file.getAbsolutePath());
- }
- filein = new FileInputStream(file);
- bufferin = new BufferedInputStream(filein);
- inputStream = new DataInputStream(bufferin);
- for (int i = 0; i < numBuffers; ++i) {
- ExtendedDataOutput extendedDataOutput =
- WritableUtils.readExtendedDataOutput(inputStream, conf);
- partition.addPartitionVertices(
- new VertexIterator<I, V, E>(extendedDataOutput, conf));
- }
- inputStream.close();
- checkState(file.delete(), "loadPartition: failed to delete %s",
- file.getAbsolutePath());
- }
- // Second, applying vertex buffers already in memory
- Pair<Integer, List<ExtendedDataOutput>> vertexPair;
- vertexBufferRWLock.writeLock().lock();
- vertexPair = pendingInputVertices.remove(partitionId);
- vertexBufferRWLock.writeLock().unlock();
- if (vertexPair != null) {
- for (ExtendedDataOutput extendedDataOutput : vertexPair.getRight()) {
- partition.addPartitionVertices(
- new VertexIterator<I, V, E>(extendedDataOutput, conf));
- }
- }
-
- // Edge store
- if (serviceWorker.getSuperstep() == BspServiceWorker.INPUT_SUPERSTEP) {
- checkState(hasEdgeStoreOnDisk.containsKey(partitionId),
- "loadPartition: partition is written to disk in INPUT_SUPERSTEP, " +
- "but it is not clear whether its edge store is on disk or not " +
- "(impossible)");
-
- if (hasEdgeStoreOnDisk.remove(partitionId)) {
- file = new File(getEdgeStorePath(partitionId));
- if (LOG.isDebugEnabled()) {
- LOG.debug("loadPartition: loading edge store of partition " +
- partitionId + " from " + file.getAbsolutePath());
- }
- filein = new FileInputStream(file);
- bufferin = new BufferedInputStream(filein);
- inputStream = new DataInputStream(bufferin);
- edgeStore.readPartitionEdgeStore(partitionId, inputStream);
- inputStream.close();
- checkState(file.delete(), "loadPartition: failed to delete %s",
- file.getAbsolutePath());
- }
-
- // Input edge buffers
- // First, applying edge buffers on disk (since they came earlier)
- numBuffers = numPendingInputEdgesOnDisk.remove(partitionId);
- if (numBuffers != null) {
- file = new File(getPendingEdgesBufferPath(partitionId));
- if (LOG.isDebugEnabled()) {
- LOG.debug("loadPartition: loading " + numBuffers + " input edge " +
- "buffers of partition " + partitionId + " from " +
- file.getAbsolutePath());
- }
- filein = new FileInputStream(file);
- bufferin = new BufferedInputStream(filein);
- inputStream = new DataInputStream(bufferin);
- for (int i = 0; i < numBuffers; ++i) {
- VertexIdEdges<I, E> vertexIdEdges =
- new ByteArrayVertexIdEdges<I, E>();
- vertexIdEdges.setConf(conf);
- vertexIdEdges.readFields(inputStream);
- edgeStore.addPartitionEdges(partitionId, vertexIdEdges);
- }
- inputStream.close();
- checkState(file.delete(), "loadPartition: failed to delete %s",
- file.getAbsolutePath());
- }
- // Second, applying edge buffers already in memory
- Pair<Integer, List<VertexIdEdges<I, E>>> edgePair = null;
- edgeBufferRWLock.writeLock().lock();
- edgePair = pendingInputEdges.remove(partitionId);
- edgeBufferRWLock.writeLock().unlock();
- if (edgePair != null) {
- for (VertexIdEdges<I, E> vertexIdEdges : edgePair.getRight()) {
- edgeStore.addPartitionEdges(partitionId, vertexIdEdges);
- }
- }
- }
- return partition;
- }
-
- /**
- * Write a partition to disk.
- *
- * @param meta meta partition containing the partition to offload
- * @throws IOException
- */
- private void offloadPartition(MetaPartition meta) throws IOException {
- Partition<I, V, E> partition = meta.getPartition();
- int partitionId = meta.getId();
- File file = new File(getVerticesPath(partitionId));
- File parent = file.getParentFile();
- if (!parent.exists() && !parent.mkdirs() && LOG.isDebugEnabled()) {
- LOG.debug("offloadPartition: directory " + parent.getAbsolutePath() +
- " already exists.");
- }
-
- checkState(file.createNewFile(),
- "offloadPartition: file %s already exists.", parent.getAbsolutePath());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("offloadPartition: writing partition vertices " +
- partitionId + " to " + file.getAbsolutePath());
- }
-
- FileOutputStream fileout = new FileOutputStream(file);
- BufferedOutputStream bufferout = new BufferedOutputStream(fileout);
- DataOutputStream outputStream = new DataOutputStream(bufferout);
- for (Vertex<I, V, E> vertex : partition) {
- writeVertexData(outputStream, vertex);
- }
- outputStream.close();
-
- // Avoid writing back edges if we have already written them once and
- // the graph is not changing.
- // If we are in the input superstep, we need to write the files
- // at least the first time, even though the graph is static.
- file = new File(getEdgesPath(partitionId));
- if (serviceWorker.getSuperstep() == BspServiceWorker.INPUT_SUPERSTEP ||
- meta.getPrevVertexCount() != partition.getVertexCount() ||
- !conf.isStaticGraph() || !file.exists()) {
-
- meta.setPrevVertexCount(partition.getVertexCount());
-
- if (!file.createNewFile() && LOG.isDebugEnabled()) {
- LOG.debug("offloadPartition: file " + file.getAbsolutePath() +
- " already exists.");
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("offloadPartition: writing partition edges " +
- partitionId + " to " + file.getAbsolutePath());
- }
-
- fileout = new FileOutputStream(file);
- bufferout = new BufferedOutputStream(fileout);
- outputStream = new DataOutputStream(bufferout);
- for (Vertex<I, V, E> vertex : partition) {
- writeOutEdges(outputStream, vertex);
- }
- outputStream.close();
- }
-
- if (currentMessageStore != null &&
- !conf.getOutgoingMessageClasses().useMessageCombiner() &&
- currentMessageStore.hasMessagesForPartition(partitionId)) {
- writeMessageData(currentMessageStore, currentMessagesOnDisk, partitionId,
- serviceWorker.getSuperstep());
- }
- if (incomingMessageStore != null &&
- !conf.getIncomingMessageClasses().useMessageCombiner() &&
- incomingMessageStore.hasMessagesForPartition(partitionId)) {
- writeMessageData(incomingMessageStore, incomingMessagesOnDisk,
- partitionId, serviceWorker.getSuperstep() + 1);
- }
-
- // Writing edge store to disk in the input superstep
- if (serviceWorker.getSuperstep() == BspServiceWorker.INPUT_SUPERSTEP) {
- if (edgeStore.hasPartitionEdges(partitionId)) {
- hasEdgeStoreOnDisk.put(partitionId, true);
- file = new File(getEdgeStorePath(partitionId));
- if (!file.createNewFile() && LOG.isDebugEnabled()) {
- LOG.debug("offloadPartition: file " + file.getAbsolutePath() +
- " already exists.");
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("offloadPartition: writing partition edge store of " +
- partitionId + " to " + file.getAbsolutePath());
- }
-
- fileout = new FileOutputStream(file);
- bufferout = new BufferedOutputStream(fileout);
- outputStream = new DataOutputStream(bufferout);
- edgeStore.writePartitionEdgeStore(partitionId, outputStream);
- outputStream.close();
- } else {
- hasEdgeStoreOnDisk.put(partitionId, false);
- }
- }
- }
-
- /**
- * Offload message data of a particular type of store (current or incoming) to
- * disk.
- *
- * @param messageStore The message store to write to disk
- * @param messagesOnDisk Map to update and let others know that this message
- * store is on disk
- * @param partitionId Id of the partition we want to offload the message store
- * of
- * @param superstep Superstep for which we want to offload message data for.
- * It is equal the current superstep number for offloading
- * currentMessageStore, and is equal to next superstep
- * number for offloading incomingMessageStore
- * @throws IOException
- */
- private void writeMessageData(MessageStore<I, Writable> messageStore,
- ConcurrentMap<Integer, Boolean> messagesOnDisk, int partitionId,
- long superstep) throws IOException {
- File file = new File(getMessagesPath(partitionId, superstep));
- checkState(!file.exists(),
- "writeMessageData: message store file for partition " +
- partitionId + " for messages in superstep " +
- superstep + " already exist (impossible).");
-
- checkState(file.createNewFile(),
- "offloadPartition: cannot create message store file for " +
- "partition " + partitionId);
-
- FileOutputStream fileout = new FileOutputStream(file);
- BufferedOutputStream bufferout = new BufferedOutputStream(fileout);
- DataOutputStream outputStream = new DataOutputStream(bufferout);
- messageStore.writePartition(outputStream, partitionId);
- messageStore.clearPartition(partitionId);
- outputStream.close();
- messagesOnDisk.put(partitionId, true);
- }
-
- /**
- * Delete a partition's files.
- *
- * @param id The id of the partition owning the file.
- */
- private void deletePartitionFiles(Integer id) {
- // File containing vertices
- File file = new File(getVerticesPath(id));
- checkState(!file.exists() || file.delete(),
- "deletePartitionFiles: Failed to delete file " +
- file.getAbsolutePath());
-
- // File containing edges
- file = new File(getEdgesPath(id));
- checkState(!file.exists() || file.delete(),
- "deletePartitionFiles: Failed to delete file " +
- file.getAbsolutePath());
- }
-
- /**
- * Get the path and basename of the storage files.
- *
- * @param partitionId The partition
- * @return The path to the given partition
- */
- private String getPartitionPath(Integer partitionId) {
- int hash = hasher.hashInt(partitionId).asInt();
- int idx = Math.abs(hash % basePaths.length);
- return basePaths[idx] + "/partition-" + partitionId;
- }
-
- /**
- * Get the path to the file where vertices are stored.
- *
- * @param partitionId The partition
- * @return The path to the vertices file
- */
- private String getVerticesPath(Integer partitionId) {
- return getPartitionPath(partitionId) + "_vertices";
- }
-
- /**
- * Get the path to the file where pending vertices in INPUT_SUPERSTEP
- * are stored.
- *
- * @param partitionId The partition
- * @return The path to the file
- */
- private String getPendingVerticesBufferPath(Integer partitionId) {
- return getPartitionPath(partitionId) + "_pending_vertices";
- }
-
- /**
- * Get the path to the file where edge store of a partition in INPUT_SUPERSTEP
- * is stored.
- *
- * @param partitionId The partition
- * @return The path to the file
- */
- private String getEdgeStorePath(Integer partitionId) {
- return getPartitionPath(partitionId) + "_edge_store";
- }
-
- /**
- * Get the path to the file where pending edges in INPUT_SUPERSTEP
- * are stored.
- *
- * @param partitionId The partition
- * @return The path to the file
- */
- private String getPendingEdgesBufferPath(Integer partitionId) {
- return getPartitionPath(partitionId) + "_pending_edges";
- }
-
- /**
- * Get the path to the file where edges are stored.
- *
- * @param partitionId The partition
- * @return The path to the edges file
- */
- private String getEdgesPath(Integer partitionId) {
- return getPartitionPath(partitionId) + "_edges";
- }
-
- /**
- * Get the path to the file where pending incoming messages are stored.
- *
- * @param partitionId The partition
- * @param superstep superstep number
- * @return The path to the file
- */
- private String getPendingMessagesBufferPath(Integer partitionId,
- long superstep) {
- return getPartitionPath(partitionId) + "_pending_messages_" + superstep;
- }
-
- /**
- * Get the path to the file where messages are stored.
- *
- * @param partitionId The partition
- * @param superstep superstep number
- * @return The path to the file
- */
- private String getMessagesPath(Integer partitionId, long superstep) {
- return getPartitionPath(partitionId) + "_messages_" + superstep;
- }
-
- /**
- * Partition container holding additional meta data associated with each
- * partition.
- */
- private class MetaPartition {
- // ---- META INFORMATION ----
- /** ID of the partition */
- private int id;
- /** State in which the partition is */
- private State state;
- /** Number of vertices contained in the partition */
- private long vertexCount;
- /** Previous number of vertices contained in the partition */
- private long prevVertexCount;
- /** Number of edges contained in the partition */
- private long edgeCount;
- /**
- * Whether the partition is already processed in the current iteration
- * cycle
- */
- private boolean isProcessed;
-
- // ---- PARTITION ----
- /** the actual partition. Depending on the state of the partition,
- this object could be empty. */
- private Partition<I, V, E> partition;
-
- /**
- * Initialization of the metadata enriched partition.
- *
- * @param id id of the partition
- */
- public MetaPartition(int id) {
- this.id = id;
- this.state = State.INIT;
- this.vertexCount = 0;
- this.prevVertexCount = 0;
- this.edgeCount = 0;
- this.isProcessed = false;
-
- this.partition = null;
- }
-
- /**
- * @return the id
- */
- public int getId() {
- return id;
- }
-
- /**
- * @return the state
- */
- public State getState() {
- return state;
- }
-
- /**
- * This function sets the metadata for on-disk partition.
- */
- public void setOnDisk() {
- this.state = State.ON_DISK;
- this.vertexCount = partition.getVertexCount();
- this.edgeCount = partition.getEdgeCount();
- this.partition = null;
- }
-
- /**
- *
- * @param partition the partition associated to this container
- */
- public void setActive(Partition<I, V, E> partition) {
- if (partition != null) {
- this.partition = partition;
- }
- this.state = State.ACTIVE;
- this.prevVertexCount = this.vertexCount;
- this.vertexCount = 0;
- }
-
- /**
- * @param state the state to set
- */
- public void setState(State state) {
- this.state = state;
- }
-
- /**
- * set previous number of vertexes
- * @param vertexCount number of vertexes
- */
- public void setPrevVertexCount(long vertexCount) {
- this.prevVertexCount = vertexCount;
- }
-
- /**
- * @return the vertexCount
- */
- public long getPrevVertexCount() {
- return prevVertexCount;
- }
-
- /**
- * @return the vertexCount
- */
- public long getVertexCount() {
- return vertexCount;
- }
-
- /**
- * @return the edgeCount
- */
- public long getEdgeCount() {
- return edgeCount;
- }
-
- /**
- * @return true iff the partition is marked as processed.
- */
- public boolean isProcessed() {
- return isProcessed;
- }
-
- /**
- * Set the state of this partition in terms of being already processed or
- * not
- * @param isProcessed whether the partition is processed or not
- */
- public void setProcessed(boolean isProcessed) {
- this.isProcessed = isProcessed;
- }
-
- /**
- * @return the partition
- */
- public Partition<I, V, E> getPartition() {
- return partition;
- }
-
- /**
- * @param partition the partition to set
- */
- public void setPartition(Partition<I, V, E> partition) {
- this.partition = partition;
- }
-
- @Override
- public String toString() {
- StringBuffer sb = new StringBuffer();
-
- sb.append("Meta Data: { ");
- sb.append("ID: " + id + "; ");
- sb.append("State: " + state + "; ");
- sb.append("Number of Vertices: " + vertexCount + "; ");
- sb.append("Previous number of Vertices: " + prevVertexCount + "; ");
- sb.append("Number of edges: " + edgeCount + "; ");
- sb.append("Is processed: " + isProcessed + "; }");
- sb.append("Partition: " + partition + "; }");
-
- return sb.toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fafecee7/giraph-core/src/main/java/org/apache/giraph/ooc/FixedOutOfCoreEngine.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/FixedOutOfCoreEngine.java b/giraph-core/src/main/java/org/apache/giraph/ooc/FixedOutOfCoreEngine.java
new file mode 100644
index 0000000..9324239
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/FixedOutOfCoreEngine.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.ooc;
+
+import org.apache.giraph.bsp.CentralizedServiceWorker;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.ooc.data.MetaPartitionManager;
+import org.apache.giraph.ooc.io.IOCommand;
+import org.apache.giraph.ooc.io.LoadPartitionIOCommand;
+import org.apache.giraph.ooc.io.StorePartitionIOCommand;
+import org.apache.giraph.ooc.io.WaitIOCommand;
+import org.apache.log4j.Logger;
+
+/**
+ * Out-of-core engine maintaining fixed number of partitions in memory.
+ */
+public class FixedOutOfCoreEngine extends OutOfCoreEngine {
+ /** Class logger. */
+ private static final Logger LOG =
+ Logger.getLogger(FixedOutOfCoreEngine.class);
+ /**
+ * When getting partitions, how many milliseconds to wait if no partition was
+ * available in memory
+ */
+ private static final long MSEC_TO_WAIT = 1000;
+ /**
+ * Dummy object to wait on until a partition becomes available in memory
+ * for processing
+ */
+ private final Object partitionAvailable = new Object();
+
+ /**
+ * Constructor
+ *
+ * @param conf Configuration
+ * @param service Service worker
+ * @param maxPartitionsInMemory Maximum number of partitions that can be kept
+ * in memory
+ */
+ public FixedOutOfCoreEngine(ImmutableClassesGiraphConfiguration<?, ?, ?> conf,
+ CentralizedServiceWorker<?, ?, ?> service,
+ int maxPartitionsInMemory) {
+ super(conf, service);
+ this.ioScheduler = new FixedOutOfCoreIOScheduler(maxPartitionsInMemory,
+ numIOThreads, this, conf);
+ }
+
+ @Override
+ public Integer getNextPartition() {
+ Integer partitionId;
+ synchronized (partitionAvailable) {
+ while ((partitionId = metaPartitionManager.getNextPartition()) == null) {
+ try {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("getNextPartition: waiting until a partition becomes " +
+ "available!");
+ }
+ partitionAvailable.wait(MSEC_TO_WAIT);
+ } catch (InterruptedException e) {
+ throw new IllegalStateException("getNextPartition: caught " +
+ "InterruptedException while waiting to retrieve a partition to " +
+ "process");
+ }
+ if (jobFailed) {
+ throw new RuntimeException("Job Failed due to a failure in an " +
+ "out-of-core IO thread");
+ }
+ }
+ }
+ if (partitionId == MetaPartitionManager.NO_PARTITION_TO_PROCESS) {
+ partitionId = null;
+ }
+ return partitionId;
+ }
+
+ @Override
+ public void doneProcessingPartition(int partitionId) {
+ metaPartitionManager.setPartitionIsProcessed(partitionId);
+ // Put the partition in store IO command queue and announce this partition
+ // as a candidate to offload to disk.
+ if (LOG.isInfoEnabled()) {
+ LOG.info("doneProcessingPartition: processing partition " + partitionId +
+ " is done!");
+ }
+ ioScheduler.addIOCommand(new StorePartitionIOCommand(this, partitionId));
+ }
+
+ @Override
+ public void startIteration() {
+ getSuperstepLock().writeLock().lock();
+ metaPartitionManager.resetPartition();
+ ((FixedOutOfCoreIOScheduler) ioScheduler).clearStoreCommandQueue();
+ getSuperstepLock().writeLock().unlock();
+ }
+
+ @Override
+ public void retrievePartition(int partitionId) {
+ long superstep = service.getSuperstep();
+ if (metaPartitionManager.isPartitionOnDisk(partitionId)) {
+ ioScheduler.addIOCommand(new LoadPartitionIOCommand(this, partitionId,
+ superstep));
+ synchronized (partitionAvailable) {
+ while (metaPartitionManager.isPartitionOnDisk(partitionId)) {
+ try {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("retrievePartition: waiting until partition " +
+ partitionId + " becomes available");
+ }
+ partitionAvailable.wait();
+ } catch (InterruptedException e) {
+ throw new IllegalStateException("retrievePartition: caught " +
+ "InterruptedException while waiting to retrieve partition " +
+ partitionId);
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void ioCommandCompleted(IOCommand command) {
+ if (command instanceof LoadPartitionIOCommand ||
+ command instanceof WaitIOCommand) {
+ // Notifying compute threads who are waiting for a partition to become
+ // available in memory to process.
+ synchronized (partitionAvailable) {
+ partitionAvailable.notifyAll();
+ }
+ }
+ }
+}