You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2012/12/20 05:25:31 UTC

[15/51] [partial] GIRAPH-457: update module names (nitay)

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/DefaultVertexResolver.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/DefaultVertexResolver.java b/giraph-core/src/main/java/org/apache/giraph/graph/DefaultVertexResolver.java
new file mode 100644
index 0000000..0b5272d
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/DefaultVertexResolver.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+/**
+ * Default implementation of how to resolve vertex creation/removal, messages
+ * to nonexistent vertices, etc.
+ *
+ * @param <I> Vertex id
+ * @param <V> Vertex data
+ * @param <E> Edge data
+ * @param <M> Message data
+ */
+@SuppressWarnings("rawtypes")
+public class DefaultVertexResolver<I extends WritableComparable,
+    V extends Writable, E extends Writable, M extends Writable>
+    implements VertexResolver<I, V, E, M>,
+    ImmutableClassesGiraphConfigurable<I, V, E, M> {
+  /** Class logger */
+  private static final Logger LOG = Logger.getLogger(
+      DefaultVertexResolver.class);
+  /** Configuration */
+  private ImmutableClassesGiraphConfiguration<I, V, E, M> conf = null;
+  /** Stored graph state */
+  private GraphState<I, V, E, M> graphState;
+
+  /** Whether to create vertices when they receive a message */
+  private boolean createVertexesOnMessages = true;
+
+  @Override
+  public Vertex<I, V, E, M> resolve(
+      I vertexId,
+      Vertex<I, V, E, M> vertex,
+      VertexChanges<I, V, E, M> vertexChanges,
+      boolean hasMessages) {
+    // This is the default vertex resolution algorithm
+
+    // 1. If the vertex exists, first prune the edges
+    removeEdges(vertex, vertexChanges);
+
+    // 2. If vertex removal desired, remove the vertex.
+    vertex = removeVertexIfDesired(vertex, vertexChanges);
+
+    // 3. If creation of vertex desired, pick first vertex
+    // 4. If vertex doesn't exist, but got messages or added edges, create
+    vertex = addVertexIfDesired(vertexId, vertex, vertexChanges, hasMessages);
+
+    // 5. If edge addition, add the edges
+    addEdges(vertex, vertexChanges);
+
+    return vertex;
+  }
+
+  /**
+   * Remove edges as specifed in changes given.
+   *
+   * @param vertex Vertex to remove edges from
+   * @param vertexChanges contains list of edges to remove.
+   */
+  protected void removeEdges(Vertex<I, V, E, M> vertex,
+                             VertexChanges<I, V, E, M> vertexChanges) {
+    if (vertex == null) {
+      return;
+    }
+    if (hasEdgeRemovals(vertexChanges)) {
+      MutableVertex<I, V, E, M> mv = (MutableVertex<I, V, E, M>) vertex;
+      for (I removedDestVertex : vertexChanges.getRemovedEdgeList()) {
+        mv.removeEdges(removedDestVertex);
+      }
+    }
+  }
+
+  /**
+   * Remove the vertex itself if the changes desire it. The actual removal is
+   * notified by returning null. That is, this method does not do the actual
+   * removal but rather returns null if it should be done.
+   *
+   * @param vertex Vertex to remove.
+   * @param vertexChanges specifies if we should remove vertex
+   * @return null if vertex should be removed, otherwise the vertex itself.
+   */
+  protected Vertex<I, V, E, M> removeVertexIfDesired(
+      Vertex<I, V, E, M> vertex,
+      VertexChanges<I, V, E, M> vertexChanges) {
+    if (hasVertexRemovals(vertexChanges)) {
+      vertex = null;
+    }
+    return vertex;
+  }
+
+  /**
+   * Add the Vertex if desired. Returns the vertex itself, or null if no vertex
+   * added.
+   *
+   * @param vertexId ID of vertex
+   * @param vertex Vertex, if not null just returns it as vertex already exists
+   * @param vertexChanges specifies if we should add the vertex
+   * @param hasMessages true if this vertex received any messages
+   * @return Vertex created or passed in, or null if no vertex should be added
+   */
+  protected Vertex<I, V, E, M> addVertexIfDesired(
+      I vertexId,
+      Vertex<I, V, E, M> vertex,
+      VertexChanges<I, V, E, M> vertexChanges,
+      boolean hasMessages) {
+    if (vertex == null) {
+      if (hasVertexAdditions(vertexChanges)) {
+        vertex = vertexChanges.getAddedVertexList().get(0);
+      } else if ((hasMessages && createVertexesOnMessages) ||
+                 hasEdgeAdditions(vertexChanges)) {
+        vertex = conf.createVertex();
+        vertex.setGraphState(graphState);
+        vertex.initialize(vertexId, getConf().createVertexValue());
+      }
+    } else if (hasVertexAdditions(vertexChanges)) {
+      LOG.warn("resolve: Tried to add a vertex with id = " +
+          vertex.getId() + " when one already " +
+          "exists.  Ignoring the add vertex request.");
+    }
+    return vertex;
+  }
+
+  /**
+   * Add edges to the Vertex.
+   *
+   * @param vertex Vertex to add edges to
+   * @param vertexChanges contains edges to add
+   */
+  protected void addEdges(Vertex<I, V, E, M> vertex,
+                          VertexChanges<I, V, E, M> vertexChanges) {
+    if (vertex == null) {
+      return;
+    }
+    if (hasEdgeAdditions(vertexChanges)) {
+      MutableVertex<I, V, E, M> mv = (MutableVertex<I, V, E, M>) vertex;
+      for (Edge<I, E> edge : vertexChanges.getAddedEdgeList()) {
+        mv.addEdge(edge);
+      }
+    }
+  }
+
+  /**
+   * Check if changes contain vertex removal requests
+   *
+   * @param changes VertexChanges to check
+   * @return true if changes contains vertex removal requests
+   */
+  protected  boolean hasVertexRemovals(VertexChanges<I, V, E, M> changes) {
+    return changes != null && changes.getRemovedVertexCount() > 0;
+  }
+
+  /**
+   * Check if changes contain vertex addition requests
+   *
+   * @param changes VertexChanges to check
+   * @return true if changes contains vertex addition requests
+   */
+  protected boolean hasVertexAdditions(VertexChanges<I, V, E, M> changes) {
+    return changes != null && !changes.getAddedVertexList().isEmpty();
+  }
+
+  /**
+   * Check if changes contain edge addition requests
+   *
+   * @param changes VertexChanges to check
+   * @return true if changes contains edge addition requests
+   */
+  protected boolean hasEdgeAdditions(VertexChanges<I, V, E, M> changes) {
+    return changes != null && !changes.getAddedEdgeList().isEmpty();
+  }
+
+  /**
+   * Check if changes contain edge removal requests
+   *
+   * @param changes VertexChanges to check
+   * @return true if changes contains edge removal requests
+   */
+  protected boolean hasEdgeRemovals(VertexChanges<I, V, E, M> changes) {
+    return changes != null && !changes.getRemovedEdgeList().isEmpty();
+  }
+
+  @Override
+  public ImmutableClassesGiraphConfiguration<I, V, E, M> getConf() {
+    return conf;
+  }
+
+  @Override
+  public void setConf(ImmutableClassesGiraphConfiguration<I, V, E, M> conf) {
+    this.conf = conf;
+    createVertexesOnMessages = conf.getResolverCreateVertexOnMessages();
+  }
+
+  @Override
+  public GraphState<I, V, E, M> getGraphState() {
+    return graphState;
+  }
+
+  @Override
+  public void setGraphState(GraphState<I, V, E, M> graphState) {
+    this.graphState = graphState;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/DefaultWorkerContext.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/DefaultWorkerContext.java b/giraph-core/src/main/java/org/apache/giraph/graph/DefaultWorkerContext.java
new file mode 100644
index 0000000..b1042f7
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/DefaultWorkerContext.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+/**
+ * A dumb implementation of {@link WorkerContext}. This is the default
+ * implementation when no WorkerContext is defined by the user. It does
+ * nothing.
+ */
+public class DefaultWorkerContext extends WorkerContext {
+
+  @Override
+  public void preApplication()
+    throws InstantiationException, IllegalAccessException {
+  }
+
+  @Override
+  public void postApplication() { }
+
+  @Override
+  public void preSuperstep() { }
+
+  @Override
+  public void postSuperstep() { }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/Edge.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/Edge.java b/giraph-core/src/main/java/org/apache/giraph/graph/Edge.java
new file mode 100644
index 0000000..b8c4004
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/Edge.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * A complete edge, the target vertex and the edge value.  Can only be one
+ * edge with a destination vertex id per edge map.
+ *
+ * @param <I> Vertex index
+ * @param <E> Edge value
+ */
+@SuppressWarnings("rawtypes")
+public class Edge<I extends WritableComparable, E extends Writable>
+    implements Comparable<Edge<I, E>> {
+  /** Target vertex id */
+  private I targetVertexId = null;
+  /** Edge value */
+  private E value = null;
+
+  /**
+   * Constructor for reflection
+   */
+  public Edge() { }
+
+  /**
+   * Create the edge with final values
+   *
+   * @param targetVertexId Desination vertex id.
+   * @param value Value of the edge.
+   */
+  public Edge(I targetVertexId, E value) {
+    this.targetVertexId = targetVertexId;
+    this.value = value;
+  }
+
+  /**
+   * Get the target vertex index of this edge
+   *
+   * @return Target vertex index of this edge
+   */
+  public I getTargetVertexId() {
+    return targetVertexId;
+  }
+
+  /**
+   * Get the edge value of the edge
+   *
+   * @return Edge value of this edge
+   */
+  public E getValue() {
+    return value;
+  }
+
+  /**
+   * Set the destination vertex index of this edge.
+   *
+   * @param targetVertexId new destination vertex
+   */
+  public void setTargetVertexId(I targetVertexId) {
+    this.targetVertexId = targetVertexId;
+  }
+
+  /**
+   * Set the value for this edge.
+   *
+   * @param value new edge value
+   */
+  public void setValue(E value) {
+    this.value = value;
+  }
+
+  @Override
+  public String toString() {
+    return "(TargetVertexId = " + targetVertexId + ", " +
+        "value = " + value + ")";
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public int compareTo(Edge<I, E> edge) {
+    return targetVertexId.compareTo(edge.getTargetVertexId());
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    Edge edge = (Edge) o;
+
+    if (targetVertexId != null ? !targetVertexId.equals(edge.targetVertexId) :
+      edge.targetVertexId != null) {
+      return false;
+    }
+    if (value != null ? !value.equals(edge.value) : edge.value != null) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = targetVertexId != null ? targetVertexId.hashCode() : 0;
+    result = 31 * result + (value != null ? value.hashCode() : 0);
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputFormat.java
new file mode 100644
index 0000000..cdc1891
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputFormat.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Input format for reading single edges.
+ *
+ * @param <I> Vertex id
+ * @param <E> Edge data
+ */
+public abstract class EdgeInputFormat<I extends WritableComparable,
+    E extends Writable> implements GiraphInputFormat {
+  /**
+   * Logically split the vertices for a graph processing application.
+   *
+   * Each {@link InputSplit} is then assigned to a worker for processing.
+   *
+   * <p><i>Note</i>: The split is a <i>logical</i> split of the inputs and the
+   * input files are not physically split into chunks. For e.g. a split could
+   * be <i>&lt;input-file-path, start, offset&gt;</i> tuple. The InputFormat
+   * also creates the {@link VertexReader} to read the {@link InputSplit}.
+   *
+   * Also, the number of workers is a hint given to the developer to try to
+   * intelligently determine how many splits to create (if this is
+   * adjustable) at runtime.
+   *
+   * @param context Context of the job
+   * @param numWorkers Number of workers used for this job
+   * @return an array of {@link InputSplit}s for the job.
+   */
+  @Override
+  public abstract List<InputSplit> getSplits(
+      JobContext context, int numWorkers) throws IOException,
+      InterruptedException;
+
+  /**
+   * Create an edge reader for a given split. The framework will call
+   * {@link EdgeReader#initialize(InputSplit, TaskAttemptContext)} before
+   * the split is used.
+   *
+   * @param split the split to be read
+   * @param context the information about the task
+   * @return a new record reader
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public abstract EdgeReader<I, E> createEdgeReader(
+      InputSplit split,
+      TaskAttemptContext context) throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallable.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallable.java
new file mode 100644
index 0000000..4e0a543
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallable.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.metrics.GiraphMetrics;
+import org.apache.giraph.metrics.GiraphMetricsRegistry;
+import org.apache.giraph.utils.LoggerUtils;
+import org.apache.giraph.utils.MemoryUtils;
+import org.apache.giraph.zk.ZooKeeperExt;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import com.yammer.metrics.core.Counter;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Load as many edge input splits as possible.
+ * Every thread will has its own instance of WorkerClientRequestProcessor
+ * to send requests.
+ *
+ * @param <I> Vertex id
+ * @param <V> Vertex value
+ * @param <E> Edge value
+ * @param <M> Message data
+ */
+public class EdgeInputSplitsCallable<I extends WritableComparable,
+    V extends Writable, E extends Writable, M extends Writable>
+    extends InputSplitsCallable<I, V, E, M> {
+  /** Class logger */
+  private static final Logger LOG = Logger.getLogger(
+      EdgeInputSplitsCallable.class);
+  /** Total edges loaded */
+  private long totalEdgesLoaded = 0;
+  /** Input split max edges (-1 denotes all) */
+  private final long inputSplitMaxEdges;
+
+  // Metrics
+  /** number of edges loaded counter */
+  private final Counter edgesLoadedCounter;
+
+  /**
+   * Constructor.
+   *
+   * @param context Context
+   * @param graphState Graph state
+   * @param configuration Configuration
+   * @param bspServiceWorker service worker
+   * @param inputSplitPathList List of the paths of the input splits
+   * @param workerInfo This worker's info
+   * @param zooKeeperExt Handle to ZooKeeperExt
+   */
+  public EdgeInputSplitsCallable(
+      Mapper<?, ?, ?, ?>.Context context,
+      GraphState<I, V, E, M> graphState,
+      ImmutableClassesGiraphConfiguration<I, V, E, M> configuration,
+      BspServiceWorker<I, V, E, M> bspServiceWorker,
+      List<String> inputSplitPathList,
+      WorkerInfo workerInfo,
+      ZooKeeperExt zooKeeperExt)  {
+    super(context, graphState, configuration, bspServiceWorker,
+        inputSplitPathList, workerInfo, zooKeeperExt,
+        BspServiceWorker.EDGE_INPUT_SPLIT_RESERVED_NODE,
+        BspServiceWorker.EDGE_INPUT_SPLIT_FINISHED_NODE,
+        bspServiceWorker.edgeInputSplitsEvents);
+
+    inputSplitMaxEdges = configuration.getInputSplitMaxEdges();
+
+    // Initialize Metrics
+    GiraphMetricsRegistry jobMetrics = GiraphMetrics.get().perJob();
+    edgesLoadedCounter = jobMetrics.getCounter(COUNTER_EDGES_LOADED);
+  }
+
+  /**
+   * Read edges from input split.  If testing, the user may request a
+   * maximum number of edges to be read from an input split.
+   *
+   * @param inputSplit Input split to process with edge reader
+   * @param graphState Current graph state
+   * @return Edges loaded from this input split
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @Override
+  protected VertexEdgeCount readInputSplit(
+      InputSplit inputSplit,
+      GraphState<I, V, E, M> graphState) throws IOException,
+      InterruptedException {
+    EdgeInputFormat<I, E> edgeInputFormat =
+        configuration.createEdgeInputFormat();
+    EdgeReader<I, E> edgeReader =
+        edgeInputFormat.createEdgeReader(inputSplit, context);
+    edgeReader.initialize(inputSplit, context);
+    long inputSplitEdgesLoaded = 0;
+    while (edgeReader.nextEdge()) {
+      EdgeWithSource<I, E> readerEdge = edgeReader.getCurrentEdge();
+      if (readerEdge.getSourceVertexId() == null) {
+        throw new IllegalArgumentException(
+            "readInputSplit: Edge reader returned an edge " +
+                "without a source vertex id!  - " + readerEdge);
+      }
+      if (readerEdge.getEdge().getTargetVertexId() == null) {
+        throw new IllegalArgumentException(
+            "readInputSplit: Edge reader returned an edge " +
+                "without a target vertex id!  - " + readerEdge);
+      }
+      if (readerEdge.getEdge().getValue() == null) {
+        throw new IllegalArgumentException(
+            "readInputSplit: Edge reader returned an edge " +
+                "without a value!  - " + readerEdge);
+      }
+
+      graphState.getWorkerClientRequestProcessor().addEdgeRequest(
+          readerEdge.getSourceVertexId(), readerEdge.getEdge());
+      context.progress(); // do this before potential data transfer
+      ++inputSplitEdgesLoaded;
+
+      // Update status every 1M edges
+      if (((inputSplitEdgesLoaded + totalEdgesLoaded) % 1000000) == 0) {
+        LoggerUtils.setStatusAndLog(context, LOG, Level.INFO,
+            "readInputSplit: Loaded " +
+                (inputSplitEdgesLoaded + totalEdgesLoaded) + " edges " +
+                MemoryUtils.getRuntimeMemoryStats());
+      }
+
+      // For sampling, or to limit outlier input splits, the number of
+      // records per input split can be limited
+      if (inputSplitMaxEdges > 0 &&
+          inputSplitEdgesLoaded >= inputSplitMaxEdges) {
+        if (LOG.isInfoEnabled()) {
+          LOG.info("readInputSplit: Leaving the input " +
+              "split early, reached maximum edges " +
+              inputSplitEdgesLoaded);
+        }
+        break;
+      }
+    }
+    edgeReader.close();
+    totalEdgesLoaded += inputSplitEdgesLoaded;
+    edgesLoadedCounter.inc(inputSplitEdgesLoaded);
+    return new VertexEdgeCount(0, inputSplitEdgesLoaded);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallableFactory.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallableFactory.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallableFactory.java
new file mode 100644
index 0000000..41292c7
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallableFactory.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.zk.ZooKeeperExt;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.Mapper;
+
+import java.util.List;
+
+/**
+ * Factory for {@link EdgeInputSplitsCallable}s.
+ *
+ * @param <I> Vertex id
+ * @param <V> Vertex value
+ * @param <E> Edge value
+ * @param <M> Message data
+ */
+public class EdgeInputSplitsCallableFactory<I extends WritableComparable,
+    V extends Writable, E extends Writable, M extends Writable>
+    implements InputSplitsCallableFactory<I, V, E, M> {
+  /** Mapper context. */
+  private final Mapper<?, ?, ?, ?>.Context context;
+  /** Graph state. */
+  private final GraphState<I, V, E, M> graphState;
+  /** Configuration. */
+  private final ImmutableClassesGiraphConfiguration<I, V, E, M> configuration;
+  /** {@link BspServiceWorker} we're running on. */
+  private final BspServiceWorker<I, V, E, M> bspServiceWorker;
+  /** List of input split paths. */
+  private final List<String> inputSplitPathList;
+  /** Worker info. */
+  private final WorkerInfo workerInfo;
+  /** {@link ZooKeeperExt} for this worker. */
+  private final ZooKeeperExt zooKeeperExt;
+
+  /**
+   * Constructor.
+   *
+   * @param context Mapper context
+   * @param graphState Graph state
+   * @param configuration Configuration
+   * @param bspServiceWorker Calling {@link BspServiceWorker}
+   * @param inputSplitPathList List of input split paths
+   * @param workerInfo Worker info
+   * @param zooKeeperExt {@link ZooKeeperExt} for this worker
+   */
+  public EdgeInputSplitsCallableFactory(
+      Mapper<?, ?, ?, ?>.Context context,
+      GraphState<I, V, E, M> graphState,
+      ImmutableClassesGiraphConfiguration<I, V, E, M> configuration,
+      BspServiceWorker<I, V, E, M> bspServiceWorker,
+      List<String> inputSplitPathList,
+      WorkerInfo workerInfo,
+      ZooKeeperExt zooKeeperExt) {
+    this.context = context;
+    this.graphState = graphState;
+    this.configuration = configuration;
+    this.bspServiceWorker = bspServiceWorker;
+    this.inputSplitPathList = inputSplitPathList;
+    this.workerInfo = workerInfo;
+    this.zooKeeperExt = zooKeeperExt;
+  }
+
+  @Override
+  public InputSplitsCallable<I, V, E, M> newCallable() {
+    return new EdgeInputSplitsCallable<I, V, E, M>(
+        context,
+        graphState,
+        configuration,
+        bspServiceWorker,
+        inputSplitPathList,
+        workerInfo,
+        zooKeeperExt);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertex.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertex.java
new file mode 100644
index 0000000..3ca7776
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertex.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+import java.util.Iterator;
+
+/**
+ * User applications can subclass {@link EdgeListVertex}, which stores
+ * the outbound edges in an ArrayList (less memory as the cost of expensive
+ * random-access lookup).  Good for static graphs.  Not nearly as memory
+ * efficient as using RepresentativeVertex + ByteArrayPartition
+ * (probably about 10x more), but not bad when keeping vertices as objects in
+ * memory (SimplePartition).
+ *
+ * @param <I> Vertex index value
+ * @param <V> Vertex value
+ * @param <E> Edge value
+ * @param <M> Message value
+ */
+@SuppressWarnings("rawtypes")
+public abstract class EdgeListVertex<I extends WritableComparable,
+    V extends Writable, E extends Writable, M extends Writable>
+    extends EdgeListVertexBase<I, V, E, M> {
+  /** Class logger */
+  private static final Logger LOG = Logger.getLogger(EdgeListVertex.class);
+  @Override
+  public final boolean addEdge(Edge<I, E> edge) {
+    for (Edge<I, E> currentEdge : getEdges()) {
+      if (currentEdge.getTargetVertexId().equals(edge.getTargetVertexId())) {
+        LOG.warn("addEdge: Vertex=" + getId() +
+            ": already added an edge value for target vertex id " +
+            edge.getTargetVertexId());
+        return false;
+      }
+    }
+    appendEdge(edge);
+    return true;
+  }
+
+  @Override
+  public int removeEdges(I targetVertexId) {
+    for (Iterator<Edge<I, E>> edges = getEdges().iterator(); edges.hasNext();) {
+      Edge<I, E> edge = edges.next();
+      if (edge.getTargetVertexId().equals(targetVertexId)) {
+        edges.remove();
+        return 1;
+      }
+    }
+    return 0;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertexBase.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertexBase.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertexBase.java
new file mode 100644
index 0000000..b1f7be8
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertexBase.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Common base class for edge-list backed vertices.
+ *
+ * @param <I> Vertex id
+ * @param <V> Vertex value
+ * @param <E> Edge value
+ * @param <M> Message data
+ */
+@SuppressWarnings("rawtypes")
+public abstract class EdgeListVertexBase<I extends WritableComparable,
+    V extends Writable, E extends Writable, M extends Writable>
+    extends MutableVertex<I, V, E, M> {
+  /** Class logger */
+  private static final Logger LOG = Logger.getLogger(EdgeListVertexBase.class);
+  /** List of edges */
+  private List<Edge<I, E>> edgeList = Lists.newArrayList();
+
+  /**
+   * Append an edge to the list.
+   *
+   * @param edge Edge to append
+   */
+  protected void appendEdge(Edge<I, E> edge) {
+    edgeList.add(edge);
+  }
+
+  @Override
+  public void setEdges(Iterable<Edge<I, E>> edges) {
+    edgeList.clear();
+    Iterables.addAll(edgeList, edges);
+  }
+
+  @Override
+  public Iterable<Edge<I, E>> getEdges() {
+    return edgeList;
+  }
+
+  @Override
+  public int getNumEdges() {
+    return edgeList.size();
+  }
+
+  @Override
+  public final void readFields(DataInput in) throws IOException {
+    I vertexId = getConf().createVertexId();
+    vertexId.readFields(in);
+    V vertexValue = getConf().createVertexValue();
+    vertexValue.readFields(in);
+    initialize(vertexId, vertexValue);
+
+    int numEdges = in.readInt();
+    edgeList = Lists.newArrayListWithCapacity(numEdges);
+    for (int i = 0; i < numEdges; ++i) {
+      I targetVertexId = getConf().createVertexId();
+      targetVertexId.readFields(in);
+      E edgeValue = getConf().createEdgeValue();
+      edgeValue.readFields(in);
+      edgeList.add(new Edge<I, E>(targetVertexId, edgeValue));
+    }
+
+    readHaltBoolean(in);
+  }
+
+  @Override
+  public final void write(DataOutput out) throws IOException {
+    getId().write(out);
+    getValue().write(out);
+
+    out.writeInt(edgeList.size());
+    for (Edge<I, E> edge : edgeList) {
+      edge.getTargetVertexId().write(out);
+      edge.getValue().write(out);
+    }
+
+    out.writeBoolean(isHalted());
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeReader.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeReader.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeReader.java
new file mode 100644
index 0000000..aba76bd
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeReader.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import java.io.IOException;
+
+/**
+ * Analogous to {@link RecordReader} for edges.  Will read the edges
+ * from an input split.
+ *
+ * @param <I> Vertex id
+ * @param <E> Edge data
+ */
+@SuppressWarnings("rawtypes")
+public interface EdgeReader<I extends WritableComparable, E extends Writable> {
+  /**
+   * Use the input split and context to setup reading the edges.
+   * Guaranteed to be called prior to any other function.
+   *
+   * @param inputSplit Input split to be used for reading edges.
+   * @param context Context from the task.
+   * @throws java.io.IOException
+   * @throws InterruptedException
+   */
+  void initialize(InputSplit inputSplit, TaskAttemptContext context) throws
+      IOException, InterruptedException;
+
+  /**
+   *
+   * @return false iff there are no more edges
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  boolean nextEdge() throws IOException, InterruptedException;
+
+  /**
+   * Get the current edge.
+   *
+   * @return the current edge which has been read.
+   *         nextEdge() should be called first.
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  EdgeWithSource<I, E> getCurrentEdge() throws IOException,
+      InterruptedException;
+
+  /**
+   * Close this {@link EdgeReader} to future operations.
+   *
+   * @throws IOException
+   */
+  void close() throws IOException;
+
+  /**
+   * How much of the input has the {@link EdgeReader} consumed i.e.
+   * has been processed by?
+   *
+   * @return Progress from <code>0.0</code> to <code>1.0</code>.
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  float getProgress() throws IOException, InterruptedException;
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeWithSource.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeWithSource.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeWithSource.java
new file mode 100644
index 0000000..84232c9
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeWithSource.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * A pair of source vertex id and Edge object (that is,
+ * all the information about an edge).
+ *
+ * @param <I> Vertex id
+ * @param <E> Edge data
+ */
+public class EdgeWithSource<I extends WritableComparable, E extends Writable> {
+  /** Source vertex id. */
+  private final I sourceVertexId;
+  /** Edge. */
+  private final Edge<I, E> edge;
+
+  /**
+   * Constructor.
+   *
+   * @param sourceVertexId Source vertex id
+   * @param edge Edge
+   */
+  public EdgeWithSource(I sourceVertexId, Edge<I, E> edge) {
+    this.sourceVertexId = sourceVertexId;
+    this.edge = edge;
+  }
+
+  /**
+   * Get the source vertex id.
+   *
+   * @return Source vertex id.
+   */
+  public I getSourceVertexId() {
+    return sourceVertexId;
+  }
+
+  /**
+   * Get the edge object.
+   *
+   * @return The edge.
+   */
+  public Edge<I, E> getEdge() {
+    return edge;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/FinishedSuperstepStats.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/FinishedSuperstepStats.java b/giraph-core/src/main/java/org/apache/giraph/graph/FinishedSuperstepStats.java
new file mode 100644
index 0000000..d888d10
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/FinishedSuperstepStats.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.graph;
+
+/**
+ * Immutable results of finishSuperste()
+ */
+public class FinishedSuperstepStats extends VertexEdgeCount {
+  /** Are all the graph vertices halted? */
+  private final boolean allVerticesHalted;
+
+  /**
+   * Constructor.
+   *
+   * @param allVerticesHalted Are all the vertices halted
+   * @param numVertices Number of vertices
+   * @param numEdges Number of edges
+   */
+  public FinishedSuperstepStats(boolean allVerticesHalted,
+                                long numVertices,
+                                long numEdges) {
+    super(numVertices, numEdges);
+    this.allVerticesHalted = allVerticesHalted;
+  }
+
+  public boolean getAllVerticesHalted() {
+    return allVerticesHalted;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/GiraphInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GiraphInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphInputFormat.java
new file mode 100644
index 0000000..e1cc844
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphInputFormat.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Common interface for {@link VertexInputFormat} and {@link EdgeInputFormat}.
+ */
+public interface GiraphInputFormat {
+  /**
+   * Get the list of input splits for the format.
+   *
+   * @param context The job context
+   * @param numWorkers Number of workers
+   * @return The list of input splits
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  List<InputSplit> getSplits(JobContext context, int numWorkers)
+    throws IOException, InterruptedException;
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/GiraphJob.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GiraphJob.java b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphJob.java
new file mode 100644
index 0000000..ae7e39c
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphJob.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.giraph.bsp.BspInputFormat;
+import org.apache.giraph.bsp.BspOutputFormat;
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.conf.GiraphConstants;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * Generates an appropriate internal {@link Job} for using Giraph in Hadoop.
+ * Uses composition to avoid unwanted {@link Job} methods from exposure
+ * to the user.
+ */
+public class GiraphJob {
+  static {
+    Configuration.addDefaultResource("giraph-site.xml");
+  }
+
+  /** Class logger */
+  private static final Logger LOG = Logger.getLogger(GiraphJob.class);
+  /** Internal delegated job to proxy interface requests for Job */
+  private final DelegatedJob delegatedJob;
+  /** Name of the job */
+  private final String jobName;
+  /** Helper configuration from the job */
+  private final GiraphConfiguration giraphConfiguration;
+
+  /**
+   * Delegated job that simply passes along the class GiraphConfiguration.
+   */
+  private class DelegatedJob extends Job {
+    /** Ensure that for job initiation the super.getConfiguration() is used */
+    private boolean jobInited = false;
+
+    /**
+     * Constructor
+     *
+     * @throws IOException
+     */
+    DelegatedJob() throws IOException { }
+
+    @Override
+    public Configuration getConfiguration() {
+      if (jobInited) {
+        return giraphConfiguration;
+      } else {
+        return super.getConfiguration();
+      }
+    }
+  }
+
+  /**
+   * Constructor that will instantiate the configuration
+   *
+   * @param jobName User-defined job name
+   * @throws IOException
+   */
+  public GiraphJob(String jobName) throws IOException {
+    this(new GiraphConfiguration(), jobName);
+  }
+
+  /**
+   * Constructor.
+   *
+   * @param configuration User-defined configuration
+   * @param jobName User-defined job name
+   * @throws IOException
+   */
+  public GiraphJob(Configuration configuration,
+                   String jobName) throws IOException {
+    this(new GiraphConfiguration(configuration), jobName);
+  }
+
+  /**
+   * Constructor.
+   *
+   * @param giraphConfiguration User-defined configuration
+   * @param jobName User-defined job name
+   * @throws IOException
+   */
+  public GiraphJob(GiraphConfiguration giraphConfiguration,
+                   String jobName) throws IOException {
+    this.jobName = jobName;
+    this.giraphConfiguration = giraphConfiguration;
+    this.delegatedJob = new DelegatedJob();
+  }
+
+  /**
+   * Get the configuration from the internal job.
+   *
+   * @return Configuration used by the job.
+   */
+  public GiraphConfiguration getConfiguration() {
+    return giraphConfiguration;
+  }
+
+  /**
+   * Be very cautious when using this method as it returns the internal job
+   * of {@link GiraphJob}.  This should only be used for methods that require
+   * access to the actual {@link Job}, i.e. FileInputFormat#addInputPath().
+   *
+   * @return Internal job that will actually be submitted to Hadoop.
+   */
+  public Job getInternalJob() {
+    delegatedJob.jobInited = true;
+    return delegatedJob;
+  }
+  /**
+   * Make sure the configuration is set properly by the user prior to
+   * submitting the job.
+   *
+   * @param conf Configuration to check
+   */
+  private void checkConfiguration(ImmutableClassesGiraphConfiguration conf) {
+    if (conf.getMaxWorkers() < 0) {
+      throw new RuntimeException("checkConfiguration: No valid " +
+          GiraphConstants.MAX_WORKERS);
+    }
+    if (conf.getMinPercentResponded() <= 0.0f ||
+        conf.getMinPercentResponded() > 100.0f) {
+      throw new IllegalArgumentException(
+          "checkConfiguration: Invalid " + conf.getMinPercentResponded() +
+              " for " + GiraphConstants.MIN_PERCENT_RESPONDED);
+    }
+    if (conf.getMinWorkers() < 0) {
+      throw new IllegalArgumentException("checkConfiguration: No valid " +
+          GiraphConstants.MIN_WORKERS);
+    }
+    if (conf.getVertexClass() == null) {
+      throw new IllegalArgumentException("checkConfiguration: Null" +
+          GiraphConstants.VERTEX_CLASS);
+    }
+    if (conf.getVertexInputFormatClass() == null &&
+        conf.getEdgeInputFormatClass() == null) {
+      throw new IllegalArgumentException("checkConfiguration: One of " +
+          GiraphConstants.VERTEX_INPUT_FORMAT_CLASS + " and " +
+          GiraphConstants.EDGE_INPUT_FORMAT_CLASS + " must be non-null");
+    }
+    if (conf.getEdgeInputFormatClass() != null &&
+        !(MutableVertex.class.isAssignableFrom(conf.getVertexClass()))) {
+      throw new IllegalArgumentException("checkConfiguration: EdgeInputFormat" +
+          " only works with mutable vertices");
+    }
+    if (conf.getVertexResolverClass() == null) {
+      if (LOG.isInfoEnabled()) {
+        LOG.info("checkConfiguration: No class found for " +
+            GiraphConstants.VERTEX_RESOLVER_CLASS + ", defaulting to " +
+            DefaultVertexResolver.class.getCanonicalName());
+      }
+    }
+  }
+
+
+  /**
+   * Check if the configuration is local.  If it is local, do additional
+   * checks due to the restrictions of LocalJobRunner.
+   *
+   * @param conf Configuration
+   */
+  private static void checkLocalJobRunnerConfiguration(
+      GiraphConfiguration conf) {
+    String jobTracker = conf.get("mapred.job.tracker", null);
+    if (!jobTracker.equals("local")) {
+      // Nothing to check
+      return;
+    }
+
+    int maxWorkers = conf.getMaxWorkers();
+    if (maxWorkers != 1) {
+      throw new IllegalArgumentException(
+          "checkLocalJobRunnerConfiguration: When using " +
+              "LocalJobRunner, must have only one worker since " +
+          "only 1 task at a time!");
+    }
+    if (conf.getSplitMasterWorker()) {
+      throw new IllegalArgumentException(
+          "checkLocalJobRunnerConfiguration: When using " +
+              "LocalJobRunner, you cannot run in split master / worker " +
+          "mode since there is only 1 task at a time!");
+    }
+  }
+
+  /**
+   * Check whether a specified int conf value is set and if not, set it.
+   *
+   * @param param Conf value to check
+   * @param defaultValue Assign to value if not set
+   */
+  private void setIntConfIfDefault(String param, int defaultValue) {
+    if (giraphConfiguration.getInt(param, Integer.MIN_VALUE) ==
+        Integer.MIN_VALUE) {
+      giraphConfiguration.setInt(param, defaultValue);
+    }
+  }
+
+  /**
+   * Runs the actual graph application through Hadoop Map-Reduce.
+   *
+   * @param verbose If true, provide verbose output, false otherwise
+   * @return True if success, false otherwise
+   * @throws ClassNotFoundException
+   * @throws InterruptedException
+   * @throws IOException
+   */
+  public final boolean run(boolean verbose)
+    throws IOException, InterruptedException, ClassNotFoundException {
+    // Most users won't hit this hopefully and can set it higher if desired
+    setIntConfIfDefault("mapreduce.job.counters.limit", 512);
+
+    // Capacity scheduler-specific settings.  These should be enough for
+    // a reasonable Giraph job
+    setIntConfIfDefault("mapred.job.map.memory.mb", 1024);
+    setIntConfIfDefault("mapred.job.reduce.memory.mb", 1024);
+
+    // Speculative execution doesn't make sense for Giraph
+    giraphConfiguration.setBoolean(
+        "mapred.map.tasks.speculative.execution", false);
+
+    // Set the ping interval to 5 minutes instead of one minute
+    // (DEFAULT_PING_INTERVAL)
+    Client.setPingInterval(giraphConfiguration, 60000 * 5);
+
+    // Should work in MAPREDUCE-1938 to let the user jars/classes
+    // get loaded first
+    giraphConfiguration.setBoolean("mapreduce.user.classpath.first", true);
+    giraphConfiguration.setBoolean("mapreduce.job.user.classpath.first", true);
+
+    // If the checkpoint frequency is 0 (no failure handling), set the max
+    // tasks attempts to be 0 to encourage faster failure of unrecoverable jobs
+    if (giraphConfiguration.getCheckpointFrequency() == 0) {
+      int oldMaxTaskAttempts = giraphConfiguration.getMaxTaskAttempts();
+      giraphConfiguration.setMaxTaskAttempts(0);
+      if (LOG.isInfoEnabled()) {
+        LOG.info("run: Since checkpointing is disabled (default), " +
+            "do not allow any task retries (setting " +
+            GiraphConstants.MAX_TASK_ATTEMPTS + " = 0, " +
+            "old value = " + oldMaxTaskAttempts + ")");
+      }
+    }
+
+    // Set the job properties, check them, and submit the job
+    ImmutableClassesGiraphConfiguration immutableClassesGiraphConfiguration =
+        new ImmutableClassesGiraphConfiguration(giraphConfiguration);
+    checkConfiguration(immutableClassesGiraphConfiguration);
+    checkLocalJobRunnerConfiguration(immutableClassesGiraphConfiguration);
+    Job submittedJob = new Job(immutableClassesGiraphConfiguration, jobName);
+    if (submittedJob.getJar() == null) {
+      submittedJob.setJarByClass(GiraphJob.class);
+    }
+    submittedJob.setNumReduceTasks(0);
+    submittedJob.setMapperClass(GraphMapper.class);
+    submittedJob.setInputFormatClass(BspInputFormat.class);
+    submittedJob.setOutputFormatClass(BspOutputFormat.class);
+    return submittedJob.waitForCompletion(verbose);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTransferRegulator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTransferRegulator.java b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTransferRegulator.java
new file mode 100644
index 0000000..8bec886
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTransferRegulator.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.giraph.graph.partition.PartitionOwner;
+import com.google.common.collect.Maps;
+import java.util.Map;
+
+
+/** Utility class to manage data transfers from
+ * a local worker reading InputSplits.
+ * Currently, this class measures # of vertices and edges
+ * per outgoing Collection of graph data (destined for a
+ * particular Partition and remote worker node, preselected
+ * by the master.)
+ *
+ * TODO: implement defaults and configurable options for
+ * measuring the size of input &lt;V&gt; or &lt;E&gt; data
+ * per read vertex, and setting limits on totals per outgoing
+ * graph data Collection etc. (See GIRAPH-260)
+ */
+public class GiraphTransferRegulator {
+  /** Maximum vertices to read from an InputSplit locally that are
+   * to be routed to a remote worker, before sending them. */
+  public static final String MAX_VERTICES_PER_TRANSFER =
+    "giraph.maxVerticesPerTransfer";
+  /** Default maximum number of vertices per
+   * temp partition before sending. */
+  public static final int MAX_VERTICES_PER_TRANSFER_DEFAULT = 10000;
+  /**
+   * Maximum edges to read from an InputSplit locally that are
+   * to be routed to a remote worker, before sending them.
+   */
+  public static final String MAX_EDGES_PER_TRANSFER =
+    "giraph.maxEdgesPerTransfer";
+  /** Default maximum number of vertices per
+   * temp partition before sending. */
+  public static final int MAX_EDGES_PER_TRANSFER_DEFAULT = 80000;
+
+  /** Internal state to measure when
+   * the next data transfer of a Collection
+   * of vertices read by the local worker that
+   * owns this regulator is ready to be sent
+   * to the remote worker node that the master
+   * has assigned the vertices to */
+  private Map<Integer, Integer> edgeAccumulator;
+
+  /** Internal state to measure when
+   * the next data transfer of a Collection
+   * of vertices read by the local worker that
+   * owns this regulator is ready to be sent
+   * to the remote worker node that the master
+   * has assigned the vertices to */
+  private Map<Integer, Integer> vertexAccumulator;
+
+  /** Number of vertices per data transfer */
+  private final int maxVerticesPerTransfer;
+
+  /** Number of edges per data transfer */
+  private final int maxEdgesPerTransfer;
+
+  /** Vertex count total for this InputSplit */
+  private long totalVertexCount;
+
+  /** Edge count total for this InputSplit */
+  private long totalEdgeCount;
+
+  /** Default constructor
+   * @param conf the Configuration for this job
+   */
+  public GiraphTransferRegulator(Configuration conf) {
+    vertexAccumulator = Maps.<Integer, Integer>newHashMap();
+    edgeAccumulator = Maps.<Integer, Integer>newHashMap();
+    maxVerticesPerTransfer = conf.getInt(
+        MAX_VERTICES_PER_TRANSFER,
+        MAX_VERTICES_PER_TRANSFER_DEFAULT);
+    maxEdgesPerTransfer = conf.getInt(
+        MAX_EDGES_PER_TRANSFER,
+        MAX_EDGES_PER_TRANSFER_DEFAULT);
+    totalEdgeCount = 0;
+    totalVertexCount = 0;
+  }
+
+  /** Is this outbound data Collection full,
+   * and ready to transfer?
+   * @param owner the partition owner for the outbound data
+   * @return 'true' if the temp partition data is ready to transfer
+   */
+  public boolean transferThisPartition(PartitionOwner owner) {
+    final int partitionId = owner.getPartitionId();
+    if (getEdgesForPartition(partitionId) >=
+      maxEdgesPerTransfer ||
+      getVerticesForPartition(partitionId) >=
+      maxVerticesPerTransfer) {
+      vertexAccumulator.put(partitionId, 0);
+      edgeAccumulator.put(partitionId, 0);
+      return true;
+    }
+    return false;
+  }
+
+  /** get current vertex count for a given Collection of
+   * data soon to be transfered to its permanent home.
+   * @param partId the partition id to check the count on.
+   * @return the count of vertices.
+   */
+  private int getVerticesForPartition(final int partId) {
+    return vertexAccumulator.get(partId) == null ?
+      0 : vertexAccumulator.get(partId);
+  }
+
+  /** get current edge count for a given Collection of
+   * data soon to be transfered to its permanent home.
+   * @param partId the partition id to check the count on.
+   * @return the count of edges.
+   */
+  private int getEdgesForPartition(final int partId) {
+    return edgeAccumulator.get(partId) == null ?
+      0 : edgeAccumulator.get(partId);
+  }
+
+  /** Clear storage to reset for reading new InputSplit */
+  public void clearCounters() {
+    totalEdgeCount = 0;
+    totalVertexCount = 0;
+    vertexAccumulator.clear();
+    edgeAccumulator.clear();
+  }
+
+  /** Increment V & E counts for new vertex read, store values
+   * for that outgoing _temporary_ Partition, which shares the
+   * Partition ID for the actual remote Partition the collection
+   * will eventually be processed in.
+   * @param partitionOwner the owner of the Partition this data
+   *  will eventually belong to.
+   * @param vertex the vertex to extract counts from.
+   * @param <I> the vertex id type.
+   * @param <V> the vertex value type.
+   * @param <E> the edge value type.
+   * @param <M> the message value type.
+   */
+  public <I extends WritableComparable, V extends Writable,
+  E extends Writable, M extends Writable> void
+  incrementCounters(PartitionOwner partitionOwner,
+    Vertex<I, V, E, M> vertex) {
+    final int id = partitionOwner.getPartitionId();
+    // vertex counts
+    vertexAccumulator
+      .put(id, getVerticesForPartition(id) + 1);
+    totalVertexCount++;
+    // edge counts
+    totalEdgeCount += vertex.getNumEdges();
+    edgeAccumulator.put(id, getEdgesForPartition(id) +
+      vertex.getNumEdges());
+  }
+
+  /** Getter for MAX edge count to initiate a transfer
+    * @return max edge count per transfer */
+  public long getMaxEdgesPerTransfer() {
+    return maxEdgesPerTransfer;
+  }
+
+  /** Getter for MAX vertex count to initiate a transfer
+   * @return max edge count per transfer */
+  public long getMaxVerticesPerTransfer() {
+    return maxVerticesPerTransfer;
+  }
+
+  /** Getter for total edge count for the current InputSplit
+    * @return the # of total edges counted in this InputSplit */
+  public long getTotalEdges() {
+    return totalEdgeCount;
+  }
+
+  /** Getter for total vetex count for the current InputSplit
+   * @return the total # of vertices counted in this InputSplit */
+  public long getTotalVertices() {
+    return totalVertexCount;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTypeValidator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTypeValidator.java b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTypeValidator.java
new file mode 100644
index 0000000..f0b4b77
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTypeValidator.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.giraph.utils.ReflectionUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+import java.lang.reflect.Type;
+import java.util.List;
+
+/**
+ * GiraphTypeValidator attempts to verify the consistency of
+ * user-chosen InputFormat, OutputFormat, and Vertex type
+ * parameters before the job run actually begins.
+ *
+ * @param <I> the Vertex ID type
+ * @param <V> the Vertex Value type
+ * @param <E> the Edge Value type
+ * @param <M> the Message type
+ */
+public class GiraphTypeValidator<I extends WritableComparable,
+  V extends Writable, E extends Writable, M extends Writable> {
+  /**
+   * Class logger object.
+   */
+  private static Logger LOG =
+    Logger.getLogger(GiraphTypeValidator.class);
+
+  /** I param vertex index in classList */
+  private static final int ID_PARAM_INDEX = 0;
+  /** V param vertex index in classList */
+  private static final int VALUE_PARAM_INDEX = 1;
+  /** E param vertex index in classList */
+  private static final int EDGE_PARAM_INDEX = 2;
+  /** M param vertex index in classList */
+  private static final int MSG_PARAM_INDEX = 3;
+  /** M param vertex combiner index in classList */
+  private static final int MSG_COMBINER_PARAM_INDEX = 1;
+
+  /** Vertex Index Type */
+  private Type vertexIndexType;
+  /** Vertex Index Type */
+  private Type vertexValueType;
+  /** Vertex Index Type */
+  private Type edgeValueType;
+  /** Vertex Index Type */
+  private Type messageValueType;
+
+  /**
+   * The Configuration object for use in the validation test.
+   */
+  private Configuration conf;
+
+  /**
+   * Constructor to execute the validation test, throws
+   * unchecked exception to end job run on failure.
+   *
+   * @param conf the Configuration for this run.
+   */
+  public GiraphTypeValidator(Configuration conf) {
+    this.conf = conf;
+  }
+
+  /**
+   * Make sure that all registered classes have matching types.  This
+   * is a little tricky due to type erasure, cannot simply get them from
+   * the class type arguments.  Also, set the vertex index, vertex value,
+   * edge value and message value classes.
+   */
+  public void validateClassTypes() {
+    Class<? extends Vertex<I, V, E, M>> vertexClass =
+      BspUtils.<I, V, E, M>getVertexClass(conf);
+    List<Class<?>> classList = ReflectionUtils.getTypeArguments(
+      Vertex.class, vertexClass);
+    vertexIndexType = classList.get(ID_PARAM_INDEX);
+    vertexValueType = classList.get(VALUE_PARAM_INDEX);
+    edgeValueType = classList.get(EDGE_PARAM_INDEX);
+    messageValueType = classList.get(MSG_PARAM_INDEX);
+    verifyVertexInputFormatGenericTypes();
+    verifyVertexOutputFormatGenericTypes();
+    verifyVertexResolverGenericTypes();
+    verifyVertexCombinerGenericTypes();
+  }
+
+  /** Verify matching generic types in VertexInputFormat. */
+  private void verifyVertexInputFormatGenericTypes() {
+    Class<? extends VertexInputFormat<I, V, E, M>> vertexInputFormatClass =
+      BspUtils.<I, V, E, M>getVertexInputFormatClass(conf);
+    List<Class<?>> classList =
+      ReflectionUtils.getTypeArguments(
+        VertexInputFormat.class, vertexInputFormatClass);
+    if (classList.get(ID_PARAM_INDEX) == null) {
+      LOG.warn("Input format vertex index type is not known");
+    } else if (!vertexIndexType.equals(classList.get(ID_PARAM_INDEX))) {
+      throw new IllegalArgumentException(
+        "checkClassTypes: Vertex index types don't match, " +
+          "vertex - " + vertexIndexType +
+          ", vertex input format - " + classList.get(ID_PARAM_INDEX));
+    }
+    if (classList.get(VALUE_PARAM_INDEX) == null) {
+      LOG.warn("Input format vertex value type is not known");
+    } else if (!vertexValueType.equals(classList.get(VALUE_PARAM_INDEX))) {
+      throw new IllegalArgumentException(
+        "checkClassTypes: Vertex value types don't match, " +
+          "vertex - " + vertexValueType +
+          ", vertex input format - " + classList.get(VALUE_PARAM_INDEX));
+    }
+    if (classList.get(EDGE_PARAM_INDEX) == null) {
+      LOG.warn("Input format edge value type is not known");
+    } else if (!edgeValueType.equals(classList.get(EDGE_PARAM_INDEX))) {
+      throw new IllegalArgumentException(
+        "checkClassTypes: Edge value types don't match, " +
+          "vertex - " + edgeValueType +
+          ", vertex input format - " + classList.get(EDGE_PARAM_INDEX));
+    }
+  }
+
+  /** If there is a combiner type, verify its generic params match the job. */
+  private void verifyVertexCombinerGenericTypes() {
+    Class<? extends Combiner<I, M>> vertexCombinerClass =
+      BspUtils.<I, M>getCombinerClass(conf);
+    if (vertexCombinerClass != null) {
+      List<Class<?>> classList =
+        ReflectionUtils.getTypeArguments(
+          Combiner.class, vertexCombinerClass);
+      if (!vertexIndexType.equals(classList.get(ID_PARAM_INDEX))) {
+        throw new IllegalArgumentException(
+          "checkClassTypes: Vertex index types don't match, " +
+            "vertex - " + vertexIndexType +
+            ", vertex combiner - " + classList.get(ID_PARAM_INDEX));
+      }
+      if (!messageValueType.equals(classList.get(MSG_COMBINER_PARAM_INDEX))) {
+        throw new IllegalArgumentException(
+          "checkClassTypes: Message value types don't match, " +
+            "vertex - " + messageValueType +
+            ", vertex combiner - " + classList.get(MSG_COMBINER_PARAM_INDEX));
+      }
+    }
+  }
+
+  /** Verify that the output format's generic params match the job. */
+  private void verifyVertexOutputFormatGenericTypes() {
+    Class<? extends VertexOutputFormat<I, V, E>>
+      vertexOutputFormatClass =
+      BspUtils.<I, V, E>getVertexOutputFormatClass(conf);
+    if (vertexOutputFormatClass != null) {
+      List<Class<?>> classList =
+        ReflectionUtils.getTypeArguments(
+          VertexOutputFormat.class, vertexOutputFormatClass);
+      if (classList.get(ID_PARAM_INDEX) == null) {
+        LOG.warn("Output format vertex index type is not known");
+      } else if (!vertexIndexType.equals(classList.get(ID_PARAM_INDEX))) {
+        throw new IllegalArgumentException(
+          "checkClassTypes: Vertex index types don't match, " +
+            "vertex - " + vertexIndexType +
+            ", vertex output format - " + classList.get(ID_PARAM_INDEX));
+      }
+      if (classList.get(VALUE_PARAM_INDEX) == null) {
+        LOG.warn("Output format vertex value type is not known");
+      } else if (!vertexValueType.equals(classList.get(VALUE_PARAM_INDEX))) {
+        throw new IllegalArgumentException(
+          "checkClassTypes: Vertex value types don't match, " +
+            "vertex - " + vertexValueType +
+            ", vertex output format - " + classList.get(VALUE_PARAM_INDEX));
+      }
+      if (classList.get(EDGE_PARAM_INDEX) == null) {
+        LOG.warn("Output format edge value type is not known");
+      } else if (!edgeValueType.equals(classList.get(EDGE_PARAM_INDEX))) {
+        throw new IllegalArgumentException(
+          "checkClassTypes: Edge value types don't match, " +
+            "vertex - " + edgeValueType +
+            ", vertex output format - " + classList.get(EDGE_PARAM_INDEX));
+      }
+    }
+  }
+
+  /** If there is a vertex resolver,
+   * validate the generic parameter types. */
+  private void verifyVertexResolverGenericTypes() {
+    Class<? extends VertexResolver<I, V, E, M>>
+      vrClass = BspUtils.<I, V, E, M>getVertexResolverClass(conf);
+    if (!DefaultVertexResolver.class.isAssignableFrom(vrClass)) {
+      return;
+    }
+    Class<? extends DefaultVertexResolver<I, V, E, M>>
+      dvrClass =
+        (Class<? extends DefaultVertexResolver<I, V, E, M>>) vrClass;
+    List<Class<?>> classList =
+      ReflectionUtils.getTypeArguments(
+          DefaultVertexResolver.class, dvrClass);
+    if (classList.get(ID_PARAM_INDEX) != null &&
+      !vertexIndexType.equals(classList.get(ID_PARAM_INDEX))) {
+      throw new IllegalArgumentException(
+        "checkClassTypes: Vertex index types don't match, " +
+          "vertex - " + vertexIndexType +
+          ", vertex resolver - " + classList.get(ID_PARAM_INDEX));
+    }
+    if (classList.get(VALUE_PARAM_INDEX) != null &&
+      !vertexValueType.equals(classList.get(VALUE_PARAM_INDEX))) {
+      throw new IllegalArgumentException(
+        "checkClassTypes: Vertex value types don't match, " +
+          "vertex - " + vertexValueType +
+          ", vertex resolver - " + classList.get(VALUE_PARAM_INDEX));
+    }
+    if (classList.get(EDGE_PARAM_INDEX) != null &&
+      !edgeValueType.equals(classList.get(EDGE_PARAM_INDEX))) {
+      throw new IllegalArgumentException(
+        "checkClassTypes: Edge value types don't match, " +
+          "vertex - " + edgeValueType +
+          ", vertex resolver - " + classList.get(EDGE_PARAM_INDEX));
+    }
+    if (classList.get(MSG_PARAM_INDEX) != null &&
+      !messageValueType.equals(classList.get(MSG_PARAM_INDEX))) {
+      throw new IllegalArgumentException(
+        "checkClassTypes: Message value types don't match, " +
+          "vertex - " + messageValueType +
+          ", vertex resolver - " + classList.get(MSG_PARAM_INDEX));
+    }
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/GlobalStats.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GlobalStats.java b/giraph-core/src/main/java/org/apache/giraph/graph/GlobalStats.java
new file mode 100644
index 0000000..aa03c4b
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GlobalStats.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.giraph.graph.partition.PartitionStats;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Aggregated stats by the master.
+ */
+public class GlobalStats implements Writable {
+  /** All vertices in the application */
+  private long vertexCount = 0;
+  /** All finished vertices in the last superstep */
+  private long finishedVertexCount = 0;
+  /** All edges in the last superstep */
+  private long edgeCount = 0;
+  /** All messages sent in the last superstep */
+  private long messageCount = 0;
+  /** Whether the computation should be halted */
+  private boolean haltComputation = false;
+
+  /**
+   * Add the stats of a partition to the global stats.
+   *
+   * @param partitionStats Partition stats to be added.
+   */
+  public void addPartitionStats(PartitionStats partitionStats) {
+    this.vertexCount += partitionStats.getVertexCount();
+    this.finishedVertexCount += partitionStats.getFinishedVertexCount();
+    this.edgeCount += partitionStats.getEdgeCount();
+  }
+
+  public long getVertexCount() {
+    return vertexCount;
+  }
+
+  public long getFinishedVertexCount() {
+    return finishedVertexCount;
+  }
+
+  public long getEdgeCount() {
+    return edgeCount;
+  }
+
+  public long getMessageCount() {
+    return messageCount;
+  }
+
+  public boolean getHaltComputation() {
+    return haltComputation;
+  }
+
+  public void setHaltComputation(boolean value) {
+    haltComputation = value;
+  }
+
+  /**
+   * Add messages to the global stats.
+   *
+   * @param messageCount Number of messages to be added.
+   */
+  public void addMessageCount(long messageCount) {
+    this.messageCount += messageCount;
+  }
+
+  @Override
+  public void readFields(DataInput input) throws IOException {
+    vertexCount = input.readLong();
+    finishedVertexCount = input.readLong();
+    edgeCount = input.readLong();
+    messageCount = input.readLong();
+    haltComputation = input.readBoolean();
+  }
+
+  @Override
+  public void write(DataOutput output) throws IOException {
+    output.writeLong(vertexCount);
+    output.writeLong(finishedVertexCount);
+    output.writeLong(edgeCount);
+    output.writeLong(messageCount);
+    output.writeBoolean(haltComputation);
+  }
+
+  @Override
+  public String toString() {
+    return "(vtx=" + vertexCount + ",finVtx=" +
+        finishedVertexCount + ",edges=" + edgeCount + ",msgCount=" +
+        messageCount + ",haltComputation=" + haltComputation + ")";
+  }
+}