You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2012/12/20 05:25:31 UTC
[15/51] [partial] GIRAPH-457: update module names (nitay)
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/DefaultVertexResolver.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/DefaultVertexResolver.java b/giraph-core/src/main/java/org/apache/giraph/graph/DefaultVertexResolver.java
new file mode 100644
index 0000000..0b5272d
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/DefaultVertexResolver.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+/**
+ * Default implementation of how to resolve vertex creation/removal, messages
+ * to nonexistent vertices, etc.
+ *
+ * @param <I> Vertex id
+ * @param <V> Vertex data
+ * @param <E> Edge data
+ * @param <M> Message data
+ */
+@SuppressWarnings("rawtypes")
+public class DefaultVertexResolver<I extends WritableComparable,
+ V extends Writable, E extends Writable, M extends Writable>
+ implements VertexResolver<I, V, E, M>,
+ ImmutableClassesGiraphConfigurable<I, V, E, M> {
+ /** Class logger */
+ private static final Logger LOG = Logger.getLogger(
+ DefaultVertexResolver.class);
+ /** Configuration */
+ private ImmutableClassesGiraphConfiguration<I, V, E, M> conf = null;
+ /** Stored graph state */
+ private GraphState<I, V, E, M> graphState;
+
+ /** Whether to create vertices when they receive a message */
+ private boolean createVertexesOnMessages = true;
+
+ @Override
+ public Vertex<I, V, E, M> resolve(
+ I vertexId,
+ Vertex<I, V, E, M> vertex,
+ VertexChanges<I, V, E, M> vertexChanges,
+ boolean hasMessages) {
+ // This is the default vertex resolution algorithm
+
+ // 1. If the vertex exists, first prune the edges
+ removeEdges(vertex, vertexChanges);
+
+ // 2. If vertex removal desired, remove the vertex.
+ vertex = removeVertexIfDesired(vertex, vertexChanges);
+
+ // 3. If creation of vertex desired, pick first vertex
+ // 4. If vertex doesn't exist, but got messages or added edges, create
+ vertex = addVertexIfDesired(vertexId, vertex, vertexChanges, hasMessages);
+
+ // 5. If edge addition, add the edges
+ addEdges(vertex, vertexChanges);
+
+ return vertex;
+ }
+
+ /**
+ * Remove edges as specifed in changes given.
+ *
+ * @param vertex Vertex to remove edges from
+ * @param vertexChanges contains list of edges to remove.
+ */
+ protected void removeEdges(Vertex<I, V, E, M> vertex,
+ VertexChanges<I, V, E, M> vertexChanges) {
+ if (vertex == null) {
+ return;
+ }
+ if (hasEdgeRemovals(vertexChanges)) {
+ MutableVertex<I, V, E, M> mv = (MutableVertex<I, V, E, M>) vertex;
+ for (I removedDestVertex : vertexChanges.getRemovedEdgeList()) {
+ mv.removeEdges(removedDestVertex);
+ }
+ }
+ }
+
+ /**
+ * Remove the vertex itself if the changes desire it. The actual removal is
+ * notified by returning null. That is, this method does not do the actual
+ * removal but rather returns null if it should be done.
+ *
+ * @param vertex Vertex to remove.
+ * @param vertexChanges specifies if we should remove vertex
+ * @return null if vertex should be removed, otherwise the vertex itself.
+ */
+ protected Vertex<I, V, E, M> removeVertexIfDesired(
+ Vertex<I, V, E, M> vertex,
+ VertexChanges<I, V, E, M> vertexChanges) {
+ if (hasVertexRemovals(vertexChanges)) {
+ vertex = null;
+ }
+ return vertex;
+ }
+
+ /**
+ * Add the Vertex if desired. Returns the vertex itself, or null if no vertex
+ * added.
+ *
+ * @param vertexId ID of vertex
+ * @param vertex Vertex, if not null just returns it as vertex already exists
+ * @param vertexChanges specifies if we should add the vertex
+ * @param hasMessages true if this vertex received any messages
+ * @return Vertex created or passed in, or null if no vertex should be added
+ */
+ protected Vertex<I, V, E, M> addVertexIfDesired(
+ I vertexId,
+ Vertex<I, V, E, M> vertex,
+ VertexChanges<I, V, E, M> vertexChanges,
+ boolean hasMessages) {
+ if (vertex == null) {
+ if (hasVertexAdditions(vertexChanges)) {
+ vertex = vertexChanges.getAddedVertexList().get(0);
+ } else if ((hasMessages && createVertexesOnMessages) ||
+ hasEdgeAdditions(vertexChanges)) {
+ vertex = conf.createVertex();
+ vertex.setGraphState(graphState);
+ vertex.initialize(vertexId, getConf().createVertexValue());
+ }
+ } else if (hasVertexAdditions(vertexChanges)) {
+ LOG.warn("resolve: Tried to add a vertex with id = " +
+ vertex.getId() + " when one already " +
+ "exists. Ignoring the add vertex request.");
+ }
+ return vertex;
+ }
+
+ /**
+ * Add edges to the Vertex.
+ *
+ * @param vertex Vertex to add edges to
+ * @param vertexChanges contains edges to add
+ */
+ protected void addEdges(Vertex<I, V, E, M> vertex,
+ VertexChanges<I, V, E, M> vertexChanges) {
+ if (vertex == null) {
+ return;
+ }
+ if (hasEdgeAdditions(vertexChanges)) {
+ MutableVertex<I, V, E, M> mv = (MutableVertex<I, V, E, M>) vertex;
+ for (Edge<I, E> edge : vertexChanges.getAddedEdgeList()) {
+ mv.addEdge(edge);
+ }
+ }
+ }
+
+ /**
+ * Check if changes contain vertex removal requests
+ *
+ * @param changes VertexChanges to check
+ * @return true if changes contains vertex removal requests
+ */
+ protected boolean hasVertexRemovals(VertexChanges<I, V, E, M> changes) {
+ return changes != null && changes.getRemovedVertexCount() > 0;
+ }
+
+ /**
+ * Check if changes contain vertex addition requests
+ *
+ * @param changes VertexChanges to check
+ * @return true if changes contains vertex addition requests
+ */
+ protected boolean hasVertexAdditions(VertexChanges<I, V, E, M> changes) {
+ return changes != null && !changes.getAddedVertexList().isEmpty();
+ }
+
+ /**
+ * Check if changes contain edge addition requests
+ *
+ * @param changes VertexChanges to check
+ * @return true if changes contains edge addition requests
+ */
+ protected boolean hasEdgeAdditions(VertexChanges<I, V, E, M> changes) {
+ return changes != null && !changes.getAddedEdgeList().isEmpty();
+ }
+
+ /**
+ * Check if changes contain edge removal requests
+ *
+ * @param changes VertexChanges to check
+ * @return true if changes contains edge removal requests
+ */
+ protected boolean hasEdgeRemovals(VertexChanges<I, V, E, M> changes) {
+ return changes != null && !changes.getRemovedEdgeList().isEmpty();
+ }
+
+ @Override
+ public ImmutableClassesGiraphConfiguration<I, V, E, M> getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(ImmutableClassesGiraphConfiguration<I, V, E, M> conf) {
+ this.conf = conf;
+ createVertexesOnMessages = conf.getResolverCreateVertexOnMessages();
+ }
+
+ @Override
+ public GraphState<I, V, E, M> getGraphState() {
+ return graphState;
+ }
+
+ @Override
+ public void setGraphState(GraphState<I, V, E, M> graphState) {
+ this.graphState = graphState;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/DefaultWorkerContext.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/DefaultWorkerContext.java b/giraph-core/src/main/java/org/apache/giraph/graph/DefaultWorkerContext.java
new file mode 100644
index 0000000..b1042f7
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/DefaultWorkerContext.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+/**
+ * A dumb implementation of {@link WorkerContext}. This is the default
+ * implementation when no WorkerContext is defined by the user. It does
+ * nothing.
+ */
+public class DefaultWorkerContext extends WorkerContext {
+
+ @Override
+ public void preApplication()
+ throws InstantiationException, IllegalAccessException {
+ }
+
+ @Override
+ public void postApplication() { }
+
+ @Override
+ public void preSuperstep() { }
+
+ @Override
+ public void postSuperstep() { }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/Edge.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/Edge.java b/giraph-core/src/main/java/org/apache/giraph/graph/Edge.java
new file mode 100644
index 0000000..b8c4004
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/Edge.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * A complete edge, the target vertex and the edge value. Can only be one
+ * edge with a destination vertex id per edge map.
+ *
+ * @param <I> Vertex index
+ * @param <E> Edge value
+ */
+@SuppressWarnings("rawtypes")
+public class Edge<I extends WritableComparable, E extends Writable>
+ implements Comparable<Edge<I, E>> {
+ /** Target vertex id */
+ private I targetVertexId = null;
+ /** Edge value */
+ private E value = null;
+
+ /**
+ * Constructor for reflection
+ */
+ public Edge() { }
+
+ /**
+ * Create the edge with final values
+ *
+ * @param targetVertexId Desination vertex id.
+ * @param value Value of the edge.
+ */
+ public Edge(I targetVertexId, E value) {
+ this.targetVertexId = targetVertexId;
+ this.value = value;
+ }
+
+ /**
+ * Get the target vertex index of this edge
+ *
+ * @return Target vertex index of this edge
+ */
+ public I getTargetVertexId() {
+ return targetVertexId;
+ }
+
+ /**
+ * Get the edge value of the edge
+ *
+ * @return Edge value of this edge
+ */
+ public E getValue() {
+ return value;
+ }
+
+ /**
+ * Set the destination vertex index of this edge.
+ *
+ * @param targetVertexId new destination vertex
+ */
+ public void setTargetVertexId(I targetVertexId) {
+ this.targetVertexId = targetVertexId;
+ }
+
+ /**
+ * Set the value for this edge.
+ *
+ * @param value new edge value
+ */
+ public void setValue(E value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ return "(TargetVertexId = " + targetVertexId + ", " +
+ "value = " + value + ")";
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public int compareTo(Edge<I, E> edge) {
+ return targetVertexId.compareTo(edge.getTargetVertexId());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ Edge edge = (Edge) o;
+
+ if (targetVertexId != null ? !targetVertexId.equals(edge.targetVertexId) :
+ edge.targetVertexId != null) {
+ return false;
+ }
+ if (value != null ? !value.equals(edge.value) : edge.value != null) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = targetVertexId != null ? targetVertexId.hashCode() : 0;
+ result = 31 * result + (value != null ? value.hashCode() : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputFormat.java
new file mode 100644
index 0000000..cdc1891
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputFormat.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Input format for reading single edges.
+ *
+ * @param <I> Vertex id
+ * @param <E> Edge data
+ */
+public abstract class EdgeInputFormat<I extends WritableComparable,
+ E extends Writable> implements GiraphInputFormat {
+ /**
+ * Logically split the vertices for a graph processing application.
+ *
+ * Each {@link InputSplit} is then assigned to a worker for processing.
+ *
+ * <p><i>Note</i>: The split is a <i>logical</i> split of the inputs and the
+ * input files are not physically split into chunks. For e.g. a split could
+ * be <i><input-file-path, start, offset></i> tuple. The InputFormat
+ * also creates the {@link VertexReader} to read the {@link InputSplit}.
+ *
+ * Also, the number of workers is a hint given to the developer to try to
+ * intelligently determine how many splits to create (if this is
+ * adjustable) at runtime.
+ *
+ * @param context Context of the job
+ * @param numWorkers Number of workers used for this job
+ * @return an array of {@link InputSplit}s for the job.
+ */
+ @Override
+ public abstract List<InputSplit> getSplits(
+ JobContext context, int numWorkers) throws IOException,
+ InterruptedException;
+
+ /**
+ * Create an edge reader for a given split. The framework will call
+ * {@link EdgeReader#initialize(InputSplit, TaskAttemptContext)} before
+ * the split is used.
+ *
+ * @param split the split to be read
+ * @param context the information about the task
+ * @return a new record reader
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public abstract EdgeReader<I, E> createEdgeReader(
+ InputSplit split,
+ TaskAttemptContext context) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallable.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallable.java
new file mode 100644
index 0000000..4e0a543
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallable.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.metrics.GiraphMetrics;
+import org.apache.giraph.metrics.GiraphMetricsRegistry;
+import org.apache.giraph.utils.LoggerUtils;
+import org.apache.giraph.utils.MemoryUtils;
+import org.apache.giraph.zk.ZooKeeperExt;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import com.yammer.metrics.core.Counter;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Load as many edge input splits as possible.
+ * Every thread will has its own instance of WorkerClientRequestProcessor
+ * to send requests.
+ *
+ * @param <I> Vertex id
+ * @param <V> Vertex value
+ * @param <E> Edge value
+ * @param <M> Message data
+ */
+public class EdgeInputSplitsCallable<I extends WritableComparable,
+ V extends Writable, E extends Writable, M extends Writable>
+ extends InputSplitsCallable<I, V, E, M> {
+ /** Class logger */
+ private static final Logger LOG = Logger.getLogger(
+ EdgeInputSplitsCallable.class);
+ /** Total edges loaded */
+ private long totalEdgesLoaded = 0;
+ /** Input split max edges (-1 denotes all) */
+ private final long inputSplitMaxEdges;
+
+ // Metrics
+ /** number of edges loaded counter */
+ private final Counter edgesLoadedCounter;
+
+ /**
+ * Constructor.
+ *
+ * @param context Context
+ * @param graphState Graph state
+ * @param configuration Configuration
+ * @param bspServiceWorker service worker
+ * @param inputSplitPathList List of the paths of the input splits
+ * @param workerInfo This worker's info
+ * @param zooKeeperExt Handle to ZooKeeperExt
+ */
+ public EdgeInputSplitsCallable(
+ Mapper<?, ?, ?, ?>.Context context,
+ GraphState<I, V, E, M> graphState,
+ ImmutableClassesGiraphConfiguration<I, V, E, M> configuration,
+ BspServiceWorker<I, V, E, M> bspServiceWorker,
+ List<String> inputSplitPathList,
+ WorkerInfo workerInfo,
+ ZooKeeperExt zooKeeperExt) {
+ super(context, graphState, configuration, bspServiceWorker,
+ inputSplitPathList, workerInfo, zooKeeperExt,
+ BspServiceWorker.EDGE_INPUT_SPLIT_RESERVED_NODE,
+ BspServiceWorker.EDGE_INPUT_SPLIT_FINISHED_NODE,
+ bspServiceWorker.edgeInputSplitsEvents);
+
+ inputSplitMaxEdges = configuration.getInputSplitMaxEdges();
+
+ // Initialize Metrics
+ GiraphMetricsRegistry jobMetrics = GiraphMetrics.get().perJob();
+ edgesLoadedCounter = jobMetrics.getCounter(COUNTER_EDGES_LOADED);
+ }
+
+ /**
+ * Read edges from input split. If testing, the user may request a
+ * maximum number of edges to be read from an input split.
+ *
+ * @param inputSplit Input split to process with edge reader
+ * @param graphState Current graph state
+ * @return Edges loaded from this input split
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Override
+ protected VertexEdgeCount readInputSplit(
+ InputSplit inputSplit,
+ GraphState<I, V, E, M> graphState) throws IOException,
+ InterruptedException {
+ EdgeInputFormat<I, E> edgeInputFormat =
+ configuration.createEdgeInputFormat();
+ EdgeReader<I, E> edgeReader =
+ edgeInputFormat.createEdgeReader(inputSplit, context);
+ edgeReader.initialize(inputSplit, context);
+ long inputSplitEdgesLoaded = 0;
+ while (edgeReader.nextEdge()) {
+ EdgeWithSource<I, E> readerEdge = edgeReader.getCurrentEdge();
+ if (readerEdge.getSourceVertexId() == null) {
+ throw new IllegalArgumentException(
+ "readInputSplit: Edge reader returned an edge " +
+ "without a source vertex id! - " + readerEdge);
+ }
+ if (readerEdge.getEdge().getTargetVertexId() == null) {
+ throw new IllegalArgumentException(
+ "readInputSplit: Edge reader returned an edge " +
+ "without a target vertex id! - " + readerEdge);
+ }
+ if (readerEdge.getEdge().getValue() == null) {
+ throw new IllegalArgumentException(
+ "readInputSplit: Edge reader returned an edge " +
+ "without a value! - " + readerEdge);
+ }
+
+ graphState.getWorkerClientRequestProcessor().addEdgeRequest(
+ readerEdge.getSourceVertexId(), readerEdge.getEdge());
+ context.progress(); // do this before potential data transfer
+ ++inputSplitEdgesLoaded;
+
+ // Update status every 1M edges
+ if (((inputSplitEdgesLoaded + totalEdgesLoaded) % 1000000) == 0) {
+ LoggerUtils.setStatusAndLog(context, LOG, Level.INFO,
+ "readInputSplit: Loaded " +
+ (inputSplitEdgesLoaded + totalEdgesLoaded) + " edges " +
+ MemoryUtils.getRuntimeMemoryStats());
+ }
+
+ // For sampling, or to limit outlier input splits, the number of
+ // records per input split can be limited
+ if (inputSplitMaxEdges > 0 &&
+ inputSplitEdgesLoaded >= inputSplitMaxEdges) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("readInputSplit: Leaving the input " +
+ "split early, reached maximum edges " +
+ inputSplitEdgesLoaded);
+ }
+ break;
+ }
+ }
+ edgeReader.close();
+ totalEdgesLoaded += inputSplitEdgesLoaded;
+ edgesLoadedCounter.inc(inputSplitEdgesLoaded);
+ return new VertexEdgeCount(0, inputSplitEdgesLoaded);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallableFactory.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallableFactory.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallableFactory.java
new file mode 100644
index 0000000..41292c7
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeInputSplitsCallableFactory.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.zk.ZooKeeperExt;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.Mapper;
+
+import java.util.List;
+
+/**
+ * Factory for {@link EdgeInputSplitsCallable}s.
+ *
+ * @param <I> Vertex id
+ * @param <V> Vertex value
+ * @param <E> Edge value
+ * @param <M> Message data
+ */
+public class EdgeInputSplitsCallableFactory<I extends WritableComparable,
+ V extends Writable, E extends Writable, M extends Writable>
+ implements InputSplitsCallableFactory<I, V, E, M> {
+ /** Mapper context. */
+ private final Mapper<?, ?, ?, ?>.Context context;
+ /** Graph state. */
+ private final GraphState<I, V, E, M> graphState;
+ /** Configuration. */
+ private final ImmutableClassesGiraphConfiguration<I, V, E, M> configuration;
+ /** {@link BspServiceWorker} we're running on. */
+ private final BspServiceWorker<I, V, E, M> bspServiceWorker;
+ /** List of input split paths. */
+ private final List<String> inputSplitPathList;
+ /** Worker info. */
+ private final WorkerInfo workerInfo;
+ /** {@link ZooKeeperExt} for this worker. */
+ private final ZooKeeperExt zooKeeperExt;
+
+ /**
+ * Constructor.
+ *
+ * @param context Mapper context
+ * @param graphState Graph state
+ * @param configuration Configuration
+ * @param bspServiceWorker Calling {@link BspServiceWorker}
+ * @param inputSplitPathList List of input split paths
+ * @param workerInfo Worker info
+ * @param zooKeeperExt {@link ZooKeeperExt} for this worker
+ */
+ public EdgeInputSplitsCallableFactory(
+ Mapper<?, ?, ?, ?>.Context context,
+ GraphState<I, V, E, M> graphState,
+ ImmutableClassesGiraphConfiguration<I, V, E, M> configuration,
+ BspServiceWorker<I, V, E, M> bspServiceWorker,
+ List<String> inputSplitPathList,
+ WorkerInfo workerInfo,
+ ZooKeeperExt zooKeeperExt) {
+ this.context = context;
+ this.graphState = graphState;
+ this.configuration = configuration;
+ this.bspServiceWorker = bspServiceWorker;
+ this.inputSplitPathList = inputSplitPathList;
+ this.workerInfo = workerInfo;
+ this.zooKeeperExt = zooKeeperExt;
+ }
+
+ @Override
+ public InputSplitsCallable<I, V, E, M> newCallable() {
+ return new EdgeInputSplitsCallable<I, V, E, M>(
+ context,
+ graphState,
+ configuration,
+ bspServiceWorker,
+ inputSplitPathList,
+ workerInfo,
+ zooKeeperExt);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertex.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertex.java
new file mode 100644
index 0000000..3ca7776
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertex.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+import java.util.Iterator;
+
+/**
+ * User applications can subclass {@link EdgeListVertex}, which stores
+ * the outbound edges in an ArrayList (less memory as the cost of expensive
+ * random-access lookup). Good for static graphs. Not nearly as memory
+ * efficient as using RepresentativeVertex + ByteArrayPartition
+ * (probably about 10x more), but not bad when keeping vertices as objects in
+ * memory (SimplePartition).
+ *
+ * @param <I> Vertex index value
+ * @param <V> Vertex value
+ * @param <E> Edge value
+ * @param <M> Message value
+ */
+@SuppressWarnings("rawtypes")
+public abstract class EdgeListVertex<I extends WritableComparable,
+ V extends Writable, E extends Writable, M extends Writable>
+ extends EdgeListVertexBase<I, V, E, M> {
+ /** Class logger */
+ private static final Logger LOG = Logger.getLogger(EdgeListVertex.class);
+ @Override
+ public final boolean addEdge(Edge<I, E> edge) {
+ for (Edge<I, E> currentEdge : getEdges()) {
+ if (currentEdge.getTargetVertexId().equals(edge.getTargetVertexId())) {
+ LOG.warn("addEdge: Vertex=" + getId() +
+ ": already added an edge value for target vertex id " +
+ edge.getTargetVertexId());
+ return false;
+ }
+ }
+ appendEdge(edge);
+ return true;
+ }
+
+ @Override
+ public int removeEdges(I targetVertexId) {
+ for (Iterator<Edge<I, E>> edges = getEdges().iterator(); edges.hasNext();) {
+ Edge<I, E> edge = edges.next();
+ if (edge.getTargetVertexId().equals(targetVertexId)) {
+ edges.remove();
+ return 1;
+ }
+ }
+ return 0;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertexBase.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertexBase.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertexBase.java
new file mode 100644
index 0000000..b1f7be8
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeListVertexBase.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Common base class for edge-list backed vertices.
+ *
+ * @param <I> Vertex id
+ * @param <V> Vertex value
+ * @param <E> Edge value
+ * @param <M> Message data
+ */
+@SuppressWarnings("rawtypes")
+public abstract class EdgeListVertexBase<I extends WritableComparable,
+ V extends Writable, E extends Writable, M extends Writable>
+ extends MutableVertex<I, V, E, M> {
+ /** Class logger */
+ private static final Logger LOG = Logger.getLogger(EdgeListVertexBase.class);
+ /** List of edges */
+ private List<Edge<I, E>> edgeList = Lists.newArrayList();
+
+ /**
+ * Append an edge to the list.
+ *
+ * @param edge Edge to append
+ */
+ protected void appendEdge(Edge<I, E> edge) {
+ edgeList.add(edge);
+ }
+
+ @Override
+ public void setEdges(Iterable<Edge<I, E>> edges) {
+ edgeList.clear();
+ Iterables.addAll(edgeList, edges);
+ }
+
+ @Override
+ public Iterable<Edge<I, E>> getEdges() {
+ return edgeList;
+ }
+
+ @Override
+ public int getNumEdges() {
+ return edgeList.size();
+ }
+
+ @Override
+ public final void readFields(DataInput in) throws IOException {
+ I vertexId = getConf().createVertexId();
+ vertexId.readFields(in);
+ V vertexValue = getConf().createVertexValue();
+ vertexValue.readFields(in);
+ initialize(vertexId, vertexValue);
+
+ int numEdges = in.readInt();
+ edgeList = Lists.newArrayListWithCapacity(numEdges);
+ for (int i = 0; i < numEdges; ++i) {
+ I targetVertexId = getConf().createVertexId();
+ targetVertexId.readFields(in);
+ E edgeValue = getConf().createEdgeValue();
+ edgeValue.readFields(in);
+ edgeList.add(new Edge<I, E>(targetVertexId, edgeValue));
+ }
+
+ readHaltBoolean(in);
+ }
+
+ @Override
+ public final void write(DataOutput out) throws IOException {
+ getId().write(out);
+ getValue().write(out);
+
+ out.writeInt(edgeList.size());
+ for (Edge<I, E> edge : edgeList) {
+ edge.getTargetVertexId().write(out);
+ edge.getValue().write(out);
+ }
+
+ out.writeBoolean(isHalted());
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeReader.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeReader.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeReader.java
new file mode 100644
index 0000000..aba76bd
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeReader.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import java.io.IOException;
+
+/**
+ * Analogous to {@link RecordReader} for edges. Will read the edges
+ * from an input split.
+ *
+ * @param <I> Vertex id
+ * @param <E> Edge data
+ */
+@SuppressWarnings("rawtypes")
+public interface EdgeReader<I extends WritableComparable, E extends Writable> {
+ /**
+ * Use the input split and context to setup reading the edges.
+ * Guaranteed to be called prior to any other function.
+ *
+ * @param inputSplit Input split to be used for reading edges.
+ * @param context Context from the task.
+ * @throws java.io.IOException
+ * @throws InterruptedException
+ */
+ void initialize(InputSplit inputSplit, TaskAttemptContext context) throws
+ IOException, InterruptedException;
+
+ /**
+ *
+ * @return false iff there are no more edges
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ boolean nextEdge() throws IOException, InterruptedException;
+
+ /**
+ * Get the current edge.
+ *
+ * @return the current edge which has been read.
+ * nextEdge() should be called first.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ EdgeWithSource<I, E> getCurrentEdge() throws IOException,
+ InterruptedException;
+
+ /**
+ * Close this {@link EdgeReader} to future operations.
+ *
+ * @throws IOException
+ */
+ void close() throws IOException;
+
+ /**
+ * How much of the input has the {@link EdgeReader} consumed i.e.
+ * has been processed by?
+ *
+ * @return Progress from <code>0.0</code> to <code>1.0</code>.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ float getProgress() throws IOException, InterruptedException;
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/EdgeWithSource.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/EdgeWithSource.java b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeWithSource.java
new file mode 100644
index 0000000..84232c9
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/EdgeWithSource.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * A pair of source vertex id and Edge object (that is,
+ * all the information about an edge).
+ *
+ * @param <I> Vertex id
+ * @param <E> Edge data
+ */
+public class EdgeWithSource<I extends WritableComparable, E extends Writable> {
+ /** Source vertex id. */
+ private final I sourceVertexId;
+ /** Edge. */
+ private final Edge<I, E> edge;
+
+ /**
+ * Constructor.
+ *
+ * @param sourceVertexId Source vertex id
+ * @param edge Edge
+ */
+ public EdgeWithSource(I sourceVertexId, Edge<I, E> edge) {
+ this.sourceVertexId = sourceVertexId;
+ this.edge = edge;
+ }
+
+ /**
+ * Get the source vertex id.
+ *
+ * @return Source vertex id.
+ */
+ public I getSourceVertexId() {
+ return sourceVertexId;
+ }
+
+ /**
+ * Get the edge object.
+ *
+ * @return The edge.
+ */
+ public Edge<I, E> getEdge() {
+ return edge;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/FinishedSuperstepStats.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/FinishedSuperstepStats.java b/giraph-core/src/main/java/org/apache/giraph/graph/FinishedSuperstepStats.java
new file mode 100644
index 0000000..d888d10
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/FinishedSuperstepStats.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.graph;
+
+/**
+ * Immutable results of finishSuperste()
+ */
+public class FinishedSuperstepStats extends VertexEdgeCount {
+ /** Are all the graph vertices halted? */
+ private final boolean allVerticesHalted;
+
+ /**
+ * Constructor.
+ *
+ * @param allVerticesHalted Are all the vertices halted
+ * @param numVertices Number of vertices
+ * @param numEdges Number of edges
+ */
+ public FinishedSuperstepStats(boolean allVerticesHalted,
+ long numVertices,
+ long numEdges) {
+ super(numVertices, numEdges);
+ this.allVerticesHalted = allVerticesHalted;
+ }
+
+ public boolean getAllVerticesHalted() {
+ return allVerticesHalted;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/GiraphInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GiraphInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphInputFormat.java
new file mode 100644
index 0000000..e1cc844
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphInputFormat.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Common interface for {@link VertexInputFormat} and {@link EdgeInputFormat}.
+ */
+public interface GiraphInputFormat {
+ /**
+ * Get the list of input splits for the format.
+ *
+ * @param context The job context
+ * @param numWorkers Number of workers
+ * @return The list of input splits
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ List<InputSplit> getSplits(JobContext context, int numWorkers)
+ throws IOException, InterruptedException;
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/GiraphJob.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GiraphJob.java b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphJob.java
new file mode 100644
index 0000000..ae7e39c
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphJob.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.giraph.bsp.BspInputFormat;
+import org.apache.giraph.bsp.BspOutputFormat;
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.conf.GiraphConstants;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * Generates an appropriate internal {@link Job} for using Giraph in Hadoop.
+ * Uses composition to avoid unwanted {@link Job} methods from exposure
+ * to the user.
+ */
+public class GiraphJob {
+ static {
+ Configuration.addDefaultResource("giraph-site.xml");
+ }
+
+ /** Class logger */
+ private static final Logger LOG = Logger.getLogger(GiraphJob.class);
+ /** Internal delegated job to proxy interface requests for Job */
+ private final DelegatedJob delegatedJob;
+ /** Name of the job */
+ private final String jobName;
+ /** Helper configuration from the job */
+ private final GiraphConfiguration giraphConfiguration;
+
+ /**
+ * Delegated job that simply passes along the class GiraphConfiguration.
+ */
+ private class DelegatedJob extends Job {
+ /** Ensure that for job initiation the super.getConfiguration() is used */
+ private boolean jobInited = false;
+
+ /**
+ * Constructor
+ *
+ * @throws IOException
+ */
+ DelegatedJob() throws IOException { }
+
+ @Override
+ public Configuration getConfiguration() {
+ if (jobInited) {
+ return giraphConfiguration;
+ } else {
+ return super.getConfiguration();
+ }
+ }
+ }
+
+ /**
+ * Constructor that will instantiate the configuration
+ *
+ * @param jobName User-defined job name
+ * @throws IOException
+ */
+ public GiraphJob(String jobName) throws IOException {
+ this(new GiraphConfiguration(), jobName);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param configuration User-defined configuration
+ * @param jobName User-defined job name
+ * @throws IOException
+ */
+ public GiraphJob(Configuration configuration,
+ String jobName) throws IOException {
+ this(new GiraphConfiguration(configuration), jobName);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param giraphConfiguration User-defined configuration
+ * @param jobName User-defined job name
+ * @throws IOException
+ */
+ public GiraphJob(GiraphConfiguration giraphConfiguration,
+ String jobName) throws IOException {
+ this.jobName = jobName;
+ this.giraphConfiguration = giraphConfiguration;
+ this.delegatedJob = new DelegatedJob();
+ }
+
+ /**
+ * Get the configuration from the internal job.
+ *
+ * @return Configuration used by the job.
+ */
+ public GiraphConfiguration getConfiguration() {
+ return giraphConfiguration;
+ }
+
+ /**
+ * Be very cautious when using this method as it returns the internal job
+ * of {@link GiraphJob}. This should only be used for methods that require
+ * access to the actual {@link Job}, i.e. FileInputFormat#addInputPath().
+ *
+ * @return Internal job that will actually be submitted to Hadoop.
+ */
+ public Job getInternalJob() {
+ delegatedJob.jobInited = true;
+ return delegatedJob;
+ }
+ /**
+ * Make sure the configuration is set properly by the user prior to
+ * submitting the job.
+ *
+ * @param conf Configuration to check
+ */
+ private void checkConfiguration(ImmutableClassesGiraphConfiguration conf) {
+ if (conf.getMaxWorkers() < 0) {
+ throw new RuntimeException("checkConfiguration: No valid " +
+ GiraphConstants.MAX_WORKERS);
+ }
+ if (conf.getMinPercentResponded() <= 0.0f ||
+ conf.getMinPercentResponded() > 100.0f) {
+ throw new IllegalArgumentException(
+ "checkConfiguration: Invalid " + conf.getMinPercentResponded() +
+ " for " + GiraphConstants.MIN_PERCENT_RESPONDED);
+ }
+ if (conf.getMinWorkers() < 0) {
+ throw new IllegalArgumentException("checkConfiguration: No valid " +
+ GiraphConstants.MIN_WORKERS);
+ }
+ if (conf.getVertexClass() == null) {
+ throw new IllegalArgumentException("checkConfiguration: Null" +
+ GiraphConstants.VERTEX_CLASS);
+ }
+ if (conf.getVertexInputFormatClass() == null &&
+ conf.getEdgeInputFormatClass() == null) {
+ throw new IllegalArgumentException("checkConfiguration: One of " +
+ GiraphConstants.VERTEX_INPUT_FORMAT_CLASS + " and " +
+ GiraphConstants.EDGE_INPUT_FORMAT_CLASS + " must be non-null");
+ }
+ if (conf.getEdgeInputFormatClass() != null &&
+ !(MutableVertex.class.isAssignableFrom(conf.getVertexClass()))) {
+ throw new IllegalArgumentException("checkConfiguration: EdgeInputFormat" +
+ " only works with mutable vertices");
+ }
+ if (conf.getVertexResolverClass() == null) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("checkConfiguration: No class found for " +
+ GiraphConstants.VERTEX_RESOLVER_CLASS + ", defaulting to " +
+ DefaultVertexResolver.class.getCanonicalName());
+ }
+ }
+ }
+
+
+ /**
+ * Check if the configuration is local. If it is local, do additional
+ * checks due to the restrictions of LocalJobRunner.
+ *
+ * @param conf Configuration
+ */
+ private static void checkLocalJobRunnerConfiguration(
+ GiraphConfiguration conf) {
+ String jobTracker = conf.get("mapred.job.tracker", null);
+ if (!jobTracker.equals("local")) {
+ // Nothing to check
+ return;
+ }
+
+ int maxWorkers = conf.getMaxWorkers();
+ if (maxWorkers != 1) {
+ throw new IllegalArgumentException(
+ "checkLocalJobRunnerConfiguration: When using " +
+ "LocalJobRunner, must have only one worker since " +
+ "only 1 task at a time!");
+ }
+ if (conf.getSplitMasterWorker()) {
+ throw new IllegalArgumentException(
+ "checkLocalJobRunnerConfiguration: When using " +
+ "LocalJobRunner, you cannot run in split master / worker " +
+ "mode since there is only 1 task at a time!");
+ }
+ }
+
+ /**
+ * Check whether a specified int conf value is set and if not, set it.
+ *
+ * @param param Conf value to check
+ * @param defaultValue Assign to value if not set
+ */
+ private void setIntConfIfDefault(String param, int defaultValue) {
+ if (giraphConfiguration.getInt(param, Integer.MIN_VALUE) ==
+ Integer.MIN_VALUE) {
+ giraphConfiguration.setInt(param, defaultValue);
+ }
+ }
+
+ /**
+ * Runs the actual graph application through Hadoop Map-Reduce.
+ *
+ * @param verbose If true, provide verbose output, false otherwise
+ * @return True if success, false otherwise
+ * @throws ClassNotFoundException
+ * @throws InterruptedException
+ * @throws IOException
+ */
+ public final boolean run(boolean verbose)
+ throws IOException, InterruptedException, ClassNotFoundException {
+ // Most users won't hit this hopefully and can set it higher if desired
+ setIntConfIfDefault("mapreduce.job.counters.limit", 512);
+
+ // Capacity scheduler-specific settings. These should be enough for
+ // a reasonable Giraph job
+ setIntConfIfDefault("mapred.job.map.memory.mb", 1024);
+ setIntConfIfDefault("mapred.job.reduce.memory.mb", 1024);
+
+ // Speculative execution doesn't make sense for Giraph
+ giraphConfiguration.setBoolean(
+ "mapred.map.tasks.speculative.execution", false);
+
+ // Set the ping interval to 5 minutes instead of one minute
+ // (DEFAULT_PING_INTERVAL)
+ Client.setPingInterval(giraphConfiguration, 60000 * 5);
+
+ // Should work in MAPREDUCE-1938 to let the user jars/classes
+ // get loaded first
+ giraphConfiguration.setBoolean("mapreduce.user.classpath.first", true);
+ giraphConfiguration.setBoolean("mapreduce.job.user.classpath.first", true);
+
+ // If the checkpoint frequency is 0 (no failure handling), set the max
+ // tasks attempts to be 0 to encourage faster failure of unrecoverable jobs
+ if (giraphConfiguration.getCheckpointFrequency() == 0) {
+ int oldMaxTaskAttempts = giraphConfiguration.getMaxTaskAttempts();
+ giraphConfiguration.setMaxTaskAttempts(0);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("run: Since checkpointing is disabled (default), " +
+ "do not allow any task retries (setting " +
+ GiraphConstants.MAX_TASK_ATTEMPTS + " = 0, " +
+ "old value = " + oldMaxTaskAttempts + ")");
+ }
+ }
+
+ // Set the job properties, check them, and submit the job
+ ImmutableClassesGiraphConfiguration immutableClassesGiraphConfiguration =
+ new ImmutableClassesGiraphConfiguration(giraphConfiguration);
+ checkConfiguration(immutableClassesGiraphConfiguration);
+ checkLocalJobRunnerConfiguration(immutableClassesGiraphConfiguration);
+ Job submittedJob = new Job(immutableClassesGiraphConfiguration, jobName);
+ if (submittedJob.getJar() == null) {
+ submittedJob.setJarByClass(GiraphJob.class);
+ }
+ submittedJob.setNumReduceTasks(0);
+ submittedJob.setMapperClass(GraphMapper.class);
+ submittedJob.setInputFormatClass(BspInputFormat.class);
+ submittedJob.setOutputFormatClass(BspOutputFormat.class);
+ return submittedJob.waitForCompletion(verbose);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTransferRegulator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTransferRegulator.java b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTransferRegulator.java
new file mode 100644
index 0000000..8bec886
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTransferRegulator.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.graph;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.giraph.graph.partition.PartitionOwner;
+import com.google.common.collect.Maps;
+import java.util.Map;
+
+
+/** Utility class to manage data transfers from
+ * a local worker reading InputSplits.
+ * Currently, this class measures # of vertices and edges
+ * per outgoing Collection of graph data (destined for a
+ * particular Partition and remote worker node, preselected
+ * by the master.)
+ *
+ * TODO: implement defaults and configurable options for
+ * measuring the size of input <V> or <E> data
+ * per read vertex, and setting limits on totals per outgoing
+ * graph data Collection etc. (See GIRAPH-260)
+ */
+public class GiraphTransferRegulator {
+ /** Maximum vertices to read from an InputSplit locally that are
+ * to be routed to a remote worker, before sending them. */
+ public static final String MAX_VERTICES_PER_TRANSFER =
+ "giraph.maxVerticesPerTransfer";
+ /** Default maximum number of vertices per
+ * temp partition before sending. */
+ public static final int MAX_VERTICES_PER_TRANSFER_DEFAULT = 10000;
+ /**
+ * Maximum edges to read from an InputSplit locally that are
+ * to be routed to a remote worker, before sending them.
+ */
+ public static final String MAX_EDGES_PER_TRANSFER =
+ "giraph.maxEdgesPerTransfer";
+ /** Default maximum number of vertices per
+ * temp partition before sending. */
+ public static final int MAX_EDGES_PER_TRANSFER_DEFAULT = 80000;
+
+ /** Internal state to measure when
+ * the next data transfer of a Collection
+ * of vertices read by the local worker that
+ * owns this regulator is ready to be sent
+ * to the remote worker node that the master
+ * has assigned the vertices to */
+ private Map<Integer, Integer> edgeAccumulator;
+
+ /** Internal state to measure when
+ * the next data transfer of a Collection
+ * of vertices read by the local worker that
+ * owns this regulator is ready to be sent
+ * to the remote worker node that the master
+ * has assigned the vertices to */
+ private Map<Integer, Integer> vertexAccumulator;
+
+ /** Number of vertices per data transfer */
+ private final int maxVerticesPerTransfer;
+
+ /** Number of edges per data transfer */
+ private final int maxEdgesPerTransfer;
+
+ /** Vertex count total for this InputSplit */
+ private long totalVertexCount;
+
+ /** Edge count total for this InputSplit */
+ private long totalEdgeCount;
+
+ /** Default constructor
+ * @param conf the Configuration for this job
+ */
+ public GiraphTransferRegulator(Configuration conf) {
+ vertexAccumulator = Maps.<Integer, Integer>newHashMap();
+ edgeAccumulator = Maps.<Integer, Integer>newHashMap();
+ maxVerticesPerTransfer = conf.getInt(
+ MAX_VERTICES_PER_TRANSFER,
+ MAX_VERTICES_PER_TRANSFER_DEFAULT);
+ maxEdgesPerTransfer = conf.getInt(
+ MAX_EDGES_PER_TRANSFER,
+ MAX_EDGES_PER_TRANSFER_DEFAULT);
+ totalEdgeCount = 0;
+ totalVertexCount = 0;
+ }
+
+ /** Is this outbound data Collection full,
+ * and ready to transfer?
+ * @param owner the partition owner for the outbound data
+ * @return 'true' if the temp partition data is ready to transfer
+ */
+ public boolean transferThisPartition(PartitionOwner owner) {
+ final int partitionId = owner.getPartitionId();
+ if (getEdgesForPartition(partitionId) >=
+ maxEdgesPerTransfer ||
+ getVerticesForPartition(partitionId) >=
+ maxVerticesPerTransfer) {
+ vertexAccumulator.put(partitionId, 0);
+ edgeAccumulator.put(partitionId, 0);
+ return true;
+ }
+ return false;
+ }
+
+ /** get current vertex count for a given Collection of
+ * data soon to be transfered to its permanent home.
+ * @param partId the partition id to check the count on.
+ * @return the count of vertices.
+ */
+ private int getVerticesForPartition(final int partId) {
+ return vertexAccumulator.get(partId) == null ?
+ 0 : vertexAccumulator.get(partId);
+ }
+
+ /** get current edge count for a given Collection of
+ * data soon to be transfered to its permanent home.
+ * @param partId the partition id to check the count on.
+ * @return the count of edges.
+ */
+ private int getEdgesForPartition(final int partId) {
+ return edgeAccumulator.get(partId) == null ?
+ 0 : edgeAccumulator.get(partId);
+ }
+
+ /** Clear storage to reset for reading new InputSplit */
+ public void clearCounters() {
+ totalEdgeCount = 0;
+ totalVertexCount = 0;
+ vertexAccumulator.clear();
+ edgeAccumulator.clear();
+ }
+
+ /** Increment V & E counts for new vertex read, store values
+ * for that outgoing _temporary_ Partition, which shares the
+ * Partition ID for the actual remote Partition the collection
+ * will eventually be processed in.
+ * @param partitionOwner the owner of the Partition this data
+ * will eventually belong to.
+ * @param vertex the vertex to extract counts from.
+ * @param <I> the vertex id type.
+ * @param <V> the vertex value type.
+ * @param <E> the edge value type.
+ * @param <M> the message value type.
+ */
+ public <I extends WritableComparable, V extends Writable,
+ E extends Writable, M extends Writable> void
+ incrementCounters(PartitionOwner partitionOwner,
+ Vertex<I, V, E, M> vertex) {
+ final int id = partitionOwner.getPartitionId();
+ // vertex counts
+ vertexAccumulator
+ .put(id, getVerticesForPartition(id) + 1);
+ totalVertexCount++;
+ // edge counts
+ totalEdgeCount += vertex.getNumEdges();
+ edgeAccumulator.put(id, getEdgesForPartition(id) +
+ vertex.getNumEdges());
+ }
+
+ /** Getter for MAX edge count to initiate a transfer
+ * @return max edge count per transfer */
+ public long getMaxEdgesPerTransfer() {
+ return maxEdgesPerTransfer;
+ }
+
+ /** Getter for MAX vertex count to initiate a transfer
+ * @return max edge count per transfer */
+ public long getMaxVerticesPerTransfer() {
+ return maxVerticesPerTransfer;
+ }
+
+ /** Getter for total edge count for the current InputSplit
+ * @return the # of total edges counted in this InputSplit */
+ public long getTotalEdges() {
+ return totalEdgeCount;
+ }
+
+ /** Getter for total vetex count for the current InputSplit
+ * @return the total # of vertices counted in this InputSplit */
+ public long getTotalVertices() {
+ return totalVertexCount;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTypeValidator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTypeValidator.java b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTypeValidator.java
new file mode 100644
index 0000000..f0b4b77
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GiraphTypeValidator.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import org.apache.giraph.utils.ReflectionUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+import java.lang.reflect.Type;
+import java.util.List;
+
+/**
+ * GiraphTypeValidator attempts to verify the consistency of
+ * user-chosen InputFormat, OutputFormat, and Vertex type
+ * parameters before the job run actually begins.
+ *
+ * @param <I> the Vertex ID type
+ * @param <V> the Vertex Value type
+ * @param <E> the Edge Value type
+ * @param <M> the Message type
+ */
+public class GiraphTypeValidator<I extends WritableComparable,
+ V extends Writable, E extends Writable, M extends Writable> {
+ /**
+ * Class logger object.
+ */
+ private static Logger LOG =
+ Logger.getLogger(GiraphTypeValidator.class);
+
+ /** I param vertex index in classList */
+ private static final int ID_PARAM_INDEX = 0;
+ /** V param vertex index in classList */
+ private static final int VALUE_PARAM_INDEX = 1;
+ /** E param vertex index in classList */
+ private static final int EDGE_PARAM_INDEX = 2;
+ /** M param vertex index in classList */
+ private static final int MSG_PARAM_INDEX = 3;
+ /** M param vertex combiner index in classList */
+ private static final int MSG_COMBINER_PARAM_INDEX = 1;
+
+ /** Vertex Index Type */
+ private Type vertexIndexType;
+ /** Vertex Index Type */
+ private Type vertexValueType;
+ /** Vertex Index Type */
+ private Type edgeValueType;
+ /** Vertex Index Type */
+ private Type messageValueType;
+
+ /**
+ * The Configuration object for use in the validation test.
+ */
+ private Configuration conf;
+
+ /**
+ * Constructor to execute the validation test, throws
+ * unchecked exception to end job run on failure.
+ *
+ * @param conf the Configuration for this run.
+ */
+ public GiraphTypeValidator(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Make sure that all registered classes have matching types. This
+ * is a little tricky due to type erasure, cannot simply get them from
+ * the class type arguments. Also, set the vertex index, vertex value,
+ * edge value and message value classes.
+ */
+ public void validateClassTypes() {
+ Class<? extends Vertex<I, V, E, M>> vertexClass =
+ BspUtils.<I, V, E, M>getVertexClass(conf);
+ List<Class<?>> classList = ReflectionUtils.getTypeArguments(
+ Vertex.class, vertexClass);
+ vertexIndexType = classList.get(ID_PARAM_INDEX);
+ vertexValueType = classList.get(VALUE_PARAM_INDEX);
+ edgeValueType = classList.get(EDGE_PARAM_INDEX);
+ messageValueType = classList.get(MSG_PARAM_INDEX);
+ verifyVertexInputFormatGenericTypes();
+ verifyVertexOutputFormatGenericTypes();
+ verifyVertexResolverGenericTypes();
+ verifyVertexCombinerGenericTypes();
+ }
+
+ /** Verify matching generic types in VertexInputFormat. */
+ private void verifyVertexInputFormatGenericTypes() {
+ Class<? extends VertexInputFormat<I, V, E, M>> vertexInputFormatClass =
+ BspUtils.<I, V, E, M>getVertexInputFormatClass(conf);
+ List<Class<?>> classList =
+ ReflectionUtils.getTypeArguments(
+ VertexInputFormat.class, vertexInputFormatClass);
+ if (classList.get(ID_PARAM_INDEX) == null) {
+ LOG.warn("Input format vertex index type is not known");
+ } else if (!vertexIndexType.equals(classList.get(ID_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Vertex index types don't match, " +
+ "vertex - " + vertexIndexType +
+ ", vertex input format - " + classList.get(ID_PARAM_INDEX));
+ }
+ if (classList.get(VALUE_PARAM_INDEX) == null) {
+ LOG.warn("Input format vertex value type is not known");
+ } else if (!vertexValueType.equals(classList.get(VALUE_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Vertex value types don't match, " +
+ "vertex - " + vertexValueType +
+ ", vertex input format - " + classList.get(VALUE_PARAM_INDEX));
+ }
+ if (classList.get(EDGE_PARAM_INDEX) == null) {
+ LOG.warn("Input format edge value type is not known");
+ } else if (!edgeValueType.equals(classList.get(EDGE_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Edge value types don't match, " +
+ "vertex - " + edgeValueType +
+ ", vertex input format - " + classList.get(EDGE_PARAM_INDEX));
+ }
+ }
+
+ /** If there is a combiner type, verify its generic params match the job. */
+ private void verifyVertexCombinerGenericTypes() {
+ Class<? extends Combiner<I, M>> vertexCombinerClass =
+ BspUtils.<I, M>getCombinerClass(conf);
+ if (vertexCombinerClass != null) {
+ List<Class<?>> classList =
+ ReflectionUtils.getTypeArguments(
+ Combiner.class, vertexCombinerClass);
+ if (!vertexIndexType.equals(classList.get(ID_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Vertex index types don't match, " +
+ "vertex - " + vertexIndexType +
+ ", vertex combiner - " + classList.get(ID_PARAM_INDEX));
+ }
+ if (!messageValueType.equals(classList.get(MSG_COMBINER_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Message value types don't match, " +
+ "vertex - " + messageValueType +
+ ", vertex combiner - " + classList.get(MSG_COMBINER_PARAM_INDEX));
+ }
+ }
+ }
+
+ /** Verify that the output format's generic params match the job. */
+ private void verifyVertexOutputFormatGenericTypes() {
+ Class<? extends VertexOutputFormat<I, V, E>>
+ vertexOutputFormatClass =
+ BspUtils.<I, V, E>getVertexOutputFormatClass(conf);
+ if (vertexOutputFormatClass != null) {
+ List<Class<?>> classList =
+ ReflectionUtils.getTypeArguments(
+ VertexOutputFormat.class, vertexOutputFormatClass);
+ if (classList.get(ID_PARAM_INDEX) == null) {
+ LOG.warn("Output format vertex index type is not known");
+ } else if (!vertexIndexType.equals(classList.get(ID_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Vertex index types don't match, " +
+ "vertex - " + vertexIndexType +
+ ", vertex output format - " + classList.get(ID_PARAM_INDEX));
+ }
+ if (classList.get(VALUE_PARAM_INDEX) == null) {
+ LOG.warn("Output format vertex value type is not known");
+ } else if (!vertexValueType.equals(classList.get(VALUE_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Vertex value types don't match, " +
+ "vertex - " + vertexValueType +
+ ", vertex output format - " + classList.get(VALUE_PARAM_INDEX));
+ }
+ if (classList.get(EDGE_PARAM_INDEX) == null) {
+ LOG.warn("Output format edge value type is not known");
+ } else if (!edgeValueType.equals(classList.get(EDGE_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Edge value types don't match, " +
+ "vertex - " + edgeValueType +
+ ", vertex output format - " + classList.get(EDGE_PARAM_INDEX));
+ }
+ }
+ }
+
+ /** If there is a vertex resolver,
+ * validate the generic parameter types. */
+ private void verifyVertexResolverGenericTypes() {
+ Class<? extends VertexResolver<I, V, E, M>>
+ vrClass = BspUtils.<I, V, E, M>getVertexResolverClass(conf);
+ if (!DefaultVertexResolver.class.isAssignableFrom(vrClass)) {
+ return;
+ }
+ Class<? extends DefaultVertexResolver<I, V, E, M>>
+ dvrClass =
+ (Class<? extends DefaultVertexResolver<I, V, E, M>>) vrClass;
+ List<Class<?>> classList =
+ ReflectionUtils.getTypeArguments(
+ DefaultVertexResolver.class, dvrClass);
+ if (classList.get(ID_PARAM_INDEX) != null &&
+ !vertexIndexType.equals(classList.get(ID_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Vertex index types don't match, " +
+ "vertex - " + vertexIndexType +
+ ", vertex resolver - " + classList.get(ID_PARAM_INDEX));
+ }
+ if (classList.get(VALUE_PARAM_INDEX) != null &&
+ !vertexValueType.equals(classList.get(VALUE_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Vertex value types don't match, " +
+ "vertex - " + vertexValueType +
+ ", vertex resolver - " + classList.get(VALUE_PARAM_INDEX));
+ }
+ if (classList.get(EDGE_PARAM_INDEX) != null &&
+ !edgeValueType.equals(classList.get(EDGE_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Edge value types don't match, " +
+ "vertex - " + edgeValueType +
+ ", vertex resolver - " + classList.get(EDGE_PARAM_INDEX));
+ }
+ if (classList.get(MSG_PARAM_INDEX) != null &&
+ !messageValueType.equals(classList.get(MSG_PARAM_INDEX))) {
+ throw new IllegalArgumentException(
+ "checkClassTypes: Message value types don't match, " +
+ "vertex - " + messageValueType +
+ ", vertex resolver - " + classList.get(MSG_PARAM_INDEX));
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/giraph/blob/45851391/giraph-core/src/main/java/org/apache/giraph/graph/GlobalStats.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GlobalStats.java b/giraph-core/src/main/java/org/apache/giraph/graph/GlobalStats.java
new file mode 100644
index 0000000..aa03c4b
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/graph/GlobalStats.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.graph;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.giraph.graph.partition.PartitionStats;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Aggregated stats by the master.
+ */
+public class GlobalStats implements Writable {
+ /** All vertices in the application */
+ private long vertexCount = 0;
+ /** All finished vertices in the last superstep */
+ private long finishedVertexCount = 0;
+ /** All edges in the last superstep */
+ private long edgeCount = 0;
+ /** All messages sent in the last superstep */
+ private long messageCount = 0;
+ /** Whether the computation should be halted */
+ private boolean haltComputation = false;
+
+ /**
+ * Add the stats of a partition to the global stats.
+ *
+ * @param partitionStats Partition stats to be added.
+ */
+ public void addPartitionStats(PartitionStats partitionStats) {
+ this.vertexCount += partitionStats.getVertexCount();
+ this.finishedVertexCount += partitionStats.getFinishedVertexCount();
+ this.edgeCount += partitionStats.getEdgeCount();
+ }
+
+ public long getVertexCount() {
+ return vertexCount;
+ }
+
+ public long getFinishedVertexCount() {
+ return finishedVertexCount;
+ }
+
+ public long getEdgeCount() {
+ return edgeCount;
+ }
+
+ public long getMessageCount() {
+ return messageCount;
+ }
+
+ public boolean getHaltComputation() {
+ return haltComputation;
+ }
+
+ public void setHaltComputation(boolean value) {
+ haltComputation = value;
+ }
+
+ /**
+ * Add messages to the global stats.
+ *
+ * @param messageCount Number of messages to be added.
+ */
+ public void addMessageCount(long messageCount) {
+ this.messageCount += messageCount;
+ }
+
+ @Override
+ public void readFields(DataInput input) throws IOException {
+ vertexCount = input.readLong();
+ finishedVertexCount = input.readLong();
+ edgeCount = input.readLong();
+ messageCount = input.readLong();
+ haltComputation = input.readBoolean();
+ }
+
+ @Override
+ public void write(DataOutput output) throws IOException {
+ output.writeLong(vertexCount);
+ output.writeLong(finishedVertexCount);
+ output.writeLong(edgeCount);
+ output.writeLong(messageCount);
+ output.writeBoolean(haltComputation);
+ }
+
+ @Override
+ public String toString() {
+ return "(vtx=" + vertexCount + ",finVtx=" +
+ finishedVertexCount + ",edges=" + edgeCount + ",msgCount=" +
+ messageCount + ",haltComputation=" + haltComputation + ")";
+ }
+}